效果对比图

30台云主机用TCP短链接(每一个请求建立一个连接,使用完释放连接)对一个云主机进行压测:

netperf -n 8 -l 60 -H 10.1.7.233 -t TCP_CRR -- -r32,1024

before_rps.jpg

设置rps之前,所有的中断由CPU0 处理,出现瓶颈。
中断全部由CPU0处理

after_rps.jpg

设置rps之后,每一个CPU都有负载,CPU0不再是瓶颈,继续加压网卡流量打满。
每一个cpu都有负载

网卡软中断绑定irq设置

使用lspci -vvv 查看网卡时候支持多个收发队列,关键字为MSI-X,并且有Enable+

...............
Capabilities: [58] MSI: Enable- Count=1/16 Maskable- 64bit+
    Address: 0000000000000000  Data: 0000
Capabilities: [a0] MSI-X: Enable+ Count=9 Masked-
    Vector table: BAR=0 offset=0000c000
    PBA: BAR=0 offset=0000e000
..................

通过 cat /proc/interrupts |grep eth0 查看相应的网络中断号,对每一个中断号绑定到指定的CPU。

rps 设置

解决网卡队列数目和cpu核心数不一致的情况,相当于软件模拟的中断分配,对IDC给的云主机有比较好的效果
使用/sys/class/net/eth0/queues/rx-*/rps_cpus对CPU进行绑定。需要2.6.32以上版本的内核支持。

设置脚本

#!/usr/bin/env python
# encoding: utf-8
def smp_affinity():
    #每8位插入一个","
    def strConv(s):
        import re
        s = str(s)
        while True:
            (s, count) = re.subn(r"(\w)(\w{8})((:?,\w{8})*)$", r"\1,\2\3", s)
            if count == 0:
                break
        return s
    import re
    regex = re.compile(r'(eth[0-9]-(tx|rx|txrx|fp|[-]?[0-9]))', re.IGNORECASE)
    num_cpus = int(
        len([x for x in open('/proc/cpuinfo') if x.startswith('processor')]))
    net_dev_list = [x.split()[-1]
                    for x in open('/proc/interrupts') if 'eth' in x]
    net_dev_irq = [x.split()[0].strip(':')
                   for x in open('/proc/interrupts') if 'eth' in x]
    if re.search(regex, ','.join(net_dev_list)):
        smp_affinity = ['/proc/irq/%s/smp_affinity' % x for x in net_dev_irq]
        bit_cpus = [strConv(str(hex(1 << x)).replace('0x', '')) for x in range(
            num_cpus)][::-1] * (len(net_dev_irq) / num_cpus + 1)
    else:
        smp_affinity = ['/proc/irq/%s/smp_affinity' % x for x in net_dev_irq]
        bit_cpus = [
            strConv(hex(int('1' * num_cpus, 2)).replace('0x', ''))] * len(smp_affinity)
    smp_affinity_list = zip(bit_cpus, smp_affinity)
    active_nics = list(set([x.split('-')[0] for x in net_dev_list]))
    if(len(active_nics)):
        from glob import glob
        rps_flow_cnt = 0
        for nic in active_nics:
            rps = glob('/sys/class/net/%s/queues/rx-*/rps_cpus' % nic)
            if len(rps):
                if len(rps) != num_cpus:
                    bit_cpus = [
                        strConv(hex(int('1' * num_cpus, 2)).replace('0x', ''))] * len(rps)
                    smp_affinity_list.extend(
                        zip([4096] * len(rps), [x.replace('rps_cpus', 'rps_flow_cnt') for x in rps]))
                    rps_flow_cnt = rps_flow_cnt + 4096 * len(rps)
                else:
                    bit_cpus = [0] * len(rps)
                smp_affinity_list.extend(zip(bit_cpus, rps))
            xps = glob('/sys/class/net/%s/queues/tx-*/xps_cpus' % nic)
            if len(xps) >= num_cpus:
                bit_cpus = [
                    strConv(str(hex(1 << x)).replace('0x', '')) for x in range(num_cpus)][::-1]
                smp_affinity_list.extend(zip(bit_cpus, rps))
            print 'ethtool -C %s  rx-usecs 333' % nic
        if rps_flow_cnt != 0:
            smp_affinity_list.append(
                (rps_flow_cnt, '/proc/sys/net/core/rps_sock_flow_entries'))
    for smp in smp_affinity_list:
        print 'echo %s > %s' % (smp[0], smp[1])
if __name__ == '__main__':
    smp_affinity()

参考连接

https://github.com/fastos/fastsocket/blob/master/scripts/nic.sh
Red Hat Customer Portal