SO_REUSEPORT 之 TCP负载均衡验证

首先启动两个tcp server，代码里开启 SO_REUSEPORT

[my_test@localhost test]$ ./tcp_server_reuseport &
[1] 1864
[my_test@localhost test]$ Server listening on port 8888[my_test@localhost test]$ ./tcp_server_reuseport &
[2] 1865
[my_test@localhost test]$ Server listening on port 8888[my_test@localhost test]$ ps -ef|grep tcp_server_reuseport
my_test         1864    1443  0 23:11 pts/0    00:00:00 ./tcp_server_reuseport
my_test         1865    1443  0 23:11 pts/0    00:00:00 ./tcp_server_reuseport

启动10条客户端连接：

[my_test@localhost test]$ ./tcp_client_reuseport
Message sent
Server response: Welcome to the server! Server PID: 1865Message sent
Server response: Welcome to the server! Server PID: 1864Message sent
Server response: Welcome to the server! Server PID: 1865Message sent
Server response: Welcome to the server! Server PID: 1864Message sent
Server response: Welcome to the server! Server PID: 1864Message sent
Server response: Welcome to the server! Server PID: 1865Message sent
Server response: Welcome to the server! Server PID: 1865Message sent
Server response: Welcome to the server! Server PID: 1865Message sent
Server response: Welcome to the server! Server PID: 1864Message sent
Server response: Welcome to the server! Server PID: 1864

从回复的server 进程id 可见，负载均衡做的很好。

追踪内核调用链（内核版本5.10.216 x86_64）：

在server端收到client的SYN连接请求时会触发 __inet_lookup_listener -> inet_lhash2_lookup -> lookup_reuseport -> reuseport_select_sock 通过哈希选择一个Listen Socket来处理这个连接请求。

(gdb) b reuseport_select_sock
Breakpoint 1 at 0xffffffff8199ac80: file net/core/sock_reuseport.c, line 277.
(gdb) c
Continuing.
[New Thread 2061]
[Switching to Thread 2061]Thread 293 hit Breakpoint 1, reuseport_select_sock (sk=sk@entry=0xffff888038c90000, hash=317205834, skb=skb@entry=0xffff88800a90dce0,hdr_len=hdr_len@entry=40) at net/core/sock_reuseport.c:277
277     net/core/sock_reuseport.c: No such file or directory.
(gdb) bt
#0  reuseport_select_sock (sk=sk@entry=0xffff888038c90000, hash=317205834, skb=skb@entry=0xffff88800a90dce0, hdr_len=hdr_len@entry=40)at net/core/sock_reuseport.c:277
#1  0xffffffff81a03ff8 in lookup_reuseport (hnum=8888, daddr=0, sport=21715, saddr=251789322, doff=40, skb=0xffff88800a90dce0,sk=0xffff888038c90000, net=0xffffffff82a1ac40 <init_net>) at net/ipv4/inet_hashtables.c:265
#2  lookup_reuseport (hnum=8888, daddr=0, sport=21715, saddr=251789322, doff=40, skb=0xffff88800a90dce0, sk=0xffff888038c90000,net=0xffffffff82a1ac40 <init_net>) at net/ipv4/inet_hashtables.c:255
#3  inet_lhash2_lookup (net=net@entry=0xffffffff82a1ac40 <init_net>, ilb2=<optimized out>, skb=skb@entry=0xffff88800a90dce0,doff=doff@entry=40, saddr=saddr@entry=251789322, sport=sport@entry=21715, daddr=0, hnum=8888, dif=2, sdif=0)at net/ipv4/inet_hashtables.c:293
#4  0xffffffff81a042f3 in __inet_lookup_listener (net=net@entry=0xffffffff82a1ac40 <init_net>,hashinfo=hashinfo@entry=0xffffffff832a69c0 <tcp_hashinfo>, skb=skb@entry=0xffff88800a90dce0, doff=doff@entry=40,saddr=saddr@entry=251789322, sport=sport@entry=21715, daddr=<optimized out>, hnum=8888, dif=2, sdif=0)at net/ipv4/inet_hashtables.c:361
#5  0xffffffff81a25978 in __inet_lookup (hashinfo=0xffffffff832a69c0 <tcp_hashinfo>, sdif=0, refcounted=<synthetic pointer>, dif=2,dport=<optimized out>, daddr=251789322, sport=<optimized out>, saddr=251789322, doff=<optimized out>, skb=0xffff88800a90dce0,net=0xffffffff82a1ac40 <init_net>) at ./include/net/inet_hashtables.h:343
#6  __inet_lookup_skb (hashinfo=0xffffffff832a69c0 <tcp_hashinfo>, sdif=0, refcounted=<synthetic pointer>, dport=<optimized out>,sport=<optimized out>, doff=<optimized out>, skb=0xffff88800a90dce0) at ./include/net/inet_hashtables.h:379
#7  tcp_v4_rcv (skb=0xffff88800a90dce0) at net/ipv4/tcp_ipv4.c:1984
#8  0xffffffff819f922b in ip_protocol_deliver_rcu (net=0xffffffff82a1ac40 <init_net>, skb=0xffff88800a90dce0,protocol=<optimized out>) at net/ipv4/ip_input.c:204
#9  0xffffffff819f93ef in ip_local_deliver_finish (net=<optimized out>, sk=<optimized out>, skb=<optimized out>)at ./include/linux/skbuff.h:2533
#10 0xffffffff819f94fa in NF_HOOK (sk=0x0 <fixed_percpu_data>, pf=2 '\002', hook=1, in=<optimized out>, out=0x0 <fixed_percpu_data>,okfn=0xffffffff819f93b0 <ip_local_deliver_finish>, skb=0xffff88800a90dce0, net=0xffffffff82a1ac40 <init_net>)at ./include/linux/netfilter.h:296
#11 NF_HOOK (pf=2 '\002', sk=0x0 <fixed_percpu_data>, out=0x0 <fixed_percpu_data>, okfn=0xffffffff819f93b0 <ip_local_deliver_finish>,in=<optimized out>, skb=0xffff88800a90dce0, net=0xffffffff82a1ac40 <init_net>, hook=1) at ./include/linux/netfilter.h:290
#12 ip_local_deliver (skb=0xffff88800a90dce0) at net/ipv4/ip_input.c:252
#13 0xffffffff819f95e3 in NF_HOOK (sk=0x0 <fixed_percpu_data>, pf=2 '\002', hook=0, in=0xffff88800391b000,out=0x0 <fixed_percpu_data>, okfn=0xffffffff819f8c40 <ip_rcv_finish>, skb=0xffff88800a90dce0, net=0xffffffff82a1ac40 <init_net>)at ./include/linux/netfilter.h:296
#14 NF_HOOK (pf=2 '\002', sk=0x0 <fixed_percpu_data>, out=0x0 <fixed_percpu_data>, okfn=0xffffffff819f8c40 <ip_rcv_finish>,
--Type <RET> for more, q to quit, c to continue without paging--in=0xffff88800391b000, skb=0xffff88800a90dce0, net=0xffffffff82a1ac40 <init_net>, hook=0) at ./include/linux/netfilter.h:290
#15 ip_rcv (skb=0xffff88800a90dce0, dev=0xffff88800391b000, pt=<optimized out>, orig_dev=<optimized out>) at net/ipv4/ip_input.c:551
#16 0xffffffff819714a4 in __netif_receive_skb_one_core (skb=<optimized out>, pfmemalloc=<optimized out>) at net/core/dev.c:5375
#17 0xffffffff819716e9 in process_backlog (napi=0xffff88807dc2e3d0, quota=64) at net/core/dev.c:6396
#18 0xffffffff81972fce in napi_poll (repoll=0xffffc90000003f60, n=0xffff88807dc2e3d0) at net/core/dev.c:6847
#19 net_rx_action (h=<optimized out>) at net/core/dev.c:6917
#20 0xffffffff820000b7 in __do_softirq () at kernel/softirq.c:298
#21 0xffffffff81e0106f in asm_call_on_stack () at arch/x86/entry/entry_64.S:801
#22 0xffffffff81022372 in __run_on_irqstack (func=<optimized out>) at ./arch/x86/include/asm/irq_stack.h:26
#23 run_on_irqstack_cond (regs=0x0 <fixed_percpu_data>, func=<optimized out>) at ./arch/x86/include/asm/irq_stack.h:77
#24 do_softirq_own_stack () at arch/x86/kernel/irq_64.c:77
#25 0xffffffff8106c7aa in do_softirq () at kernel/softirq.c:343
#26 0xffffffff8106c7fa in do_softirq () at ./arch/x86/include/asm/preempt.h:26
#27 __local_bh_enable_ip (ip=ip@entry=18446744071589316362, cnt=cnt@entry=512) at kernel/softirq.c:195
#28 0xffffffff819fc71b in local_bh_enable () at ./include/linux/bottom_half.h:32
#29 rcu_read_unlock_bh () at ./include/linux/rcupdate.h:806
#30 ip_finish_output2 (net=<optimized out>, sk=<optimized out>, skb=<optimized out>) at net/ipv4/ip_output.c:238
#31 0xffffffff819feee8 in ip_finish_output (skb=0xffff88800a90dce0, sk=0xffff888038c92bc0, net=0xffffffff82a1ac40 <init_net>)at net/ipv4/ip_output.c:325
#32 NF_HOOK_COND (pf=2 '\002', hook=4, okfn=0xffffffff819fd670 <ip_finish_output>, cond=<optimized out>, out=<optimized out>,in=<optimized out>, skb=0xffff88800a90dce0, sk=0xffff888038c92bc0, net=0xffffffff82a1ac40 <init_net>)at ./include/linux/netfilter.h:285
#33 ip_output (net=0xffffffff82a1ac40 <init_net>, sk=0xffff888038c92bc0, skb=0xffff88800a90dce0) at net/ipv4/ip_output.c:439
#34 0xffffffff819fe982 in __ip_queue_xmit (sk=0xffff888038c92bc0, skb=0xffff88800a90dce0, fl=0xffff888038c92f20, tos=<optimized out>)at net/ipv4/ip_output.c:540
#35 0xffffffff819febbc in ip_queue_xmit (sk=<optimized out>, skb=<optimized out>, fl=<optimized out>) at ./include/net/inet_sock.h:302
#36 0xffffffff81a1bef6 in __tcp_transmit_skb (sk=sk@entry=0xffff888038c92bc0, skb=0xffff88800a90dce0, skb@entry=0xffff88800a90dc00,clone_it=clone_it@entry=1, gfp_mask=<optimized out>, rcv_nxt=<optimized out>) at net/ipv4/tcp_output.c:1407
#37 0xffffffff81a1c8ed in tcp_transmit_skb (gfp_mask=<optimized out>, clone_it=1, skb=0xffff88800a90dc00, sk=0xffff888038c92bc0)at ./include/linux/tcp.h:439
#38 tcp_connect (sk=sk@entry=0xffff888038c92bc0) at net/ipv4/tcp_output.c:3888
#39 0xffffffff81a22c90 in tcp_v4_connect (sk=0xffff888038c92bc0, uaddr=0xffffc90000d43e88, addr_len=<optimized out>)at net/ipv4/tcp_ipv4.c:314
--Type <RET> for more, q to quit, c to continue without paging--
#40 0xffffffff81a3d87c in __inet_stream_connect (sock=sock@entry=0xffff88803652f000, uaddr=0x12e82d4a,uaddr@entry=0xffffc90000d43e88, addr_len=177265888, addr_len@entry=16, flags=40, flags@entry=2, is_sendmsg=21715,is_sendmsg@entry=0) at net/ipv4/af_inet.c:666
#41 0xffffffff81a3db41 in inet_stream_connect (sock=0xffff88803652f000, uaddr=0xffffc90000d43e88, addr_len=16, flags=2)at net/ipv4/af_inet.c:730
#42 0xffffffff8194bdc5 in __sys_connect (fd=<optimized out>, uservaddr=0x7ffe8421f910, addrlen=16) at net/socket.c:1882
#43 0xffffffff8194be01 in __do_sys_connect (addrlen=<optimized out>, uservaddr=<optimized out>, fd=<optimized out>)at net/socket.c:1892
#44 __se_sys_connect (addrlen=<optimized out>, uservaddr=<optimized out>, fd=<optimized out>) at net/socket.c:1889
#45 __x64_sys_connect (regs=<optimized out>) at net/socket.c:1889
#46 0xffffffff81c38090 in do_syscall_64 (nr=<optimized out>, regs=0xffffc90000d43f58) at arch/x86/entry/common.c:46
#47 0xffffffff81e0011f in entry_SYSCALL_64 () at arch/x86/entry/entry_64.S:125
#48 0x0000000000000000 in ?? ()

具体代码如下：

server端演示代码：

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/types.h>#define PORT 8888
#define MAX_PENDING_CONNECTIONS 10int main() {int server_fd, new_socket;struct sockaddr_in address;int opt = 1;int addrlen = sizeof(address);// 创建套接字if ((server_fd = socket(AF_INET, SOCK_STREAM, 0)) == 0) {perror("socket failed");exit(EXIT_FAILURE);}// 设置套接字选项if (setsockopt(server_fd, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt))) {perror("setsockopt");exit(EXIT_FAILURE);}address.sin_family = AF_INET;address.sin_addr.s_addr = INADDR_ANY;address.sin_port = htons(PORT);// 将套接字绑定到指定的地址和端口if (bind(server_fd, (struct sockaddr *)&address, sizeof(address))<0) {perror("bind failed");exit(EXIT_FAILURE);}// 开始监听连接if (listen(server_fd, MAX_PENDING_CONNECTIONS) < 0) {perror("listen");exit(EXIT_FAILURE);}printf("Server listening on port %d\n", PORT);while(1) {// 接受新连接if ((new_socket = accept(server_fd, (struct sockaddr *)&address, (socklen_t*)&addrlen))<0) {perror("accept");exit(EXIT_FAILURE);}printf("New connection from %s:%d\n", inet_ntoa(address.sin_addr), ntohs(address.sin_port));// 向客户端发送消息，包含当前进程IDchar welcome_message[100];int pid = getpid(); // 获取当前进程的 IDsprintf(welcome_message, "Welcome to the server! Server PID: %d\n", pid);send(new_socket, welcome_message, strlen(welcome_message), 0);// 关闭与客户端的连接close(new_socket);}return 0;
}

客户端演示代码：

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <arpa/inet.h>
#include <sys/socket.h>#define PORT 8888
#define SERVER_IP "10.0.2.15" // 服务器 IP 地址
#define MESSAGE "Hello from client"int main() {int sock = 0;struct sockaddr_in serv_addr;char message[1024] = {0};char buffer[1024] = {0};int i = 0;while (i++ < 10) {// 创建套接字if ((sock = socket(AF_INET, SOCK_STREAM, 0)) < 0) {perror("socket creation error");exit(EXIT_FAILURE);}serv_addr.sin_family = AF_INET;serv_addr.sin_port = htons(PORT);if(inet_pton(AF_INET, SERVER_IP, &serv_addr.sin_addr) <= 0) {perror("invalid address / address not supported");exit(EXIT_FAILURE);}// 连接到服务器if (connect(sock, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) {perror("connection failed");exit(EXIT_FAILURE);}// 向服务器发送消息send(sock, MESSAGE, strlen(MESSAGE), 0);printf("Message sent\n");// 接收服务器的响应if (recv(sock, buffer, sizeof(buffer), 0) < 0) {perror("recv failed");exit(EXIT_FAILURE);}printf("Server response: %s\n", buffer);// 关闭套接字close(sock);}return 0;
}