TCP server can not accept client connect beyond 1000? - c

Question

I met a strange question， When I run a TCP program manually(./tcpServDemo)，Tcp Server program Can receive more than 5000 client connections, But When I running tcpServerDemo in the background(systemctl start tcpServDemo.service)，It can only receive more than 900 client connections,
when I debugging, I found TCP recv-Q queue is full. I modified the TCP parameters(net.core.somaxconn = 65500 net.ipv4.tcp_max_syn_backlog = 409600)，But it didn't work
I've been debugging for several days. I really don't know where the problem is？Who can help me，
thanks for everyone!
OS: CentOS 7.9
tcpClient.c:
#include <sys/types.h>
#include <sys/socket.h>
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <unistd.h>
#include <string.h>
#include <netdb.h>
#include <netinet/in.h>
#include <pthread.h>
//#define SERVPORT 8088
int ServPort;
char ServerIP[32];
pthread_mutex_t lock;
int count;
void *cTcpConn(void *arg) {
usleep(2000);
int sockfd,sendbytes;
struct sockaddr_in serv_addr;//需要连接的服务器地址信息
memset(&serv_addr, 0, sizeof(struct sockaddr_in));
//1.创建socket
//AF_INET 表示IPV4
//SOCK_STREAM 表示TCP
if((sockfd = socket(AF_INET,SOCK_STREAM,0)) < 0) {
printf("socket create failed.");
return NULL;
}
//填充服务器地址信息
serv_addr.sin_family = AF_INET; //网络层的IP协议: IPV4
serv_addr.sin_port = htons(ServPort); //传输层的端口号
serv_addr.sin_addr.s_addr = inet_addr(ServerIP); //网络层的IP地址: 实际的服务器IP地址
//2.发起对服务器的连接信息
//三次握手,需要将sockaddr_in类型的数据结构强制转换为sockaddr
if((connect(sockfd,(struct sockaddr *)&serv_addr,sizeof(struct sockaddr))) < 0) {
printf("connect failed!\n");
close(sockfd);
return NULL;
} else {
pthread_mutex_lock(&lock);
count++;
pthread_mutex_unlock(&lock);
printf("connect successful! count:%d\n", count);
}
#if 0
//3.发送消息给服务器端
if((sendbytes = send(sockfd,"hello",5,0)) < 0) {
perror("send");
exit(1);
}
#endif
while(1) {
sleep(10);
}
//4.关闭
close(sockfd);
return NULL;
}
int main(int argc, char **argv) {
if (argc != 3) {
printf("Usage: %s ServerIP ServerPort\n", argv[0]);
return 0;
}
strncpy(ServerIP, argv[1], sizeof(ServerIP) - 1);
ServPort = atoi(argv[2]);
int i;
pthread_t pid;
for (i = 0; i < 8000; i++) {
usleep(10000);
if(0 != pthread_create(&pid, NULL, cTcpConn, NULL)) {
printf("thread create failed.\n");
}
}
while (1) {
sleep(10);
}
return 0;
}
tcpServDemo.c:
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/epoll.h>
#include <unistd.h>
#include <fcntl.h>
#include <string.h>
#include <stdio.h>
#include <errno.h>
int main() {
const int EVENTS_SIZE = 4096;
char buff[1024];
int eNum;
int socketFd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
struct sockaddr_in sockAddr;
sockAddr.sin_port = htons(8088);
sockAddr.sin_family = AF_INET;
sockAddr.sin_addr.s_addr = htons(INADDR_ANY);
if (bind(socketFd, (struct sockaddr *) &sockAddr, sizeof(sockAddr)) == -1) {
return -1;
}
if (listen(socketFd, 10) == -1) {
return -1;
}
int eFd = epoll_create(1);
struct epoll_event epev;
epev.events = EPOLLIN;
epev.data.fd = socketFd;
epoll_ctl(eFd, EPOLL_CTL_ADD, socketFd, &epev);
int i;
int count = 0;
struct epoll_event events[EVENTS_SIZE];
while (1) {
eNum = epoll_wait(eFd, events, EVENTS_SIZE, -1);
if (eNum == -1) {
return -1;
}
for (i = 0; i < eNum; i++) {
if (events[i].data.fd == socketFd) {
if (events[i].events & EPOLLIN) {
struct sockaddr_in cli_addr;
socklen_t length = sizeof(cli_addr);
int fd = accept(socketFd, (struct sockaddr *) &cli_addr, &length);
if (fd > 0) {
count++;
epev.events = EPOLLIN | EPOLLET;
epev.data.fd = fd;
int flags = fcntl(fd, F_GETFL, 0);
if (flags < 0) {
continue;
}
if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) {
continue;
}
epoll_ctl(eFd, EPOLL_CTL_ADD, fd, &epev);
printf("client on line fd:%d-count:%d\n", fd, count);
} else {
printf("accept failed.\n, fd:%d-errno:%d-strerror:%s\n", fd, errno, strerror(errno));
}
}
} else {
if (events[i].events & EPOLLERR || events[i].events & EPOLLHUP) {
epoll_ctl(eFd, EPOLL_CTL_DEL, events[i].data.fd, NULL);
close(events[i].data.fd);
}
}
}
}
}
/usr/lib/systemd/system/tcpServDemo.service:
[Unit]
Description= application service monitor daemon
After=network.target
[Service]
User=root
Type=forking
ExecStart=/var/workstation/testcode/C-Code/tcpServDemo
ExecReload=/bin/kill -HUP $MAINPID
KillMode=process
Restart=on-failure
RestartSec=1s
[Install]
WantedBy=multi-user.target graphic.target

Answer 1

You were running out of file descriptors. accept() would have returned -1 and set errno to EMFILE. It would have been helpful to share that error message with us. You raise the (soft) file descriptor limit by either
setting LimitNOFILESoft your service file to suitable higher value (see systemd.exec(5)), or
calling setrlimit(RLIMIT_NOFILE, ...) with a higher value for rlim_cur less than rlimit_max. It's pretty normal for servers to set their rlimit_cur to the rlimit_max retrieved from getrlimit(RLIMIT_NOFILE, ...) so you can tweak resource usage (in this case number of file descriptors) externally.