State machine per thread for socket client communication - c

Question

I am working on threaded TCP socket server for handling multiple socket client connection. Clients can connect and disconnect asynchronously with server, upon connection, the client should send some data in predefined custom packet protocol format.
The protocol has start of frame(SOP) and end of frame (EOP) defined.
I have written a C code such that for each successful client connection, a thread gets created that keeps on receiving the bytes from client in the predefined packet format, the thread has a thread-local state machine because each client can connect asynchronously so the states for each client may be different.
Below is the thread that receives that data from client and maintains a state based on the type of byte received:
static void *receive_handler(void *args) {
struct thread_args *local_args = args;
struct sockaddr_in6 *client_address = local_args->client_address;
//struct itimerval timer_val;
int32_t conn_fd = local_args->conn_fd;
int32_t val_read = 0;
int32_t resp_code = 0;
uint32_t sendBuffLen = 0;
int8_t buffer[BUFFER_SIZE] = { 0 };
uint8_t RetBuff[1024] = { 0 };
int8_t rx_addr_str[INET6_ADDRSTRLEN];
int8_t byte = 0;
int16_t idx = ePacketType;
int16_t packet_len = 0;
int16_t calculated_crc = 0, recv_crc = 0;
uint16_t num_bytes = 0;
memset(rx_addr_str, 0, INET6_ADDRSTRLEN);
inet_ntop(AF_INET6, &(client_address->sin6_addr), rx_addr_str, INET6_ADDRSTRLEN);
printf("\nRx Thread (%d) Created for %s\n", local_args->connection_no, rx_addr_str);
int eState = eStart_Frame;
memcpy(rx_Packet_Info[local_args->connection_no].inet6, rx_addr_str, INET6_ADDRSTRLEN);
//timerclear(&timer_val.it_interval); /* zero interval means no reset of timer */
//timerclear(&timer_val.it_value);
//timer_val.it_value.tv_sec = 10; /* 10 second timeout */
//(void) signal(SIGALRM, state_reset_handler);
while (1) {
if (eState != eChecksum_Verify) {
val_read = -1;
val_read = recv(conn_fd, &byte, sizeof(byte), 0);
debug_printf(INFO, "Amount Read: %d Byte Rxd: 0x%x => 0x%X\n", val_read, (byte & 0xFF), byte);
if (val_read <= 0) {
if (parse_packet("ERR_DISCONNECT", rx_addr_str, local_args->connection_no) < 0) {
debug_printf(ERR, "Error parsing packet: %s\n", strerror(errno));
}
debug_printf(ERR, "May be closed by client %s: %s\n", rx_addr_str, strerror(errno));
debug_printf(ERR, "Exiting Rx Thread: ConnIdx: %d", num_connections);
close(conn_fd);
pthread_exit(NULL);
}
}
switch (eState) {
case eStart_Frame:
debug_printf(DEBG, "Current State: %d\n", eState);
if ((val_read > 0) && (byte & 0xFF) == SOP) {
memset(buffer, 0, BUFFER_SIZE);
val_read = -1;
buffer[eSOP] = (byte & 0xFF);
eState = eFrame_Len;
}
break;
case eFrame_Len: {
static char MSB_Rxd = 0;
debug_printf(DEBG, "Current State: %d\n", eState);
if (val_read > 0) {
if (MSB_Rxd == 0) {
buffer[ePacket_length] = byte;
MSB_Rxd = 1;
}
else {
buffer[ePacket_length + 1] = byte;
eState = eFrame;
num_bytes = 0;
MSB_Rxd = 0;
packet_len = (buffer[ePacket_length] & 0xFF << 8) | (buffer[ePacket_length + 1]);
debug_printf(INFO, "Packet Length: %d : 0x%x 0x%x\n", packet_len,
buffer[ePacket_length], buffer[ePacket_length + 1]);
}
}
}
break;
case eFrame:
debug_printf(DEBG, "Current State: %d\n", eState);
num_bytes++;
buffer[idx] = byte;
if (num_bytes == packet_len) {
eState = eEnd_Frame;
debug_printf(DEBG, "Num bytes: 0x%x\n", num_bytes);
}
else {
debug_printf(ERR, "Num bytes: 0x%x Pkt Len: 0x%x\n", num_bytes, packet_len);
}
idx++;
break;
case eEnd_Frame:
debug_printf(ERR, "Current State: %d val read %d\n", eState, val_read);
if ((val_read > 0) && (byte & 0xFF) == EOP) {
val_read = -1;
eState = eChecksum_Verify;
}
break;
case eChecksum_Verify: {
calculated_crc = crc_16(&buffer[ePacket_length], (num_bytes));
recv_crc = buffer[num_bytes + 1] << 8 | (buffer[num_bytes + 2] & 0xFF);
if (calculated_crc != recv_crc) {
debug_printf(ERR, "CRC Error! CRC do not match!!\n");
debug_printf(ERR, "Calculated CRC: 0x%X\nCRC Rxd: 0x%X\n", calculated_crc, recv_crc);
resp_code = CRC_ERR;
send(conn_fd, &resp_code, sizeof(resp_code), 0);
}
else {
if (rx_Packet_Info[local_args->connection_no].packetUUID != NULL) {
free(rx_Packet_Info[local_args->connection_no].packetUUID);
rx_Packet_Info[local_args->connection_no].packetUUID = NULL;
}
rx_Packet_Info[local_args->connection_no].packetUUID = calloc(buffer[ePacketUUIDLen],
sizeof(uint8_t));
memcpy(rx_Packet_Info[local_args->connection_no].packetUUID, &buffer[ePacketUUID],
buffer[ePacketUUIDLen]);
rx_Packet_Info[local_args->connection_no].packetUUIDlength = buffer[ePacketUUIDLen];
printf("\nRX-Thread-UUID %d: ConnNo: %d\n", buffer[ePacketUUIDLen],
local_args->connection_no);
for (char i = 0; i < buffer[ePacketUUIDLen]; i++) {
printf("0x%x ", rx_Packet_Info[local_args->connection_no].packetUUID[i]);
}
printf("\n");
if (parse_packet(buffer, rx_addr_str, local_args->connection_no) < 0) {
debug_printf(ERR, "Error parsing packet: %s\n", strerror(errno));
}
}
num_bytes = 0;
eState = eStart_Frame;
idx = ePacketType;
}
break;
default:
debug_printf(DEBG, "Invalid State!! Should not come here.\n");
num_bytes = 0;
eState = eStart_Frame;
idx = ePacketType;
break;
}
}
return NULL;
}
My question is how should I reset this state machine if let's say after receiving start of frame the client gets stuck and is not able to send frame length or complete frame till end of frame?
One way I thought is to implement timer callback but I am not sure how should I keep track of state machine of multiple threads.
Can any one please suggest what should I do in this scenario or if I am doing anything wrong?

Answer 1

If I'm parsing the question correctly, you're asking about how to handle gracefully the situation where the connecting client isn't sending data in a timely manner -- i.e. it has sent the first part of a message, but (due to a network problem or a client-side bug or whatever) never sends the rest, leaving your server-side I/O thread blocked inside a recv() call for a long/indefinite time.
If so, the first question to ask is: is this really a problem? If each connection gets its own thread, then having one particular thread/connection blocked shouldn't cause any issues to the other threads, since they all execute independently of each other. So maybe you can just ignore the problem entirely?
However, the more likely answer is that ignoring the problem isn't quite good enough, because of a couple of subsequent problems that aren't easily ignorable: (a) what if too many client connections "freeze up" at the same time? One or two stalled TCP connections/threads isn't a big deal, but if the same problem keeps happening, eventually you'll run out of resources to spawn more threads or TCP connections, and then your server can no longer function. And (b) what if the server process wants to quit now? (i.e. because the server's user has sent it a SIGINT interrupt or similar) If one or more threads are blocked indefinitely, then it is impossible for the server to exit in a timely-and-controlled manner, because the main thread needs to wait for all the TCP threads to exit first before it can clean up its process-wide resources, and any blocked threads will not exit for a long time, if ever.
So, assuming that the problem does need to be addressed, the most reliable way I've found to address it is to never block in recv() (or send()) in the first place. Instead, make sure to put each socket in non-blocking mode, and have the thread's while-loop block only in a select() call instead. Doing it this way makes your state machine a bit more complex (since it will now have to handle partial-sends as well as partial-receives), but the compensating benefit is that the thread is now in better control of its own blocking behavior. In particular, you can tell select() to always return after a certain amount of time, no matter what, and (better yet) you can tell select() to return whenever any of a number of sockets has bytes ready to be read on it. That means that if your main thread wants to exit, it can use a pipe() or socketpair() to send a dummy-byte to each TCP thread, and the TCP thread (which is presumably blocked inside select(), waiting for either data from its client or from the pipe/socketpair socket) will immediately return from select(), see that the main thread has sent it a byte, and respond by exiting immediately.
That should be sufficient -- in my experience it is better not to impose fixed timeouts if you can avoid it, since it's hard to predict what network performance will be like in all cases, and any rule-of-thumb you might come up with (like "a client that doesn't send the whole message in 5 seconds must be broken") is likely to be wrong, and you'll end up with false-positive problems if you try to enforce that rule. Better to just let each client take as long as it wants/needs to, while also having a mechanism by which the main thread can request that a particular client thread exit immediately if/when that becomes necessary (e.g. during server-process shutdown, or if there are too many TCP threads active and you want to prune some of the old/inactive ones before spawning more)