Related
My single threaded HTTP Server works just fine, but I'm having trouble multithreading it. I know I am supposed to use pthreads, locks, and condition variables, but I can't get the logic set up properly. The trouble starts after listening to the server. Currently I have a struct that contains a client socket variable, a lock variable, a condition variable, and some variables necessary for parsing and storing headers. I create a struct array sized with the amount of threads, then create a pthread array sized with the amount of threads. I go into a while(1) loop which goes into a for loop and iterates through all the threads accepting each connection, calling pthread_create and passing them to my handle connections function, then closing the client socket. My handle connections then does the request handling that my single threaded http server did (reading, parsing, processing, constructing), then returns NULL. No request gets read when I run this using pthread_create, but if I run handle connections without the pthreads, it works just fine. And below I'll attach my code. Any help is appreciated
Thank you for commenting so well ...
Okay, I coded up, but not tested the changes.
Your loop is inherently single threaded, so a bit of refactoring is in order
You have to scan for an unused thread control slot after doing accept.
You have to pthread_join completed/done threads [from any prior invocations].
The thread function has to close the per-client socket [not main thread]
You need a global (file scope) mutex.
I've coded it up, but not tested it. I put #if 0 around most of what I clipped out and #if 1 around new code.
Note that number of simultaneous connections [second arg to listen], herein 5 has to be less than or equal to threadNum. Although I didn't do it, I'd just do listen(...,threadNum) instead of hardwiring it.
Here's the short code with just the relevant changes:
#if 1
pthread_mutex_t global_mutex = PTHREAD_MUTEX_INITIALIZER;
#endif
struct threadObject {
char method[5]; // PUT, HEAD, GET. HEAD==4 letters+null terminator
char filename[28]; // what is the file we are worried about. Max 27 ASCII characters (NULL terminated on 28)
char httpversion[9]; // HTTP/1.1
ssize_t content_length; // example: 13
uint16_t status_code; // status code for the request
char buffer[BUFFER_SIZE]; // buffer to transfer data
char rest_of_PUT[BUFFER_SIZE]; // incase client send part of PUT message in header
int client_sockd;
pthread_mutex_t *dispatch_lock;
const pthread_cond_t *job_pool_empty;
// pthread_mutex_t* log_lock;
// const pthread_cond_t* log_pool_empty;
pthread_mutex_t *read_write_lock;
pthread_cond_t *file_list_update;
// JobQueue* job_pool;
// LogQueue log_pool;
// bool is_logging;
#if 1
pthread_t tsk_threadid;
int tsk_inuse;
int tsk_done;
#endif
};
void *
handle_connections(void *ptr_thread)
{
// create a mutual exclusion to lock out any other threads from the function
// pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
// pthread_mutex_lock(&mutex);
// operations go here
struct threadObject *thread = (struct threadObject *) ptr_thread;
// reset message after each loop
memset(thread->buffer, '\0', BUFFER_SIZE);
memset(thread->method, '\0', 5);
memset(thread->filename, '\0', 28);
memset(thread->httpversion, '\0', 9);
thread->content_length = 0;
thread->status_code = 0;
memset(thread->rest_of_PUT, '\0', BUFFER_SIZE);
// read message
if (read_http_response(thread) == true) {
// process message
process_request(thread);
}
// construct a response
construct_http_response(thread);
// unlock the function
// pthread_mutex_unlock(&mutex);
#if 1
close(thread->client_sockd);
pthread_mutex_lock(&global_mutex);
thread->tsk_done = 1;
pthread_mutex_unlock(&global_mutex);
#endif
return NULL;
}
int
main(int argc, char **argv)
{
// Create sockaddr_in with server information
if (argc < 2) {
perror("No arguments passed\n");
return -1;
}
// make sure port number is above 1024 and set the port # to it
if (atoi(argv[1]) < 1024) {
return 1;
}
char *port = argv[1];
// parse the command line args for options -l and -N. -l specifies it will use a log and the following parameter is the filename. -N specifies the number of threads it will use and the following parameter will be a number
int opt;
uint8_t threadNum = 1;
char *logName = NULL;
while ((opt = getopt(argc - 1, argv + 1, "N:l:")) != -1) {
if (opt == 'N') {
threadNum = atoi(optarg);
}
else if (opt == 'l') {
logName = optarg;
}
}
struct sockaddr_in server_addr;
memset(&server_addr, 0, sizeof(server_addr));
server_addr.sin_family = AF_INET;
server_addr.sin_port = htons(atoi(port));
server_addr.sin_addr.s_addr = htonl(INADDR_ANY);
socklen_t addrlen = sizeof(server_addr);
// Create server socket
int server_sockd = socket(AF_INET, SOCK_STREAM, 0);
// Need to check if server_sockd < 0, meaning an error
if (server_sockd < 0) {
perror("socket");
return 1;
}
// Configure server socket
int enable = 1;
// This allows you to avoid: 'Bind: Address Already in Use' error
int ret = setsockopt(server_sockd, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(enable));
if (ret < 0) {
return EXIT_FAILURE;
}
// Bind server address to socket that is open
ret = bind(server_sockd, (struct sockaddr *) &server_addr, addrlen);
if (ret < 0) {
return EXIT_FAILURE;
}
// Listen for incoming connections
ret = listen(server_sockd, 5); // 5 should be enough, if not use SOMAXCONN
if (ret < 0) {
return EXIT_FAILURE;
}
struct threadObject thread[threadNum];
// Connecting with a client
struct sockaddr client_addr;
socklen_t client_addrlen = sizeof(client_addr);
// create a pthread array of size (number of threads). specify this will be using the handle connections function. join the threads together
#if 0
pthread_t thread_id[threadNum];
#endif
#if 1
struct threadObject *tsk = NULL;
int tskidx;
// clear out the thread structs
for (tskidx = 0; tskidx < threadNum; tskidx++) {
tsk = &thread[tskidx];
memset(tsk,0,sizeof(struct threadObject));
}
while (true) {
// accept connection
int client_sockd = accept(server_sockd, &client_addr, &client_addrlen);
pthread_mutex_lock(&global_mutex);
// join any previously completed threads
for (tskidx = 0; tskidx < threadNum; tskidx++) {
tsk = &thread[tskidx];
if (tsk->tsk_done) {
pthread_join(tsk->tsk_threadid,NULL);
tsk->tsk_inuse = 0;
tsk->tsk_done = 0;
}
}
// find unused task slot
for (tskidx = 0; tskidx < threadNum; tskidx++) {
tsk = &thread[tskidx];
if (! tsk->tsk_inuse)
break;
}
memset(tsk,0,sizeof(struct threadObject));
tsk->client_sockd = client_sockd;
tsk->tsk_inuse = 1;
pthread_mutex_unlock(&global_mutex);
// fire in the hole ...
pthread_create(&tsk->tsk_threadid, NULL, handle_connections, tsk);
}
#endif
#if 0
for (int i = 0; i < threadNum; i++) {
printf("\n[+] server is waiting...\n");
thread[i].client_sockd = accept(server_sockd, &client_addr, &client_addrlen);
handle_connections(&thread[i]);
// pthread_create(&thread_id[i], NULL, handle_connections, &thread[i]);
printf("Response Sent\n");
// close the current client socket
close(thread[i].client_sockd);
}
}
#endif
return EXIT_SUCCESS;
}
Here's the complete code [just in case I clipped out too much]:
#include <sys/socket.h>
#include <sys/stat.h>
#include <stdio.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <fcntl.h>
#include <unistd.h> // write
#include <string.h> // memset
#include <stdlib.h> // atoi
#include <stdbool.h> // true, false
#include <errno.h>
#include <sys/types.h>
#include <ctype.h>
#include <pthread.h>
#define BUFFER_SIZE 4096
#if 1
pthread_mutex_t global_mutex = PTHREAD_MUTEX_INITIALIZER;
#endif
struct threadObject {
char method[5]; // PUT, HEAD, GET. HEAD==4 letters+null terminator
char filename[28]; // what is the file we are worried about. Max 27 ASCII characters (NULL terminated on 28)
char httpversion[9]; // HTTP/1.1
ssize_t content_length; // example: 13
uint16_t status_code; // status code for the request
char buffer[BUFFER_SIZE]; // buffer to transfer data
char rest_of_PUT[BUFFER_SIZE]; // incase client send part of PUT message in header
int client_sockd;
pthread_mutex_t *dispatch_lock;
const pthread_cond_t *job_pool_empty;
// pthread_mutex_t* log_lock;
// const pthread_cond_t* log_pool_empty;
pthread_mutex_t *read_write_lock;
pthread_cond_t *file_list_update;
// JobQueue* job_pool;
// LogQueue log_pool;
// bool is_logging;
#if 1
pthread_t tsk_threadid;
int tsk_inuse;
int tsk_done;
#endif
};
//read in the header and store it in the appropriate places
bool
read_http_response(struct threadObject *thread)
{
printf("\nThis function will take care of reading message\n");
// how many bytes we're receiving from the header. also puts the message into the buffer
ssize_t bytes = recv(thread->client_sockd, thread->buffer, BUFFER_SIZE, 0);
// if nothing or too much gets sent in the header, return
if (bytes <= 0 || bytes >= BUFFER_SIZE) {
thread->status_code = 400;
printf("Too long or nothing in here\n");
return false;
}
// NULL terminate the last spot on the buffer
thread->buffer[bytes] = '\0';
// how many bytes we received
printf("[+] received %ld bytes from client\n[+] response: \n", bytes);
printf("those bytes are: %s\n", thread->buffer);
// make a char pointer pointer to the buffer to easily traverse it and parse it into the right spots
char *traverse = thread->buffer;
// first stop. sgnals the beginning of the filename
char *file = strstr(traverse, "/");
// 2nd stop. signls the beginning of the HTTP version. only 1.1 is accepted
char *http = strstr(traverse, "HTTP/1.1");
// 3rd stop. Signals the beginning of the content length
char *contlength1 = strstr(traverse, "Content-Length");
char *chunked = strstr(traverse, "chunked");
if (chunked != NULL) {
printf("MESSAGE NOT A FILE PUT\n");
thread->status_code = 403;
return false;
}
// store the method
sscanf(traverse, "%s", thread->method);
printf("method:%s\n", thread->method);
// if its not 1 of the 3 valid requests, throw 400 error
if (strcmp(thread->method, "GET") != 0 &&
strcmp(thread->method, "PUT") != 0 &&
strcmp(thread->method, "HEAD") != 0) {
thread->status_code = 400;
printf("Invalid Method:%s\n", thread->method);
return false;
}
// if the filename doesnt start with /, its invalid throw 400 error
if (*file != '/') {
thread->status_code = 400;
printf("bad filename\n");
return false;
}
// only store the filename portion after the required /
traverse = file + 1;
// to make sure the filename isnt too long
uint8_t size_check = 0;
// traverse filename until first whitespace
while (*traverse != ' ') {
// if any character in the filename isnt 1 of these, its invalid. throw 400 error
if (!isalnum(*traverse) && *traverse != '_' && *traverse != '-') {
// if theres no filename at all, throw a 404 error
if (size_check == 0) {
thread->status_code = 404;
printf("No file specified\n");
return thread->status_code;
}
thread->status_code = 400;
printf("Invalid filename character:%c\n", *traverse);
return false;
}
sscanf(traverse++, "%c", thread->filename + size_check++);
// if the filename breaks the 27 character limit, return a 400 error
if (size_check > 27) {
thread->status_code = 400;
printf("filename too long\n");
return false;
}
}
printf("filename:%s\n", thread->filename);
// if HTTP/1.1 isnt given, throw a 400 error
if (http == NULL) {
printf("HTTP/1.1 400 Bad Request\r\n\r\n");
thread->status_code = 400;
return false;
}
traverse = http;
// read in the http version until the first \r\n. this signals the end of the given version name
sscanf(traverse, "%[^\r\n]s", thread->httpversion);
printf("HTTP:%s\n", thread->httpversion);
// if its not a put request, this is the end of the header. return
if (strcmp(thread->method, "PUT") != 0) {
return true;
}
// for put requests only. traverse until the beginning of the content length
traverse = contlength1;
// last stop. signals the end of a normal PUT header. if a client wants to put some of the message in the header, it gets stored after this
char *end = strstr(traverse, "\r\n\r\n");
// if theres no \r\n\r\n, the header is bad. return 400
if (end == NULL) {
printf("bad header\n");
thread->status_code = 400;
return false;
}
// traverse to the next digit
while (!isdigit(*traverse)) {
// if theres no next digit after "content length", the header is bad. return 400
if (traverse == end) {
printf("bad header\n");
thread->status_code = 400;
return false;
}
traverse++;
}
// set to traverse to be sure fit the entire content length. use size_check to traverse through
char *temp = traverse;
size_check = 0;
// while its taking in digits, put them into the char array.
while (isdigit(*traverse)) {
sscanf(traverse++, "%c", temp + size_check++);
}
// convert the new string into numbers
thread->content_length = atoi(temp);
// if the content length is < 0 throw a 400 error
if (thread->content_length < 0) {
thread->status_code = 400;
printf("bad content length:%ld\n", thread->content_length);
return false;
}
// printf("Content Length:%ld\n", thread->content_length);
// move +4 spots to get to the end of this. if its a normal PUT, this will be the last spot. If the client puts part of the message in the header, it goes after this
traverse = end + 4;
// put the rest of the header into a char array to append later. if theres nothing, itll do nothing
strcpy(thread->rest_of_PUT, traverse);
// printf("Rest of PUT:%s\n", thread->rest_of_PUT);
// will only get here if status code is 0
return true;
}
//process the message we just recieved
void
process_request(struct threadObject *thread)
{
printf("\nProcessing Request\n");
// server side file descriptor
int fd;
// if the method is PUT
if (strcmp(thread->method, "PUT") == 0) {
// open the file for read only to check if its already there or not to set proper status code
fd = open(thread->filename, O_WRONLY);
// if it doesnt exist, set 201 status code
struct stat checkExist;
if (stat(thread->filename, &checkExist) != 0) {
thread->status_code = 201;
}
// if it exists, set 200 and overwrite
else {
struct stat fileStat;
fstat(fd, &fileStat);
// check write permission
if ((S_IWUSR & fileStat.st_mode) == 0) {
printf("MESSAGE NOT WRITEABLE PUT\n");
thread->status_code = 403;
return;
}
thread->status_code = 200;
}
// close it
close(fd);
// reopen it. this time for writing to or overwriting. if its there, overwrite it. if not, create it. cant use for status codes since it will always create a new file
fd = open(thread->filename, O_WRONLY | O_CREAT | O_TRUNC);
// printf("fd in process is:%d\n", fd);
// if theres a bad fd, throw a 403
if (fd < 0) {
printf("ERROR\n\n");
thread->status_code = 403;
return;
}
// to check that the amount of bytes sent = the amount received
ssize_t bytes_recv,
bytes_send;
// if theres no body, put an empty file on the server
if (thread->content_length == 0) {
bytes_send = write(fd, '\0', 0);
}
// if there is a body, put it onto the new file created on the server and make sure the received bytes = the sent ones
else {
ssize_t total = 0,
len_track = thread->content_length;
while (thread->content_length != 0) {
bytes_recv = recv(thread->client_sockd, thread->buffer, BUFFER_SIZE, 0);
bytes_send = write(fd, thread->buffer, bytes_recv);
total += bytes_send;
// if the received bytes != the sent byes, send a 500 error
if (bytes_recv != bytes_send) {
thread->status_code = 500;
printf("Recieved != sent for put request\n");
return;
}
thread->content_length -= bytes_recv;
// printf("Bytes read:%ld\nBytes sent:%ld\nMessage content length:%ld\n", bytes_recv, bytes_send, message->content_length);
}
// if the content length != bytes sent, throw a 403 error
if (len_track != total) {
thread->status_code = 403;
printf("Content length != sent for put request\n");
return;
}
}
printf("Message status code:%d\n", thread->status_code);
// close the fd
close(fd);
return;
}
// if the method is GET or HEAD
else if (strcmp(thread->method, "GET") == 0 || strcmp(thread->method, "HEAD") == 0) {
// open the file for reading only
fd = open(thread->filename, O_RDONLY);
// if bad fd, throw a 404
struct stat fileStat;
fstat(fd, &fileStat);
// check read permission and if it exists
if (((S_IRUSR & fileStat.st_mode) == 0) || stat(thread->filename, &fileStat) != 0) {
printf("BAD GET\n");
thread->status_code = 404;
return;
}
else {
thread->status_code = 200;
thread->content_length = lseek(fd, 0, SEEK_END);
}
// close the fd
close(fd);
return;
}
}
void
construct_http_response(struct threadObject *thread)
{
printf("Constructing Response\n");
// size 22 since the largest code is 21 characters + NULL
char response[22];
// 200=OK, 201=CREATED, 400=BAD REQUEST, 403=FORBIDDEN, 404=NOT FOUND, 500=INTERNAL SERVER ERROR
if (thread->status_code == 200) {
strcpy(response, "OK");
}
else if (thread->status_code == 201) {
strcpy(response, "CREATED");
}
else if (thread->status_code == 400) {
strcpy(response, "BAD REQUEST");
}
else if (thread->status_code == 403) {
strcpy(response, "FORBIDDEN");
}
else if (thread->status_code == 404) {
strcpy(response, "NOT FOUND");
}
else if (thread->status_code == 500) {
strcpy(response, "INTERNAL SERVER ERROR");
}
else {
printf("Bad response...\n");
return;
}
dprintf(thread->client_sockd, "%s %d %s\r\nContent-Length: %ld\r\n\r\n", thread->httpversion, thread->status_code, response, thread->content_length);
if (strcmp(thread->method, "GET") == 0 && thread->status_code == 200) {
int fd = open(thread->filename, O_RDONLY);
ssize_t total = 0,
len_track = thread->content_length,
bytes_recv,
bytes_send;
while (thread->content_length != 0) {
bytes_recv = read(fd, thread->buffer, BUFFER_SIZE);
bytes_send = send(thread->client_sockd, thread->buffer, bytes_recv, 0);
if (bytes_recv != bytes_send) {
thread->status_code = 500;
close(fd);
printf("Recieved != sent for GET request\nReceived:%ld\nSent:%ld\n", bytes_recv, bytes_send);
dprintf(thread->client_sockd, "%s %d %s\r\nContent-Length: %ld\r\n\r\n", thread->httpversion, thread->status_code, response, thread->content_length);
close(fd);
return;
}
total += bytes_send;
thread->content_length -= bytes_recv;
}
if (total != len_track) {
thread->status_code = 403;
printf("Content length != recvd for GET request\n");
dprintf(thread->client_sockd, "%s %d %s\r\nContent-Length: %ld\r\n\r\n", thread->httpversion, thread->status_code, response, thread->content_length);
close(fd);
return;
}
close(fd);
}
}
void *
handle_connections(void *ptr_thread)
{
// create a mutual exclusion to lock out any other threads from the function
// pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
// pthread_mutex_lock(&mutex);
// operations go here
struct threadObject *thread = (struct threadObject *) ptr_thread;
// reset message after each loop
memset(thread->buffer, '\0', BUFFER_SIZE);
memset(thread->method, '\0', 5);
memset(thread->filename, '\0', 28);
memset(thread->httpversion, '\0', 9);
thread->content_length = 0;
thread->status_code = 0;
memset(thread->rest_of_PUT, '\0', BUFFER_SIZE);
// read message
if (read_http_response(thread) == true) {
// process message
process_request(thread);
}
// construct a response
construct_http_response(thread);
// unlock the function
// pthread_mutex_unlock(&mutex);
#if 1
close(thread->client_sockd);
pthread_mutex_lock(&global_mutex);
thread->tsk_done = 1;
pthread_mutex_unlock(&global_mutex);
#endif
return NULL;
}
int
main(int argc, char **argv)
{
// Create sockaddr_in with server information
if (argc < 2) {
perror("No arguments passed\n");
return -1;
}
// make sure port number is above 1024 and set the port # to it
if (atoi(argv[1]) < 1024) {
return 1;
}
char *port = argv[1];
// parse the command line args for options -l and -N. -l specifies it will use a log and the following parameter is the filename. -N specifies the number of threads it will use and the following parameter will be a number
int opt;
uint8_t threadNum = 1;
char *logName = NULL;
while ((opt = getopt(argc - 1, argv + 1, "N:l:")) != -1) {
if (opt == 'N') {
threadNum = atoi(optarg);
}
else if (opt == 'l') {
logName = optarg;
}
}
struct sockaddr_in server_addr;
memset(&server_addr, 0, sizeof(server_addr));
server_addr.sin_family = AF_INET;
server_addr.sin_port = htons(atoi(port));
server_addr.sin_addr.s_addr = htonl(INADDR_ANY);
socklen_t addrlen = sizeof(server_addr);
// Create server socket
int server_sockd = socket(AF_INET, SOCK_STREAM, 0);
// Need to check if server_sockd < 0, meaning an error
if (server_sockd < 0) {
perror("socket");
return 1;
}
// Configure server socket
int enable = 1;
// This allows you to avoid: 'Bind: Address Already in Use' error
int ret = setsockopt(server_sockd, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(enable));
if (ret < 0) {
return EXIT_FAILURE;
}
// Bind server address to socket that is open
ret = bind(server_sockd, (struct sockaddr *) &server_addr, addrlen);
if (ret < 0) {
return EXIT_FAILURE;
}
// Listen for incoming connections
ret = listen(server_sockd, 5); // 5 should be enough, if not use SOMAXCONN
if (ret < 0) {
return EXIT_FAILURE;
}
struct threadObject thread[threadNum];
// Connecting with a client
struct sockaddr client_addr;
socklen_t client_addrlen = sizeof(client_addr);
// create a pthread array of size (number of threads). specify this will be using the handle connections function. join the threads together
#if 0
pthread_t thread_id[threadNum];
#endif
#if 1
struct threadObject *tsk = NULL;
int tskidx;
// clear out the thread structs
for (tskidx = 0; tskidx < threadNum; tskidx++) {
tsk = &thread[tskidx];
memset(tsk,0,sizeof(struct threadObject));
}
while (true) {
// accept connection
int client_sockd = accept(server_sockd, &client_addr, &client_addrlen);
pthread_mutex_lock(&global_mutex);
// join any previously completed threads
for (tskidx = 0; tskidx < threadNum; tskidx++) {
tsk = &thread[tskidx];
if (tsk->tsk_done) {
pthread_join(tsk->tsk_threadid,NULL);
tsk->tsk_inuse = 0;
tsk->tsk_done = 0;
}
}
// find unused task slot
for (tskidx = 0; tskidx < threadNum; tskidx++) {
tsk = &thread[tskidx];
if (! tsk->tsk_inuse)
break;
}
memset(tsk,0,sizeof(struct threadObject));
tsk->client_sockd = client_sockd;
tsk->tsk_inuse = 1;
pthread_mutex_unlock(&global_mutex);
// fire in the hole ...
pthread_create(&tsk->tsk_threadid, NULL, handle_connections, tsk);
}
#endif
#if 0
for (int i = 0; i < threadNum; i++) {
printf("\n[+] server is waiting...\n");
thread[i].client_sockd = accept(server_sockd, &client_addr, &client_addrlen);
handle_connections(&thread[i]);
// pthread_create(&thread_id[i], NULL, handle_connections, &thread[i]);
printf("Response Sent\n");
// close the current client socket
close(thread[i].client_sockd);
}
}
#endif
return EXIT_SUCCESS;
}
Got some trouble with TCP socket multiplexing.
//socket is non-blocking
const int MAX = 4096;
char *buff[MAX];
char *p = buff;
int fd, rvalue;
rvalue = 0;
if ( (fd = open(path, O_RDONLY)) < 0 ) {
return errno;
} else {
int didsend, didread;
int shouldsend;
while ((didread = read(fd, buff, MAX)) > 0) {
p = buff;
shouldsend = didread;
while ( 1 ) {
didsend = send(sockfd, p, shouldsend, 0);
//if send succeeds and returns the number of bytes fewer than asked for then try to send rest part in next time.
if (didsend < shouldsend) {
p += didsent;
shouldsend -= didsend;
continue;
}
//if there is no place for new data to send, then wait a brief time and try again.
if ( didsend < 0 && (errno == EWOULDBLOCK || errno == EAGAIN) ) {
usleep(1000);
continue;
}
//if all data has been sent then sending loop is over.
if (didsend == shouldsend) {
break;
}
//send error
if ( didsend < 0 ) {
rvalue = errno;
break;
}
}
}
close(fd);
if (didread == -1) {
return errno;
}
return rvalue;
}
Assume I use an I/O Multiplexing function poll() or kqueue(), and non-blocking socket, then if there are only some small data like send a short message, it works fine.
But if it comes to large data, I mean larger than send()'s buffer size, since using non-blocking socket, send() will just send a portion of data, and return how much data it sends, the rest part of data can only be sent in another call of send(), but it takes time, and can't tell how long it will takes. So the second while() is actually a blocking send which using non-blocking socket.
Equivalent to：
//socket is blocking
const int MAX = 4096;
char *buff[MAX];
int fd, n;
if ( (fd = open(path, O_RDONLY)) < 0 ) {
return errno;
} else {
while ((n = read(fd, buff, MAX)) > 0) {
if (send(sockfd, buff, n, 0) < 0) {
return errno;
}
}
close(fd);
return 0;
}
So, what is the solution to this, multithreading might work but that's kind of wasting resource maybe.
This is the general pattern for a single-threaded server that works with multiple connections and non-blocking sockets.
It's primarily pseudo-code in C and doesn't do the necessary error checking. But it gives you an idea that for each accepted connection, you keep a struct instance that maintains the socket handle, request parsing state, response stream, and any other "state" members of that connection. Then you just loop using "select" to wait or having multiple threads doing this same thing.
Again this is only pseudo-code and uses select/poll as an example. You can get even more scalability with epoll.
while (1)
{
fd_set readset = {};
fd_set writeset = {};
for (int i = 0; i < number_of_client_connections; i++)
{
if (client_connections[i].reading_request)
FD_SET(client_connection.sock, &readset);
else
FD_SET(client_connection.sock, &writeset);
}
// add the listen socket to the read set
FD_SET(listen_socket, &readset);
select(n + 1, &readset, &writeset, &timeout); // wait for a socket to be ready (not shown - check for errors and return value)
if (FD_ISSET(listen_socket, &readset))
{
int new_client_socket = accept(listen_socket, &addr, &addrlength);
// create a struct that keeps track of the connection state data
struct ConnectionData client_connection = {};
client_connection.sock = new_client_socket;
client_connection.reading_request = 1; // awaiting for all the request bytes to come in
client_connections[number_of_client_connections++] = client_connection; // pseudo code, add the client_connection to the list
}
for (int i = 0; i < number_of_client_connections; i++)
{
if (client_connections[i].reading_request)
{
if (FD_ISSET(client_connections[i], &readset))
{
char buffer[2000];
int len = recv(client_connections[i].sock, buffer, 2000, 0);
// not shown - handle error case when (recv < 0)
// not shown - handle case when (recv == 0)
ProcessIncomingData(client_connections[i], buffer, len); // do all the request parsing here. Flip the client_connections[i].reading_request to 0 if ready to respond
}
}
else if (client_connections[i].reading_request == 0)
{
if (FD_ISSET(client_connections[i], &writeset))
{
client_connection* conn = &client_connections[i];
int len = send(conn->sock, conn->response_buffer + conn->txCount, conn->response_size - conn->txCount, 0);
conn->txCount += len;
if (conn->txCount == conn->response_size)
{
// done sending response - we can close this connection or change it to back to the reading state
}
}
}
}
I want to directly read and write on hard disk partitions. I'm using C by accessing the test partition G: (2GB) for this purpose. I have successfully read the bytes sector wise. I want to read the bytes from sector 1 and writem them to sector 3908880 but i'm not able to write on the disk. Interestingly the WriteFile() method executes successfully but when i use WinHex Editor to view the bytes. It does not show up.
I have seen some similar questions which described the privilege problems but i don't have a privilege problem the function executes successfully but does not write the bytes.
Here is my code:
HANDLE getDeviceHandle(wchar_t* partition, char mode)
{
HANDLE device;
int retCode = 1;
if (mode == 'r')
{
device = CreateFile(
partition, // Partition to open
GENERIC_READ, // Access mode
FILE_SHARE_READ | FILE_SHARE_WRITE, // Share Mode
NULL, // Security Descriptor
OPEN_EXISTING, // How to create
0, // File attributes
NULL); // Handle to template
}
else if(mode == 'w')
{
device = CreateFile(
partition, // Partition to open
GENERIC_READ | GENERIC_WRITE, // Access mode
FILE_SHARE_READ | FILE_SHARE_WRITE, // Share Mode
NULL, // Security Descriptor
OPEN_EXISTING, // How to create
0, // File attributes
NULL); // Handle to template
}
if (device == INVALID_HANDLE_VALUE)
retCode = -1;
if(retCode == 1)
return device;
else
return NULL;
}
int WriteSector(HANDLE device ,BYTE* bytesToWrite, DWORD size, int sectorNo )
{
char buffForPrint[512] = { 0 };
int Code = 0;
DWORD byteswritten;
int NoOfSectorsOnPartition = 0;
DWORD bytesReturnedSize = 0;
if (NULL == device)
{
printf("Exiting from WriteSector\n");
return 0;
}
else
{
int ret = getNoOfSectors(device, bytesReturnedSize);
if (-1 != ret)
{
NoOfSectorsOnPartition = ret;
if (sectorNo > NoOfSectorsOnPartition)
{
printf("Selected sector out of range");
Code = -1;
return Code;
}else
{
DWORD status;
if (!DeviceIoControl(device, IOCTL_DISK_IS_WRITABLE, NULL, 0, NULL, 0, &status, NULL))
{
// error handling; not sure if retrying is useful
}else if (!WriteFile(device, bytesToWrite, size, &byteswritten, NULL))
{
printf("Error in writing.Error Code: %i\n", GetLastError());
Code = -1;
return Code;
}
else
{
printf("Sector Written\n");
Code = 1;
}
}
}
}
return Code;
}
int main()
{
static BYTE read[512];
HANDLE hand;
int sector =1;
hand = getDeviceHandle(L"\\\\.\\G:", 'r');
if (ReadSector(hand, read, 512, sector) == 1)
{
printf("successfully read sector %i\n", sector);
}
sector = 3908880;
hand = getDeviceHandle(L"\\\\.\\G:", 'w');
if (WriteSector(hand,read,SECTOR_SIZE,sector) == 1) //SECTOR_SIZE 512
{
printf("successfully wrote sector %i\n",sector);
}
CloseHandle(hand); // Close the handle
getch();
}
Handling & processing data from named pipes.
I am trying to implement a service provider to connect with a hardware device.
request some pointers on my approach to implement a robust system.
Mentioned are the raised requirements
Receive data from other EXE process
To process received Q information and send response information in clients response channel.
Asynchronously send information on some failure to client response channel.
TO implement the mentioned system:
Selected 2 named pipe (ClntcommandRecv & ClntRespSend) .bcz of between process (IPC)
ClntcommandRecv pipe will be used as "Named Pipe Server Using Overlapped" I/O"
ClntRespSend pipe will be used for sending the processed information.
ClntRespSend will also need to send all the async messages from service provider to connected application.
From here my implementation is straight forward.
Using "Named Pipe Server Using Overlapped I/O" by documentation I will be able to address multiple client connection request and its data processing using single thread.
On init system will create a thread to hold connection instance of clients ClntRespSend pipe.
Since, it requires for device to tell its failures to connected clients asynchronously.
Is it advisable for system to have timeout operation on "WaitForMultipleObjects" or
can we have readfile timeout counts after n timeout can we check for health info.WHich is advised
But, stuck in finding the best way to sync my ClntRespSend & ClntcommandRecv (MAPPIN).
Need to get process id of the connected process.Since the system is developed under MINGW - WIN32 - server will not be able to get the process id directly by using (GetNamedPipeClientProcessId).
Need to form a message structure on getting a client connection.
This is the code which i am trying to extend:
#include <windows.h>
#include <stdio.h>
#include <tchar.h>
//#include <strsafe.h>
//#include <glib.h>
#define CONNECTING_STATE 0
#define READING_STATE 1
#define WRITING_STATE 2
#define INSTANCES 4
#define PIPE_TIMEOUT 5000
#define BUFSIZE 4096
typedef struct
{
OVERLAPPED oOverlap;
HANDLE hPipeInst;
TCHAR chRequest[BUFSIZE];
DWORD cbRead;
TCHAR chReply[BUFSIZE];
DWORD cbToWrite;
DWORD dwState;
BOOL fPendingIO;
int processId;
} PIPEINST, *LPPIPEINST;
typedef struct
{
char appName[256];
int processId;
}PIPEHANDSHAKE;
VOID DisconnectAndReconnect(DWORD);
BOOL ConnectToNewClient(HANDLE, LPOVERLAPPED);
VOID GetAnswerToRequest(LPPIPEINST);
PIPEINST Pipe[INSTANCES];
HANDLE hEvents[INSTANCES];
HANDLE responsePipeHandle[INSTANCES];
DWORD WINAPI InstanceThread(LPVOID);
HANDLE hPipeHandles[10];
PULONG s;
LPTSTR lpszPipename = TEXT("\\\\.\\pipe\\mynamedpipe");
LPTSTR lpszResponsePipe = TEXT("\\\\.\\pipe\\mynamedpipe1");
//GHashTable* hash;
int responsePipeConnectionHandler(VOID)
{
BOOL fConnected = FALSE;
DWORD dwThreadId = 0;
HANDLE hPipe = INVALID_HANDLE_VALUE, hThread = NULL;
int cbBytesRead;
INT threadCount=0;
//hash = g_hash_table_new(g_str_hash, g_str_equal);
char bufferSize[512];
for (;;)
{
_tprintf( TEXT("\nPipe Server: Main thread awaiting client connection on %s\n"), lpszResponsePipe);
hPipe = CreateNamedPipe(
lpszResponsePipe, // pipe name
PIPE_ACCESS_DUPLEX, // read/write access
PIPE_TYPE_MESSAGE | // message type pipe
PIPE_READMODE_MESSAGE | // message-read mode
PIPE_WAIT, // blocking mode
PIPE_UNLIMITED_INSTANCES, // max. instances
BUFSIZE, // output buffer size
BUFSIZE, // input buffer size
0, // client time-out
NULL); // default security attribute
if (hPipe == INVALID_HANDLE_VALUE)
{
_tprintf(TEXT("CreateNamedPipe failed, GLE=%d.\n"), GetLastError());
return -1;
}
// Wait for the client to connect; if it succeeds,
// the function returns a nonzero value. If the function
// returns zero, GetLastError returns ERROR_PIPE_CONNECTED.
fConnected = ConnectNamedPipe(hPipe, NULL) ?
TRUE : (GetLastError() == ERROR_PIPE_CONNECTED);
if(fConnected){
PIPEHANDSHAKE processData;
fConnected = ReadFile(
hPipe, // handle to pipe
bufferSize, // buffer to receive data
sizeof(PIPEHANDSHAKE), // size of buffer
&cbBytesRead, // number of bytes read
NULL); // not overlapped I/O
memset(&processData,0,sizeof(PIPEHANDSHAKE));
memcpy(&processData,&bufferSize,sizeof(PIPEHANDSHAKE));
printf("APP Process id: %d , app name: %s",processData.processId,processData.appName);
}
/* if (fConnected)
{
printf("Client connected, creating a processing thread.\n");
// Create a thread for this client.
hThread = CreateThread(
NULL, // no security attribute
0, // default stack size
InstanceThread, // thread proc
(LPVOID) hPipe, // thread parameter
0, // not suspended
&dwThreadId); // returns thread ID
if (hThread == NULL)
{
_tprintf(TEXT("CreateThread failed, GLE=%d.\n"), GetLastError());
return -1;
}
else CloseHandle(hThread);
}
else
// The client could not connect, so close the pipe.
CloseHandle(hPipe);*/
}
return 0;
}
int _tmain(VOID)
{
DWORD i, dwWait, cbRet, dwErr,hThread;
BOOL fSuccess;
int dwThreadId;
// The initial loop creates several instances of a named pipe
// along with an event object for each instance. An
// overlapped ConnectNamedPipe operation is started for
// each instance.
// Create response pipe thread
hThread = CreateThread(
NULL, // no security attribute
0, // default stack size
responsePipeConnectionHandler, // thread proc
NULL, // thread parameter
0, // not suspended
&dwThreadId); // returns thread ID
if (hThread == NULL)
{
printf("Response server creation failed with %d.\n", GetLastError());
return 0;
}
for (i = 0; i < INSTANCES; i++)
{
// Create an event object for this instance.
hEvents[i] = CreateEvent(
NULL, // default security attribute
TRUE, // manual-reset event
TRUE, // initial state = signaled
NULL); // unnamed event object
if (hEvents[i] == NULL)
{
printf("CreateEvent failed with %d.\n", GetLastError());
return 0;
}
Pipe[i].oOverlap.hEvent = hEvents[i];
Pipe[i].hPipeInst = CreateNamedPipe(
lpszPipename, // pipe name
PIPE_ACCESS_DUPLEX | // read/write access
FILE_FLAG_OVERLAPPED, // overlapped mode
PIPE_TYPE_MESSAGE | // message-type pipe
PIPE_READMODE_MESSAGE | // message-read mode
PIPE_WAIT, // blocking mode
INSTANCES, // number of instances
BUFSIZE*sizeof(TCHAR), // output buffer size
BUFSIZE*sizeof(TCHAR), // input buffer size
PIPE_TIMEOUT, // client time-out
NULL); // default security attributes
if (Pipe[i].hPipeInst == INVALID_HANDLE_VALUE)
{
printf("CreateNamedPipe failed with %d.\n", GetLastError());
return 0;
}
// Call the subroutine to connect to the new client
Pipe[i].fPendingIO = ConnectToNewClient(
Pipe[i].hPipeInst,
&Pipe[i].oOverlap);
Pipe[i].dwState = Pipe[i].fPendingIO ?
CONNECTING_STATE : // still connecting
READING_STATE; // ready to read
}
while (1)
{
dwWait = WaitForMultipleObjects(
INSTANCES, // number of event objects
hEvents, // array of event objects
FALSE, // does not wait for all
INFINITE); // waits indefinitely
// dwWait shows which pipe completed the operation.
i = dwWait - WAIT_OBJECT_0; // determines which pipe
if (i < 0 || i > (INSTANCES - 1))
{
printf("Index out of range.\n");
return 0;
}
// Get the result if the operation was pending.
if (Pipe[i].fPendingIO)
{
fSuccess = GetOverlappedResult(
Pipe[i].hPipeInst, // handle to pipe
&Pipe[i].oOverlap, // OVERLAPPED structure
&cbRet, // bytes transferred
FALSE); // do not wait
switch (Pipe[i].dwState)
{
// Pending connect operation
case CONNECTING_STATE:
if (! fSuccess)
{
printf("Error %d.\n", GetLastError());
return 0;
}
Pipe[i].dwState = READING_STATE;
break;
// Pending read operation
case READING_STATE:
if (! fSuccess || cbRet == 0)
{
DisconnectAndReconnect(i);
continue;
}
Pipe[i].cbRead = cbRet;
Pipe[i].dwState = WRITING_STATE;
break;
// Pending write operation
case WRITING_STATE:
if (! fSuccess || cbRet != Pipe[i].cbToWrite)
{
DisconnectAndReconnect(i);
continue;
}
Pipe[i].dwState = READING_STATE;
break;
default:
{
printf("Invalid pipe state.\n");
return 0;
}
}
}
// The pipe state determines which operation to do next.
switch (Pipe[i].dwState)
{
case READING_STATE:
fSuccess = ReadFile(
Pipe[i].hPipeInst,
Pipe[i].chRequest,
BUFSIZE*sizeof(TCHAR),
&Pipe[i].cbRead,
&Pipe[i].oOverlap);
if (fSuccess && Pipe[i].cbRead != 0)
{
Pipe[i].fPendingIO = FALSE;
Pipe[i].dwState = WRITING_STATE;
continue;
}
dwErr = GetLastError();
if (! fSuccess && (dwErr == ERROR_IO_PENDING))
{
Pipe[i].fPendingIO = TRUE;
continue;
}
DisconnectAndReconnect(i);
break;
case WRITING_STATE:
GetAnswerToRequest(&Pipe[i]);
fSuccess = WriteFile(
Pipe[i].hPipeInst,
Pipe[i].chReply,
Pipe[i].cbToWrite,
&cbRet,
&Pipe[i].oOverlap);
if (fSuccess && cbRet == Pipe[i].cbToWrite)
{
Pipe[i].fPendingIO = FALSE;
Pipe[i].dwState = READING_STATE;
continue;
}
dwErr = GetLastError();
if (! fSuccess && (dwErr == ERROR_IO_PENDING))
{
Pipe[i].fPendingIO = TRUE;
continue;
}
DisconnectAndReconnect(i);
break;
default:
{
printf("Invalid pipe state.\n");
return 0;
}
}
}
return 0;
}
VOID DisconnectAndReconnect(DWORD i)
{
if (! DisconnectNamedPipe(Pipe[i].hPipeInst) )
{
printf("DisconnectNamedPipe failed with %d.\n", GetLastError());
}
Pipe[i].fPendingIO = ConnectToNewClient(
Pipe[i].hPipeInst,
&Pipe[i].oOverlap);
Pipe[i].dwState = Pipe[i].fPendingIO ?
CONNECTING_STATE : // still connecting
READING_STATE; // ready to read
}
BOOL ConnectToNewClient(HANDLE hPipe, LPOVERLAPPED lpo)
{
BOOL fConnected, fPendingIO = FALSE;
fConnected = ConnectNamedPipe(hPipe, lpo);
if (fConnected)
{
printf("ConnectNamedPipe failed with %d.\n", GetLastError());
return 0;
}
switch (GetLastError())
{
// The overlapped connection in progress.
case ERROR_IO_PENDING:
fPendingIO = TRUE;
break;
case ERROR_PIPE_CONNECTED:
if (SetEvent(lpo->hEvent))
break;
default:
{
printf("ConnectNamedPipe failed with %d.\n", GetLastError());
return 0;
}
}
return fPendingIO;
}
int rxProccesIdMsg(HANDLE pipe)
{
PIPEHANDSHAKE pipeInfo;
CHAR bufferSize[512] = {'\0'};
INT cbBytesRead;
BOOL fSuccess;
PIPEHANDSHAKE processData;
fSuccess = ReadFile(
pipe, // handle to pipe
bufferSize, // buffer to receive data
sizeof(PIPEHANDSHAKE), // size of buffer
&cbBytesRead, // number of bytes read
NULL); // not overlapped I/O
memset(&processData,0,sizeof(PIPEHANDSHAKE));
memcpy(&processData,&bufferSize,sizeof(PIPEHANDSHAKE));
if ( (!fSuccess))
{
printf("Client: READ Server Pipe Failed(%d)\n",GetLastError());
CloseHandle(pipe);
return -1;
}
else
{
printf("Client: READ Server Pipe Success(%d)\n",GetLastError());
printf("APP Process id: %d , app name: %s",processData.processId,processData.appName);
//Sleep(3*100);
}
return processData.processId;
}
VOID GetAnswerToRequest(LPPIPEINST pipe)
{
_tprintf( TEXT("[%d] %s\n"), pipe->hPipeInst, pipe->chRequest);
// StringCchCopy( pipe->chReply, BUFSIZE, TEXT("Default answer from server") );
strncpy(pipe->chReply, "Default answer from server",BUFSIZE);
pipe->cbToWrite = (lstrlen(pipe->chReply)+1)*sizeof(TCHAR);
}
I'm trying to debug a code that is using a libevent library. In that library, there is a function event_new that is suppose to create an event_cb. Somehow after I dispatch the event base, the event_cb cannot be called or accessed. This problem only happens on hpux itanium. This code works on hpux pa-risc, Redhat, AIX, and Solaris. Is there any certain thing that need to be set?
This is part of the code
int ttypread (int fd, Header *h, char **buf)
{
int c,k;
struct user_data user_data;
struct bufferevent *in_buffer;
struct event_config *evconfig;
log_debug("inside ttypread");
in_buffer = NULL;
user_data.fd = fd;
user_data.h = h;
user_data.buf = buf;
log_debug("from user_data, fd = %d",user_data.fd); //the log_debug is a debugging function for me to check the value sent by the system. I use it to compare between each platform
log_debug("from user_data, buf = %s",user_data.buf);
log_debug("from user_data, h.len = %d",user_data.h->len);
log_debug("from user_data, h.type = %d",user_data.h->type);
evconfig = event_config_new();
if (evconfig == NULL) {
log_error("event_config_new failed");
return -1;
}
if (event_config_require_features(evconfig, EV_FEATURE_FDS)!=0) {
log_error("event_config_require_features failed");
return -1;
}
base = event_base_new_with_config(evconfig);
if (!base) {
log_error("ttypread:event_base_new failed");
return -1;
}
const char* method; //these 3 lines are the new line edited
method = event_base_get_method(base);
log_debug("ttyread is using method = %s",method);
ev = event_new(base, fd, EV_READ|EV_PERSIST, ttypread_event_cb, &user_data);
c = event_add(ev, NULL);
log_debug("ttypread passed event_add with c value is %d",c);
in_buffer = bufferevent_socket_new(base, STDIN_FILENO, BEV_OPT_CLOSE_ON_FREE);
log_debug("ttypread passed bufferevent_socket_new");
if(in_buffer == NULL){
log_debug("problem with bufferevent_socket_new");
}
bufferevent_setcb(in_buffer, in_read_cb, NULL, in_event_cb, NULL);
bufferevent_disable(in_buffer, EV_WRITE);
bufferevent_enable(in_buffer, EV_READ);
k =event_base_dispatch(base);
log_debug("event_base have been dispatched"); //when looking at the debugging file, the other plaform will go to ttypread_event_cb function. But for hpux itanium, it stays here.
if (k == 0){
log_debug("event_base_dispatch returned 0");
} else if (k == -1){
log_debug("event_base_dispatch returned -1");
} else {
log_debug("event_base_dispatch returned 1");
}
event_base_free(base);
event_free(ev);
log_debug("finish ttypread");
log_debug("ttypread_ret will return [%d]",ttypread_ret);
return ttypread_ret;
}
void ttypread_event_cb(evutil_socket_t fd, short events, void *arg)
{
int nread;
struct timeval t;
struct user_data *user_data;
user_data = (struct user_data*)arg;
nread = 0;
log_debug("inside ttypread_event_cb");
if (events & EV_READ) {
log_debug("got events & EV_READ");
nread = ttyread(fd, user_data->h, user_data->buf);
if (nread == -1) {
ttypread_ret = -1;
event_del(ev);
event_base_loopexit(base, NULL);
} else if (nread == 0) {
if (access(input_filename, F_OK)!=0) {
log_debug("cannot access [%s]",input_filename);
tcsetattr(0, TCSANOW, &old); /* Return terminal state */
exit(EXIT_SUCCESS);
}
t.tv_sec = 0;
t.tv_usec = 250000;
select(0, 0, 0, 0, &t);
} else {
ttypread_ret = 1;
event_del(ev);
event_base_loopexit(base, NULL);
}
}
else if (events & EV_WRITE) {
log_debug("got events & EV_WRITE");
}
}
Not sure if this help. But just some info on the hpux itanium
uname -a = HP-UX hpux-ita B.11.23 U ia64
If you need any additional info or other declaration on function, just leave a comment and I will edit the question.
EDIT : i've added a function inside ttypread. Somehow for hpux itanium its returning devpoll while other platform are returning poll. Im not sure if this is the problem. But if that is so, is there any way for me to change it?
After checking the result from event_base_get_method, I found out that only on my hpux-itanium used devpoll method. This is how I solve it.
char string[8] = "devpoll";
struct user_data user_data;
struct bufferevent *in_buffer;
struct event_config *evconfig;
const char *method;
const char *devpoll;
devpoll = string;
in_buffer = NULL;
user_data.fd = fd;
user_data.h = h;
user_data.buf = buf;
evconfig = event_config_new();
if (evconfig == NULL) {
log_error("event_config_new failed");
return -1;
}
if (event_config_require_features(evconfig, EV_FEATURE_FDS)!=0) {
log_error("event_config_require_features failed");
return -1;
}
if (event_config_avoid_method(evconfig,devpoll) != 0)
{
log_error("Failed to ignore devpoll method");
}
Force the libevent to ignore using devpoll and use poll instead.