Related
I wrote a simple file copy application to measure the effectiveness of using sendfile API over normal read-from-file-and-write-to-socket approach for large files. However upon running the application using both the approaches, I found out that the difference in the amount of time taken for the file copy to get completed is very minimal between the two approaches.
I read from multiple sources that "sendfile" API would give tremendous performance improvement over the normal read-from-file-and-write-to-socket approach. But when I tried to benchmark with a single 2GB file following are the numbers I observed (average of 4 iterations):
Normal read-from-file-and-write-to-socket approach: 17 secs 444840 usecs
sendfile API: 17 secs 431420 usecs
I am running both the server and client pieces of the application on two different machines (Linux kernel version 4.4.162-94.72-default) in an isolated 1Gbps network.
Can someone help me what exactly am doing wrong here or missing here?
Server:
#define _GNU_SOURCE
#include "file_details.h"
void calculate_execution_time(struct timeval start, struct timeval end)
{
struct timeval time_diff;
time_diff.tv_sec = end.tv_sec - start.tv_sec;
time_diff.tv_usec = end.tv_usec - start.tv_usec;
// Adjust the time appropriately
while (time_diff.tv_usec < 0) {
time_diff.tv_sec--;
time_diff.tv_usec += 1000000;
}
printf("total execution time: = %lds.%ldus\n", time_diff.tv_sec, time_diff.tv_usec);
}
int read_from_file_pread(int client_sockfd, char *file_name, int fd, off_t file_size_in_bytes, int chunk_size)
{
ssize_t bytes_read = 0, bytes_sent = 0, total_bytes_sent = 0, bytes_sent_this_itr = 0;
off_t offset = 0;
char *buffer = NULL;
struct timeval start_time, end_time;
buffer = calloc(chunk_size, sizeof(char));
if (buffer == NULL) {
printf("Failed to allocate memory of size: %d bytes\n", chunk_size);
return -1;
}
gettimeofday(&start_time, NULL);
do {
bytes_read = pread(fd, buffer, chunk_size, offset);
switch (bytes_read) {
case -1:
printf("Failed to read from file: %s, offset: %lu, error: %d\n", file_name, offset, errno);
free(buffer);
return -1;
case 0:
printf("Completed reading from file and sending\n");
break;
default:
do {
bytes_sent = send(client_sockfd, buffer, (bytes_read - bytes_sent_this_itr), 0);
if (bytes_sent == -1) {
printf("Failed to send %lu bytes, error: %d\n", (bytes_read - bytes_sent_this_itr), errno);
free(buffer);
return -1;
}
bytes_sent_this_itr += bytes_sent;
} while (bytes_sent_this_itr < bytes_read);
bytes_sent = 0;
bytes_sent_this_itr = 0;
offset += bytes_read;
total_bytes_sent += bytes_read;
break;
}
} while (total_bytes_sent < file_size_in_bytes);
gettimeofday(&end_time, NULL);
printf("File size: %lu bytes, total bytes read from file: %lu, ", file_size_in_bytes, total_bytes_sent);
calculate_execution_time(start_time, end_time);
free(buffer);
return 0;
}
int read_from_file_sendfile(int client_sockfd, char *file_name, int fd, off_t file_size_in_bytes, int chunk_size)
{
ssize_t bytes_sent = 0, total_bytes_sent = 0;
off_t offset = 0;
struct timeval start_time, end_time;
gettimeofday(&start_time, NULL);
do {
bytes_sent = sendfile(client_sockfd, fd, &offset, chunk_size);
if (bytes_sent == -1) {
printf("Failed to sendfile: %s, offset: %lu, error: %d\n", file_name, offset, errno);
return -1;
}
total_bytes_sent += bytes_sent;
} while (total_bytes_sent < file_size_in_bytes);
gettimeofday(&end_time, NULL);
printf("File size: %lu bytes, total bytes read from file: %lu, ", file_size_in_bytes, total_bytes_sent);
calculate_execution_time(start_time, end_time);
return 0;
}
int read_from_file(int client_sockfd, char *file_name, char *type, int chunk_size)
{
int error_code = 0, fd = 0;
ssize_t hdr_length = 0, bytes_sent = 0, file_name_length = strlen(file_name);
struct stat file_stat = {0};
struct file_details *file_details_to_send = NULL;
fd = open(file_name, O_RDONLY, S_IRUSR);
if (fd == -1) {
printf("Failed to open file: %s, error: %d\n", file_name, errno);
return -1;
}
error_code = fstat(fd, &file_stat);
if (error_code == -1) {
printf("Failed to get status of file: %s, error: %d\n", file_name, errno);
close(fd);
return -1;
}
hdr_length = (sizeof(struct file_details) + file_name_length + 1);
file_details_to_send = calloc(hdr_length, sizeof(char));
if (file_details_to_send == NULL) {
perror("Failed to allocate memory");
close(fd);
return -1;
}
file_details_to_send->file_name_length = file_name_length;
file_details_to_send->file_size_in_bytes = file_stat.st_size;
strcpy(file_details_to_send->file_name, file_name);
printf("File name: %s, size: %lu bytes\n", file_name, file_stat.st_size);
bytes_sent = send(client_sockfd, file_details_to_send, hdr_length, 0);
if (bytes_sent == -1) {
printf("Failed to send header of size: %lu bytes, error: %d\n", hdr_length, errno);
close(fd);
return -1;
}
if (strcmp(type, "rw") == 0) {
printf("By pread and send\n");
read_from_file_pread(client_sockfd, file_name, fd, file_stat.st_size, chunk_size);
} else {
printf("By sendfile\n");
read_from_file_sendfile(client_sockfd, file_name, fd, file_stat.st_size, chunk_size);
}
close(fd);
return 0;
}
int main(int argc, char *argv[])
{
...
...
option_value = 1;
error_code = setsockopt(client_sockfd, SOL_TCP, TCP_NODELAY, &option_value, sizeof(int));
if (error_code == -1) {
printf("Failed to set socket option TCP_NODELAY to socket descriptor: %d, error: %d", client_sockfd, errno);
}
read_from_file(client_sockfd, file_name, type, chunk_size);
...
}
Your code almost certainly made a big performance improvement. The problem might be that you're measuring wall time. Consider calling getrusage() instead of gettimeofday(). The ru_utime and ru_stime fields represent how much time the kernel and your program spent doing actual work. sendfile() should make those numbers go down. That way you consume less energy, and free up more resources for other programs on your computer. Unfortunately however it can't make the network go faster. Optimal wall time speed to send 2GB on 1GbPS ethernet assuming zero overhead would be ~9s. You're pretty close.
I have a bunch of weird characters in my output file when I send it from the client to the server. When I stream the server into an array and print it out, it looks good and legible. However, when I send it over to the server, I get a bunch of weird characters and MICROSOFT everywhere in the text document. Anybody know what's wrong?
Client:
if(sendSize <=0){
for(;;){
unsignedchar buff[256]={0};
int nread = fread(buff,1,256, fp);
total = total + nread;
percentage =(total / fFileSize)*100;
printf("\r%s: Percentage sent: %.2f", NAME_C, percentage);
/* Send data in 256 byte chunks */
if(nread >0){
send(clientSock, buff, nread, 0);
}
if(nread <256){
if(feof(fp)){
printf("\nSend Success!\n");
break;
}
}
}
//printf("%.2f", total);
}
Server:
/* Receive data from client */
char* fileName ="test.txt";
FILE*fp = fopen(fileName,"w+");;
float total =0;
float bytesReceived;
unsignedchar buff[256]={0};
float percentage =(bytesReceived / total)*100;
while((bytesReceived = recv(listenSock, buff,sizeof(buff),0))<0){
//bytesReceived = recv(listenSock, buff, 256, 0);
if(bytesReceived >0){
printf("DONE");
}
//total = total + bytesReceived;
fwrite(buff,sizeof(char), bytesReceived, fp);
//printf("\r%s: Percentage received: %.2f", NAME_C, percentage);
}
Your server's recv() loop is using <0 when it should be using >0 instead. recv() returns -1 on error, 0 on graceful disconnect, and >0 on bytes received.
Also, is listenSock the actual listening socket, or the socket returned by accept()? You should be passing the latter to recv(). My guess is that you are passing the former instead, causing recv() to fail and return -1, which you then enter the body of the loop with garbage buff data and a bad bytesReceived value. The third parameter of fwrite() is a size_t, which is an unsigned type, so passing a signed value of -1 will get interpreted as an unsigned value of 4294967295 or even 18446744073709551615, depending on the size of size_t. Either way, you would be writing garbage to your file, or even crashing the code trying to access invalid memory.
There are other minor issues with your code as well. Try something more like this instead:
Client:
if(sendSize <=0){
unsigned char buff[256];
do{
int nread = fread(buff, sizeof(char), 256, fp);
if (nread > 0){
total += nread;
percentage = (total / fFileSize)*100;
printf("\r%s: Percentage sent: %.2f", NAME_C, percentage);
/* Send data in 256 byte chunks */
if (send(clientSock, buff, nread, 0) == -1){
printf("\nSend Failed!\n");
break;
}
}
if (nread != 256){
if (feof(fp)){
printf("\nSend Success!\n");
else
printf("\nRead Failed!\n");
break;
}
}
while (1);
//printf("%.2f", total);
}
Server:
/* Receive data from client */
char* fileName = "test.txt";
FILE* fp = fopen(fileName, "wb+");
if (!fp){
printf("\nOpen Failed!\n");
}
else{
float total = 0;
float percentage = 0;
int bytesReceived;
unsigned char buff[256];
do{
bytesReceived = recv(acceptedSock, buff, sizeof(buff), 0);
if (bytesReceived <= 0){
if (bytesReceived < 0){
printf("\nRecv Failed!\n");
}
else{
printf("\nDisconnected!\n");
}
break;
}
//total += bytesReceived;
//percentage = ...
if (fwrite(buff, sizeof(char), bytesReceived, fp) != bytesReceived){
printf("\nWrite Failed!\n");
break;
}
//printf("\n%s: Percentage received: %.2f", NAME_C, percentage);
}
while (1);
}
Trying to read a file in blocks and send each of those blocks over a TCP connection. When the read/send loop is complete, I send a confirmation message. Any time my file is larger than one block, which is almost always, the confirmation message never arrives. Can't tell if that is because it didn't get sent or just not received. It appears it was sent but I can't be sure. For small files, this works just fine. The file itself is sent correctly in all cases, but I need this confirmation message to send, too.
Can anyone see why this might be happening?
int header_size = sizeof("file,,") + sizeof(int);
int bytes_remaining, filesize;
fseek(fp1, 0L, SEEK_END);
filesize = bytes_remaining = ftell(fd);
fseek(fd, 0L, SEEK_SET);
int bytes_to_read;
if ( (BUFFSIZE - header_size ) < bytes_remaining) {
bytes_to_read = BUFFSIZE - header_size;
} else {
bytes_to_read = bytes_remaining;
}
while (bytes_read = fread(buffer, 1, bytes_to_read, fd) > 0) {
sprintf(message, "file,%d,%s", bytes_to_read, buffer);
send(sd, message, bytes_to_read + header_size, 0);
bytes_remaining -= bytes_to_read;
if ( (BUFFSIZE - header_size) < bytes_remaining) {
bytes_to_read = BUFFSIZE - header_size;
} else {
bytes_to_read = bytes_remaining;
}
bzero(buffer, BUFFSIZE);
}
// send confirmation message
bzero(buf, 256);
sprintf(buf, "send_complete");
send(sd, buf, 256, 0);
fprintf(stdout, "complete: %s\n", buf);
send(), just as write() or fwrite doesn't guarantee that all data is consumed.
You must check the return value from send() for how many bytes was actually sent.
I'm guessing that you miss the confirmation message because the TCP output buffer is full at that point.
I'm having some problems with two programs I wrote, a server and a client. To keep it easy and chronological, I first wrote the server and tested it with both telnet and netcat and everything works fine (except for a difference in the return value of read() / recv(), as it looks like the normal telnet program adds an additional char at the end of the string to be sent, but anyway...).
Now I have written the client program too, but I am not getting all the data which was correctly received by the other two clients, in particular the row[i] strings I get from the MySQL query. Things change when I introduce a usleet() call after each send() function and all data is received correctly.
Now I was thinking about an incompatible buffer size problem (?), but after playing for a while and checking the dimensions, I was not able to find out anything.
You will find the code below, please if you have any advice don't hesitate to tell me.
Tnx
/* CLIENT CODE */
#define BUFSIZE 1000
...
void *send_handler(void *);
...
int main(int argc, char *argv[]) {
char buf[BUFSIZE];
...
socket stuff...
...
connect
...
/* receive string from server - working */
bzero(buf, BUFSIZE);
n = read(sockfd, buf, BUFSIZE);
if (n < 0)
error("ERROR reading from socket");
buf[n] = '\0';
printf("%s", buf);
...
/* send username to server - working */
bzero(buf, BUFSIZE);
fgets(buf, BUFSIZE, stdin);
n = write(sockfd, buf, strlen(buf));
if (n < 0)
error("ERROR writing to socket");
...
/* start receiving handler */
if( pthread_create( &thread_id , NULL , send_handler , (void*) &sockfd) < 0) {
perror("could not create thread");
return 1;
}
/* main thread for reading data */
while(1) {
bzero(buf, BUFSIZE);
n = read(sockfd, buf, BUFSIZE);
if (n < 0)
error("ERROR reading from socket");
buf[n] = '\0';
printf("%s", buf);
}
close(sockfd);
return 0;
}
void *send_handler(void *socket_desc) {
//Get the socket descriptor
int sock = *(int*)socket_desc;
char buf[BUFSIZE];
int n;
while (1) {
bzero(buf, BUFSIZE);
fgets(buf, BUFSIZE, stdin);
n = write(sock, buf, strlen(buf));
if (n < 0)
error("ERROR writing to socket");
}
}
/* SERVER CODE */
void *connection_handler(void *);
int main(int argc , char *argv[]) {
...
/* socket variables */
...
pthread_t thread_id;
...
socket stuff...
...
while((client_sock = accept(socket_desc, (struct sockaddr *)&client_addr, (socklen_t*)&client_len))) {
if( pthread_create( &thread_id , NULL , connection_handler , (void*) &client_sock) < 0) {
perror("could not create thread");
return 1;
}
}
return 0;
}
void *connection_handler(void *socket_desc) {
//Get the socket descriptor
int sock = *(int*)socket_desc;
...
/* mysql variables */
char cmd[1000];
...
MYSQL_RES *result;
MYSQL_ROW row;
MYSQL *con;
...
/* connection variables */
int read_size, i;
char *message;
char client_message[2000];
char buffer[1000];
...
//clear the buffers
memset(client_message, '\0', 2000);
...
snprintf(cmd, 999, "SELECT field1, field2, field3, field4 FROM files WHERE key='%s' ORDER BY id DESC", var);
if (mysql_query(con, cmd)) {
error checks...
}
result = mysql_store_result(con);
if (result == NULL) {
error checks...
}
else {
num_rows = mysql_num_rows(result);
}
if (num_rows == 0) {
message = "Nothing found\n";
send(sock , message , strlen(message), 0);
}
else {
num_fields = mysql_num_fields(result);
num_rows = mysql_num_rows(result);
snprintf(buffer, 999, "Number of rows: %d\n", num_rows);
send(sock , buffer , sizeof(buffer), 0);
//usleep(10000); // commented, but necessary to work properly...
memset(buffer, '\0', sizeof(buffer));
while ((row = mysql_fetch_row(result))) {
for(i = 0; i < num_fields; i++) {
snprintf(buffer, 999, "%s\t", row[i] ? row[i] : "NULL");
send(sock , buffer , sizeof(buffer), 0);
//usleep(10000);
memset(buffer, '\0', sizeof(buffer));
}
message = "\n";
send(sock , message , strlen(message), 0);
//usleep(10000);
}
message = "\n";
send(sock , message , strlen(message), 0);
//usleep(10000);
mysql_free_result(result);
}
...
}
EDIT: I changed
printf("%s", buf);
with
printf("Bytes read: %d\n", n);
in the client code and I obtained following output:
with commented usleep():
Bytes read: 1000
Bytes read: 1000
Bytes read: 1000
Bytes read: 1000
Bytes read: 1000
Bytes read: 1000
Bytes read: 1000
Bytes read: 1000
Bytes read: 1000
Bytes read: 36
Bytes read: 1000
Bytes read: 31
Bytes read: 1000
Bytes read: 1000
Bytes read: 1000
Bytes read: 1000
Bytes read: 2
(17 lines)
with usleep(0) slowing down the send flow (correct output obtained):
Bytes read: 1000
Bytes read: 33
Bytes read: 1000
Bytes read: 1000
Bytes read: 1000
Bytes read: 1000
Bytes read: 1
Bytes read: 1000
Bytes read: 1000
Bytes read: 1000
Bytes read: 1000
Bytes read: 1
Bytes read: 1
Bytes read: 1000
Bytes read: 31
Bytes read: 1000
Bytes read: 1000
Bytes read: 1000
Bytes read: 1000
Bytes read: 1
Bytes read: 1
(21 lines)
Any hint?
SOLVED: just by replacing
sizeof(buffer);
with
strlen(buffer);
in the server part and everything works fine even without usleep(), the output is correct/complete.
Thanks anyway.
You can't assume that a single read reads an entire message. There are no messages in TCP, only bytes, and any given read may return as few as one byte, or the result of several writes at the peer all at once. You have to loop and parse.
You have a race condition in your code, you are passing a pointer to a local variable to the thread function. Since a filedescriptor (int) is not larger than a void pointer (I'm pretty sure that is guaranteed, but add an assertion nonetheless), you can also convert the descriptor value to a pointer instead of passing the address of the local filedescriptor:
int s = accept(...);
if(int e = pthread_create(.., &connection_handler, (void*)s, ..))
error(..);
BTW: pthread_create returns zero on success and otherwise an error code, which is not negative. But it's very unlikely that your code failed at that point.
My server code is as follows:
while(bytes_written < filesize){
//Send from send_ptr
bw = send(child_socket, send_ptr, newLength, 0);
printf("%d bytes written\n", (int)bw);
//Increment bytes written
bytes_written += bw;
//Move send pointer
send_ptr = send_ptr + bw;
}
And my client code is as follows:
while((num_bytes_recv = read(sd, jpgBufferPointer, BUFFER_LENGTH))>0){
total_bytes_recv += num_bytes_recv;
printf("Read %d bytes\n",num_bytes_recv);
//Check for error
if(jpgError == NULL)
jpgError = strstr(jpgBufferPointer, "404 Not Found");
if(jpgError != NULL){
//Forwarding error response
if(send(sd, jpgBuffer, num_bytes_recv, 0) == -1){
error("Failed to send response message to client");
}
}
else{
//Find content size
contentSizeBuffer = strstr(jpgBufferPointer,"Content-Length");
if(contentSizeBuffer != NULL){
contentSizeBuffer=contentSizeBuffer+16;
contentSize=atoi(contentSizeBuffer);
jpgBuffer=(char*)realloc(jpgBuffer,(contentSize+FILE_NAME_LENGTH*2)*sizeof(char));
jpgBufferPointer=jpgBuffer;
}
jpgBufferPointer+=num_bytes_recv;
}
}
The server is saying it has sent all 43000 bytes, but client says it has received only 32768 bytes.
Appreciate any help! Thanks
You have a bug in the sending part, you should update newLength, because if you have 1 byte left to send from the file, it will send more, going out of the memory area where the content you want to send is stored. You should fix in this way:
bw = send(child_socket, send_ptr, newLength<(filesize-bytes_written)?newLength:(filesize-bytes_written), 0);
In this way the last send will have the correct size.
Also, use write instead of send if you are not using any flags.
You need to have the similar loop as you have on the writing side (bytes_written < filesize) on the reading side (i.e., while you can read more bytes, you should read them and append them).
The network doesn't guarantee that one read() call will return all available data.
The best way of writing client-server socket programming is to have a header before your data. The header should state the amount of data that it is going to transfer.
For example, To send data "Hello World", then send it as "0011+HELLO WORLD"
Here 11 stands for the size of the data the sender is planning to send now. The receiver on reading the first 4 bytes can understand that he should be ready to read next 11 bytes of data from the sender.
So reader will do two read:
hRead = 5 /* With 5 you are saying it can read upto max of 9999 bytes from data".
read(sd, buff, hRead);
dRead = atoi(buff);
readn(sd, buff, dRead);
For Example : Server
size_t sendn(int fd, const void *vptr, size_t n) {
size_t nleft;
size_t nwritten;
const char *ptr;
ptr = vptr;
nleft = n;
while (nleft > 0) {
if ((nwritten = send(fd, vptr, nleft, 0)) <= 0) {
if (errno == EINTR)
nwritten = 0;
else {
fprintf(stderr, "send failed %d - %s\n", fd, strerror(errno));
return (-1);
}
}
nleft -= nwritten;
ptr += nwritten;
}
return (n);
}
To send message:
sprintf(buff, "%d + %d + %s\r\n", MSG_LOGIN, strlen("Hello World"), Hello World);
sendn(sd, buff, strlen(buff));
Client:
size_t readn(int fd, void *vptr, size_t n) {
size_t nleft;
size_t nread;
char *ptr;
ptr = vptr;
nleft = n;
while (nleft > 0) {
if ((nread = recv(fd, ptr, nleft, 0)) < 0) {
if (errno == EINTR)
nread = 0;
else {
fprintf(stderr, "read failed %d - %s\n", fd, strerror(errno));
return (-1);
}
} else if (nread == 0)
break;
nleft -= nread;
ptr += nread;
}
return (n - nleft);
}