Sending huge array through socket - c

Question

I am trying to send a char array of 100.000.000 bytes (maybe more) from a server to a client through a TCP socket in C.
I am doing this like that:
char *array; // global array malloc'd (SIZE)
//#######################
// server code
//#######################
int i;
int SIZE = 100000000
for (i = 0; i < SIZE; i = i + 4){
write(id, &array[i], 4); // write 4 bytes every time
}
//#######################
// client code
//#######################
int i;
int SIZE = 100.000.000
for (i = 0; i < SIZE; i = i + 4)
read(id, array + i, 4); // read 4 bytes
Problems:
1) When I try to send more bytes something goes wrong with the transfer. For example if I change 4 to 100, it says "broken pipe". Why does that happen?
2) I know this is not a "safe" way for reading/writing since I am not checking read() and write() return values. How can I do that?
3) Do I have to use htonl() and ntohl() functions?

Answer 1

#include<stdlib.h>
#include<stdio.h>
#include<string.h>
#include <sys/types.h>
#include <sys/socket.h>
//in #param
//#param fd the socket file descriptor
//#param array an array of data source to write to send to the connected client
//#param SIZE the size of data source to send to the client
//#param sz_emit the size of data to send in one loop step
//out #param
//total length of data emited to the client
int write_to_client(int fd, char* array, int SIZE, int sz_emit)
{
//#######################
// server code
//#######################
int i=0, sz=0;
for(i = 0; i < SIZE; i += sz_emit )
{
while(sz_emit-sz)
{
sz+=write(id, array+i+sz, sz_emit-sz);
}
sz = 0;
}
return i;
}
//#######################
// client code
//#######################
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
//in #param
//#param fd is the file descriptor of the socket to read from
//#param SIZE the size of datas you want to read from the socket
//#param sz_received the size of byte to read in one loop step
//#param length, the length of data received
//#param read_err if 0 no error if -1 an error occurs use errno from #include <errno.h> to know more about that error
//out #param
//a pointer to an array of size SIZE containing the data readed
char* receive_from_server(int fd, int SIZE, int sz_received, int* length, int* read_err)
{
*read_err = 0;
int i = 0, sz = 0, rt = 0, count=0;
char *array = (char *)malloc(SIZE);
memset(array, 0, SIZE);
for (i = 0; i < SIZE; i += sz_received)
{
while(sz_received-sz)
{
rt = read(id, array + i + sz, sz_received-sz);
if(rt==-1)
{
*read_err=rt;
printf("an error occurs\n");
goto l;
}
if(!rt)goto l;
sz+=rt;
count += sz;
}
sz = 0;
}
l: *length = count;
return array;
}
usage:
//server side
int SIZE = 100000000;
char array_to_send[SIZE]={'r'};
int sz_data_emited = write_to_client(sock, array_to_send, SIZE, 4);
printf("how many byte data emited:%d\n", sz_data_emited);
//client side
int SIZE = 100000000, length = 0, read_err=0;
char*array_received = NULL;
array_received = receive_from_server(sock, SIZE, 4, &length, &read_err);
if(!read_err)printf("get some datas\n");
// free array_received when finished...free(array_received)
some notes:
you need to pay attention on endianess when you want to transfert a multi-byte entity for example a short, int, long, utf-16 etc but if your datas are utf-8 or ascii text you don't need it.

Answer 2

I think that Narcisse Doudieu Siewe answer has some errors. I suppose it fails when SIZE isn't multiplicity of sz_emit.
Example there is 20 bytes that we sent in 8 bytes chunkes than the last data chunk (or packet) will be 4 byte long. Than if we try to send the last 8 bytes chunk and there is only 4 bytes left while loop will be infinite as while(8 - 4), and it never reaches sz = 8, as the next send will just increment by 0. So I write modification like this (not tested, I will test soon and write also second method taking into account this boundary condition).
/**
* #param sock_fd - the file descriptor of the socket to write (send) data to
* #param packetLength - the size of data to send in one packet
* #param data - binary data to send (unsigned char array)
* #param dataLength - the size of all binary data to send
* #return - status code SUCCESS or FAILURE
*/
result_t send_binary(const sock_fd_t sock_fd, const size_t packetLength, const unsigned char *data, const size_t dataLength) {
ssize_t leftPacketLength = 0;
ssize_t offset = 0;
ssize_t sentPacketLength = 0;
// send each packet of data in the loop
for(int leftDataLength=dataLength; leftDataLength>0; leftDataLength -= packetLength) {
leftPacketLength = (leftDataLength > packetLength) ? packetLength : leftDataLength;
while(leftPacketLength > 0) {
sentPacketLength = send(sock_fd, data + offset, leftPacketLength, 0);
if(sentPacketLength < 0) {
fprintf(stderr, "%s: Error while sending data to the socket.\n", __func__);
perror(errno);
return FAILURE;
}
offset += sentPacketLength;
leftPacketLength -= sentPacketLength;
}
}
if(offset != dataLength)
return FAILURE;
return SUCCESS;
}
/**
* #param sock_fd - the file descriptor of the socket to read (recieve) data from
* #param packetLength - the size of data to recieve in one packet
* #param data - binary data received (unsigned char array) - previously allocated
* #param dataLength - the size of all binary data received - previously defined
* #return - status code SUCCESS or FAILURE
*/
result_t recv_binary(const sock_fd_t sock_fd, const size_t packetLength, unsigned char *data, const size_t dataLength) {
ssize_t leftPacketLength = 0;
ssize_t offset = 0;
ssize_t recvedPacketLength = 0;
for(int leftDataLength=dataLength; leftDataLength > 0; leftDataLength -= packetLength) {
leftPacketLength = (leftDataLength > packetLength) ? packetLength : leftDataLength;
while(leftPacketLength > 0) {
recvedPacketLength = recv(sock_fd, data + offset, leftPacketLength, 0);
if(recvedPacketLength < 0) {
fprintf(stderr, "%s: Error while receiving data from the socket.\n", __func__);
perror(errno);
return FAILURE;
}
offset += recvedPacketLength;
leftPacketLength -= recvedPacketLength;
}
}
if(offset != dataLength)
return FAILURE;
return SUCCESS;
}
There is also need to send size of binary data send/recv via socket before transfering actual binary data. It's needed to know how many bytes we need to read.