I've been playing around libcrypto's BIO, and I can't find a way to detect errors during base64 decoding.
Even if data is complete garbage, BIO_read just returns zero, and error queue - as examined with ERR_get_error - remains empty.
The same issue happens with BIO_FLAGS_BASE64_NO_NL flag: in case of mismatch (that is, data contains newlines, but the flag is set; and vice versa), there's no indication of error, there's just no data.
So, is there a way to catch decoding errors?
static unsigned char *base64_decode(unsigned char *data, size_t len, size_t *out_len)
{
// chain should look like this
// b64 - mem
// so when we read from b64, it gets data from mem and decodes it
BIO *bio_b64;
BIO *bio_mem;
size_t res_capacity;
size_t res_size;
unsigned char *res;
size_t ret;
bio_b64 = BIO_new(BIO_f_base64());
bio_mem = BIO_new_mem_buf(data, len);
res_capacity = 1024;
res_size = 0;
res = malloc(res_capacity);
// don't care about newlines
BIO_set_flags(bio_b64, BIO_FLAGS_BASE64_NO_NL);
BIO_push(bio_b64, bio_mem);
// empty error queue, just in case
while (ERR_get_error() != 0);
while (1) {
ret = BIO_read(bio_b64, &res[res_size], res_capacity - res_size);
if (ret == (res_capacity - res_size)) {
res_size += ret;
res_capacity *= 2;
res = realloc(res, res_capacity);
} else if (ret == 0) {
break;
} else {
res_size += ret;
}
}
if (ERR_get_error() != 0) {
free(res);
return NULL;
}
BIO_free_all(bio_b64);
*out_len = res_size;
return res;
}
Unfortunately BIO_read does't emit an error if malformed data. You can either check expected size (4/3) or reimplement without openssl.
Related
I suspect this is a basic issue, I just have a moment of dumbness.
I'm trying to make very simple mqtt client with MQTT-C library as a part of testing tool of my other solution.When I receive subscribed message I have weired issue : inside mqtt-c library (just before a call to a function pointer which happens to be mine "subsriber callback") everything seems ok, but right after a call to my function, dereferencing a pointer shows struct with completly wrong values inside. If I go one step up in callstack, so back to mqtt, the values inspector gives (set to dereference uint32_t value - precisely pointer's address), again, correct structure.
The structure, pointer to which mqtt passes to callback, is allocated on stack inside function in mqtt library. I don't think that thread could change, which would invalidate stack. I have also included mqtt.h, which means that my function should understand the structure.
I will paste some relevant code snippets, any help will be appreciated, including general advices.
ssize_t __mqtt_recv(struct mqtt_client *client)
{
struct mqtt_response response;
ssize_t mqtt_recv_ret = MQTT_OK;
MQTT_PAL_MUTEX_LOCK(&client->mutex);
/* read until there is nothing left to read, or there was an error */
while(mqtt_recv_ret == MQTT_OK) {
/* read in as many bytes as possible */
ssize_t rv, consumed;
struct mqtt_queued_message *msg = NULL;
rv = mqtt_pal_recvall(client->socketfd, client->recv_buffer.curr, client->recv_buffer.curr_sz, 0);
if (rv < 0) {
/* an error occurred */
client->error = (enum MQTTErrors)rv;
MQTT_PAL_MUTEX_UNLOCK(&client->mutex);
return rv;
} else {
client->recv_buffer.curr += rv;
client->recv_buffer.curr_sz -= (unsigned long)rv;
}
/* attempt to parse */
consumed = mqtt_unpack_response(&response, client->recv_buffer.mem_start, (size_t) (client->recv_buffer.curr - client->recv_buffer.mem_start));
if (consumed < 0) {
client->error = (enum MQTTErrors)consumed;
MQTT_PAL_MUTEX_UNLOCK(&client->mutex);
return consumed;
} else if (consumed == 0) {
/* if curr_sz is 0 then the buffer is too small to ever fit the message */
if (client->recv_buffer.curr_sz == 0) {
printf("receive buff sz is zero??\n");
client->error = MQTT_ERROR_RECV_BUFFER_TOO_SMALL;
MQTT_PAL_MUTEX_UNLOCK(&client->mutex);
return MQTT_ERROR_RECV_BUFFER_TOO_SMALL;
}
/* just need to wait for the rest of the data */
MQTT_PAL_MUTEX_UNLOCK(&client->mutex);
return MQTT_OK;
}
switch (response.fixed_header.control_type) {
//(...)
case MQTT_CONTROL_PUBLISH:
/* stage response, none if qos==0, PUBACK if qos==1, PUBREC if qos==2 */
if (response.decoded.publish.qos_level == 1) {
rv = __mqtt_puback(client, response.decoded.publish.packet_id);
if (rv != MQTT_OK) {
client->error = (enum MQTTErrors)rv;
mqtt_recv_ret = rv;
break;
}
} else if (response.decoded.publish.qos_level == 2) {
/* check if this is a duplicate */
if (mqtt_mq_find(&client->mq, MQTT_CONTROL_PUBREC, &response.decoded.publish.packet_id) != NULL) {
break;
}
rv = __mqtt_pubrec(client, response.decoded.publish.packet_id);
if (rv != MQTT_OK) {
client->error = (enum MQTTErrors)rv;
mqtt_recv_ret = rv;
break;
}
}
/* call publish callback */
printf("address: %d; size: %d\n", (uint32_t) &response.decoded.publish, response.decoded.publish.application_message_size);
//all ok here.
client->publish_response_callback(&client->publish_response_callback_state, &response.decoded.publish);
break;
//(...)
{
/* we've handled the response, now clean the buffer */
void* dest = (unsigned char*)client->recv_buffer.mem_start;
void* src = (unsigned char*)client->recv_buffer.mem_start + consumed;
size_t n = (size_t) (client->recv_buffer.curr - client->recv_buffer.mem_start - consumed);
memmove(dest, src, n);
client->recv_buffer.curr -= consumed;
client->recv_buffer.curr_sz += (unsigned long)consumed;
}
}
/* In case there was some error handling the (well formed) message, we end up here */
MQTT_PAL_MUTEX_UNLOCK(&client->mutex);
return mqtt_recv_ret;
}
struct mqtt_response {
/** #brief The mqtt_fixed_header of the deserialized packet. */
struct mqtt_fixed_header fixed_header;
/**
* #brief A union of the possible responses from the broker.
*
* #note The fixed_header contains the control type. This control type corresponds to the
* member of this union that should be accessed. For example if
* fixed_header#control_type == \c MQTT_CONTROL_PUBLISH then
* decoded#publish should be accessed.
*/
union {
struct mqtt_response_connack connack;
struct mqtt_response_publish publish;
struct mqtt_response_puback puback;
struct mqtt_response_pubrec pubrec;
struct mqtt_response_pubrel pubrel;
struct mqtt_response_pubcomp pubcomp;
struct mqtt_response_suback suback;
struct mqtt_response_unsuback unsuback;
struct mqtt_response_pingresp pingresp;
} decoded;
};
static void netReqHandler(void) {
int sockfd = myOpenSocket(ipaddr, port);//open_nb_socket(ipaddr, port);
mqtt_init(&client, sockfd, sendbuf, buffersizes, receivebuf, buffersizes, publish_callback);
printf("client error is %d\n", (&client)->error);
void publish_callback(void** unused, struct mqtt_response_publish *published) {
printf("app msg size : %d, addr: %d\n", published->application_message_size, (uint32_t) published);
char *nullTerminatedMessage = malloc(published->application_message_size + 1);
strncpy(nullTerminatedMessage, published->application_message, published->application_message_size);
nullTerminatedMessage[published->application_message_size] = '\0';
I want to write a recvall function for socket in C. I assume, that every message in my protocol ends with \r\n. I wrote something like this below:
int recvall (int socket, char *buffer, int numBytes)
{
int bytesRcvd = 0;
numBytes = numBytes - 1;
while(bytesRcvd < numBytes)
{
int chunk = recv(socket, buffer + bytesRcvd, numBytes - bytesRcvd, 0);
if (chunk == 0 || chunk == -1)
break;
if(strstr(buffer, "\r\n"))
break;
bytesRcvd += (chunk);
}
buffer[bytesRcvd] = '\0';
return bytesRcvd;
}
But it shows me that it returns and reads 0 bytes. On the other hand, when I remove:
if(strstr(buffer, "\r\n"))
break;
it hangs. How to improve it?
One mistake here is that strstr expects a zero-terminated string. A fix:
buffer[chunk] = 0;
if(strstr(buffer, "\r\n"))
break;
However, there may be more data following "\r\n", and that data gets lost here.
A common design pattern for receiving data is:
Have a class that maintains a connection with send/receive buffers.
When it has received data into the receive buffer, it calls a message parsing callback passing buffer and buffer_size.
The message parsing callback consumes all available complete messages in the buffer and returns the number of bytes consumed (as you do).
The message parsing callback calls another callback, passing a complete message to it.
Something like this:
typedef struct
{
int socket;
// Message-parsing callback.
size_t(*on_recv_cb)(void*, void*, size_t);
void* on_recv_cb_data;
unsigned char recv_buffer[256 * 1024];
size_t recv_buffer_size; // Initialized with 0;
} Connection;
void Connection_on_socket_ready_for_read(Connection* self) {
// Assumes a non-blocking socket. Read once to avoid starvation of other sockets.
ssize_t received = read(self->socket, self->recv_buffer + self->recv_buffer_size, sizeof self->recv_buffer - self->recv_buffer_size);
if(received > 0) {
self->recv_buffer_size += received;
size_t consumed = self->on_recv_cb(self->on_recv_cb_data, self->recv_buffer, self->recv_buffer_size);
self->recv_buffer_size -= consumed;
memmove(self->on_recv_cb_data, self->recv_buffer + consumed, self->recv_buffer_size);
}
else if(received < 0) {
if(EAGAIN == errno)
return;
perror("error");
// Handle error.
}
else {
// Handle EOF.
}
}
typedef struct {
// Message callback.
void(*on_message_cb)(void*, char*, size_t);
void* on_message_cb_data;
} MessageParserCrLf;
size_t MessageParserCrLf_on_recv(void* cb_data, void* data, size_t data_size) {
MessageParserCrLf* self = cb_data;
char* message_begin = data;
char* message_end = data;
while(data_size - (message_end - (char*)data) >= 2) {
if(message_end[0] == '\r' && message_end[1] == '\n') {
message_end += 2;
self->on_message_cb(self->on_message_cb_data, message_begin, message_end - message_begin);
message_begin = message_end;
}
else {
++message_end;
}
}
return message_begin - (char*)data;
}
void on_message(void* cb_data, char* message, size_t message_size) {
(void)cb_data; // Unused here.
printf("on_message: %.*s\n", (int)message_size, message);
}
int main() {
MessageParserCrLf message_parser = {on_message, NULL};
Connection connection = {0, MessageParserCrLf_on_recv, &message_parser, {}, 0};
Connection_on_socket_ready_for_read(&connection);
}
Outputs:
[~/src/test]$ cat lines.txt
abc
def
end
[~/src/test]$ ./test < lines.txt
on_message: abc
on_message: def
on_message: end
If there is "\r\n" in the received buffer, the following line will break and end the while loop,
if(strstr(buffer, "\r\n")) break;
but bytesRcvd hasn't increased, so the function will return with bytesRcvd=0.
Try to put bytesRcvd increase before judging like this,
bytesRcvd += (chunk);
if(strstr(buffer, "\r\n"))
break;
my task is get from server data of an file with HTTP protocol
but problem is that i dont know the size of the content so i iterate through read func to get data from socked but it iterates every time only once i dont know really why
int res, len, total, boolk = 0, p=0;
while ((res = read(client_socket,bufferOut,4095)) > 0)
{
bufferOut[res]= '\0';
if(p==0)
{
buffer = calloc(strlen(bufferOut)+1,sizeof(char));
}
else
{
buffer = realloc(buffer,strlen(buffer)*sizeof(char)+strlen(bufferOut)+1*sizeof(char));
}
strcat(buffer,bufferOut);
if(isEnough(bufferOut)!=0 && boolk == 0)
{
index = getIndex(buffer);
kk = getCode(buffer);
len = getLen(buffer);
boolk = 1;
}
if(strlen(bufferOut)>=len+index && boolk == 1)
{
break;
}
p++;
}
function isEnough only looks if full http header arrived getIndex get lenght of header and getLen get length of that file so i wanted to iterate until buffer is same lenght as header file which should be
You're working much too hard.
int
read_until_enough(int client_socket, char **buf_out, size_t *buflen_out)
{
char *rdbuf = xmalloc(INITIAL_BUFFER_SIZE);
size_t buflen = 0, bufalloc = INITIAL_BUFFER_SIZE;
for (;;) {
ssize_t n = recv(client_socket, rdbuf + buflen, bufalloc - buflen);
if (n <= 0) goto recv_failure;
if (is_enough(rdbuf, buflen)) break;
buflen += n;
if (buflen == bufalloc) {
bufalloc *= 2;
rdbuf = xrealloc(rdbuf, bufalloc);
}
}
*rdbuf_out = rdbuf;
*buflen_out = buflen;
return 0;
recv_failure:;
int save_errno = errno;
free(rdbuf);
errno = save_errno;
*rdbuf_out = 0;
*buflen_out = 0;
return -1;
}
Please note that is_enough now takes the size of the received data as an argument, caller gets both the received data and its size, and the buffer is NOT nul-terminated. HTTP is a binary-transparent protocol; you MUST NOTrfc2119 assume that you never receive nul bytes.
Please also note that is_enough is stateless and is expected to do all of the parsing. This is better separation of concerns. Worry about the performance cost of re-parsing the HTTP headers each time around the loop only after you have everything working.
Finally, the functions xmalloc and xrealloc are wrappers around malloc and realloc that either succeed or crash the program. I am using them here to avoid cluttering up the example with error-recovery logic for memory allocation. You will have to decide whether they are appropriate for whatever you're doing.
I am currently working on the below requirement.
Here is the requirement: On the server side a large file is divided into 4000-byte blocks (frames). Each block is in turn compressed (using zlib) and sent to client process. For instance, if a file is 12000 bytes in size then it is divided into 3 blocks.
Above file will have 3 blocks => Block-0, Block-1, Block-2
On receipt, client decompresses each block (or frame) and writes to buffer allocated on the heap.When all the blocks corresponding to the entire file is received by the client, then the uncompressed version of the resultant file is written to the disk.
I have written a routine inflateData that does the following based on the block # received:
When the first block is received,
- inflateInit
- inflate
- inflateReset
When the intermediate blocks are received,
- inflate
- inflateReset
When the last block is received,
- inflate
- inflateEnd
With the above routine, Decompression of blocks happens as expected. But the issue that I face is it consumes lots of memory and at some point entire system slows down. When checked with valgrind, memory leak is reported with inflateInit2_. This causes the system resources to be exhausted.
==30359== 57,312 bytes in 6 blocks are possibly lost in loss record 64 of 67
==30359== at 0x4A069EE: malloc (vg_replace_malloc.c:270)
==30359== by 0x3E57808F1E: inflateInit2_ (in /lib64/libz.so.1.2.3)
==30359== by 0x40C220: inflateData (productMaker.c:1668)
Below is the routine inflateData.
int inflateData(
char* const inBuf,
unsigned long inLen,
unsigned int isFirstBlk,
unsigned int isLastBlk,
const char* outBuf,
unsigned long* outLen)
{
int have;
int readsz;
int bsize;
static z_stream zstrm;
int zerr;
int flush;
char out[CHUNK_SZ];
char in[CHUNK_SZ];
int ret,nwrite,idx = -1;
int savedByteCntr=0;
unsigned char *dstBuf;
int firstCall = 1;
int totalBytesIn=0;
int inflatedBytes=0;
int decompByteCounter = 0;
int num=0;
ret = Z_OK;
readsz = 0;
bsize = CHUNK_SZ;
dstBuf = (unsigned char *) outBuf;
if(isFirstBlk){
memset(&zstrm, '\0', sizeof(z_stream));
zstrm.zalloc = Z_NULL;
zstrm.zfree = Z_NULL;
zstrm.opaque = Z_NULL;
if ((zerr = inflateInit(&zstrm)) != Z_OK) {
uerror("ERROR %d inflateInit (%s)",
zerr, decode_zlib_err(zerr));
return -1;
}
}
while(totalBytesIn < inLen ) {
int compChunkSize = ((inLen - totalBytesIn) > 5120) ? 5120 :
(inLen - totalBytesIn);
memcpy(in, inBuf + totalBytesIn, compChunkSize);
zstrm.avail_in = inLen - totalBytesIn;
zstrm.next_in = in ;
zstrm.avail_out = CHUNK_SZ;
zstrm.next_out = out;
inflatedBytes = 0;
while(ret != Z_STREAM_END) {
ret = inflate(&zstrm, Z_NO_FLUSH);
if(ret < 0) {
uerror(" Error %d inflate (%s)", ret, decode_zlib_err(ret));
(void)inflateEnd(&zstrm);
return ret;
}
inflatedBytes = CHUNK_SZ - zstrm.avail_out;
if(inflatedBytes == 0) {
unotice("\n Unable to decompress data - truncated");
break;
}
totalBytesIn += zstrm.total_in;
decompByteCounter += inflatedBytes;
memcpy(dstBuf + savedByteCntr, out, inflatedBytes);
savedByteCntr = decompByteCounter;
}
// Reset inflater for additional input
ret = inflateReset(&zstrm);
if(ret == Z_STREAM_ERROR){
uerror(" Error %d inflateReset (%s)", ret, decode_zlib_err(ret));
(void)inflateEnd(&zstrm);
return ret;
}
}
if(isLastBlk){
ret = inflateEnd(&zstrm);
if(ret < 0) {
uerror("Fail inflateEnd %d [%s] ", ret, decode_zlib_err(ret));
return (ret);
}
}
*outLen = decompByteCounter;
return 0;
}
Thanks in advance for the support.
Thanks,
Sathya.
You are making an error in your use of your inflateData() routine.
First off, using a static variable in this way is a horrible idea. If you call your inflateData() twice with isFirstBlk true without an intermediate call with isLastBlk true, then you will wipe out the reference to the first set of allocations, resulting in a memory leak.
To avoid this sort of error, you should keep track of whether zstrm is initialized or not, and reject any attempt to initialize an already initialized stream. Better still would be to not even have an isFirstBlk, and simply initialize zstrm on the first call and on any call that immediately follows a call with isLastBlk true.
So you are either doing the above, calling twice with isFirstBlk true, or failing to call with isLastBlk true.
I'm trying to implement a circular buffer for use as a frame buffer with OpenCV (using C).
I've shamelessly stolen the circular buffer implementation from this post to save reinventing the wheel:
EDIT: OK, so I've redefined a few things. Namely I implemented my own circular buffer. Now I'm getting errors which don't make sense.
Here is the circular buffer implementation I'm using:
#define BUFFER_SIZE 100
typedef struct
{
IplImage* queue[BUFFER_SIZE];
IplImage *in;
IplImage *out;
int num_frames;
int in_ctr;
int out_ctr;
int update_flag;
} frame_buffer;
Here is the get function:
IplImage* buff_get()
{
IplImage* nextfr;
if(frbuff.num_frames == 0)
{
return NULL;
}
nextfr = frbuff.out++;
if(++frbuff.out_ctr == BUFFER_SIZE)
{
frbuff.out = &frbuff.queue[0];
frbuff.out_ctr = 0;
}
--frbuff.num_frames;
return nextfr;
}
Here is the put function:
int buff_put(IplImage* nextfr)
{
if(++frbuff.num_frames > BUFFER_SIZE)
{
return 0;
}
frbuff.in++;
frbuff.in = nextfr;
if(++frbuff.in_ctr == BUFFER_SIZE)
{
frbuff.in = &frbuff.queue[0];
frbuff.in_ctr = 0;
}
return 1;
}
Everything seems to go OK. Frames appear on the buffer, which I know because I can print the size out. But it all goes bad when I try to show the image that's on the buffer.
If I then try to do this:
IplImage* curr_frame = cvCreateImage(cvSize(640,480),8,3);
cvNamedWindow("proc_window",CV_WINDOW_AUTOSIZE);
cvShowImage("proc_window",curr_frame);
while(1)
{
if(buff_size() > 0)
{
if(buff_flag_check()) curr_frame = buff_get();
if(curr_frame != NULL)
{
cvShowImage("proc_window",curr_frame);
}
}
I recieve the following error upon calling cvShowImage():
OpenCV Error: Bad flag (parameter or structure field) (Unrecognized or unsupported array type) in cvGetMat, file /home/fagg/src/OpenCV-2.3.1/modules/core/src/array.cpp, line 2482
terminate called after throwing an instance of 'cv::Exception'
what(): /home/fagg/src/OpenCV-2.3.1/modules/core/src/array.cpp:2482: error: (-206) Unrecognized or unsupported array type in function cvGetMat
I'm quite confused as to what's going on here. Hopefully someone with fresher eyes than myself can see what's going on...
The snippet of code you provided won't compile as cb_init() takes a pointer to a circular_buffer as first argument.
and if you get a segfault in cb_init() it is because malloc() fails to return the requested size and the implementation in the link does not handle errors.
void cb_init(circular_buffer *cb, size_t capacity, size_t sz)
{
cb->buffer = malloc(capacity * sz);
if(cb->buffer == NULL)
// handle error
cb->buffer_end = (char *)cb->buffer + capacity * sz;//segfault when using cb->buffer which is null in case of malloc() failure
cb->capacity = capacity;
cb->count = 0;
cb->sz = sz;
cb->head = cb->buffer;
cb->tail = cb->buffer;
}