this is part of my code which reads an http response. It's supposed to increase the buffer size if it runs out of room. But i keep getting access violations. It happens when copying the data to the new buffer: memcpy(tmp_alloc, rec, ResponseLength); Any help/suggestions are appreciated.
#define SERVER_CHUNK 1024
char *rec = new char[10000];
char in_buff[SERVER_CHUNK];
int in_sz,
ResponseLength = 0,
rec_len = 10000;
in_sz = recv(ss,in_buff,SERVER_CHUNK,0);//get the response
while(in_sz > 0)
{
memcpy(rec + ResponseLength,in_buff,in_sz);
ResponseLength += in_sz;
if((ResponseLength + SERVER_CHUNK) > rec_len)
{
char *tmp_alloc = (char*) malloc (ResponseLength + SERVER_CHUNK);
if(!tmp_alloc)
{
printf("failed to alocate memory!\n");
break;
}
memcpy(tmp_alloc, rec, ResponseLength);
free(rec);
rec = tmp_alloc;
rec_len = ResponseLength + SERVER_CHUNK;
}
in_sz = recv(ss,in_buff,SERVER_CHUNK,0);
}
It's possible that you're corrupting the heap by mixing new[] with free(), which is not supported.
Change:
char *rec = new char[10000];
To:
char *rec = (char*) malloc( 10000);
and see if it makes any difference.
wouldn't it be better, to have this data as a list of buffers?
this way, you dont have to reallocate/copy all data every time you exceed your buffer.
you would only need to maintain a single/double linked list
Related
I am making unixlike file system as my university project and I am getting memory allocation error during making of file block with malloc method . Please help me. My code is as under :
char *BuildInode(int fileType)
{
int size = 0;
char *inode = (char *) malloc(sizeof(char) * SECTOR_SIZE_1/NUM_INODES_PER_BLOCK);
char *negativeOne = '\0';
inode[5] = fileType + 4;
int index;
for (index = 0; index < MAX_NUM_SECTORS_PER_FILE; index++)
{
inode[index + 2] = negativeOne;
}
return inode;
}
My guess is that SECTOR_SIZE_1 < NUM_INODES_PER_BLOCK and as a result (SECTOR_SIZE_1/NUM_INODES_PER_BLOCK is 0).
if SECTOR_SIZE_1 = 512 and NUM_INODE_BLOCKS = 250 then SECTOR_SIZE_1/NUM_INODES_PER_BLOCK is, guess what, 2,
Therefore the argument to malloc (sizeof(char) * SECTOR_SIZE_1/NUM_INODES_PER_BLOCK) is also 2, so you allocate a buffer of exactly 2 bytes.
Then in the for loop you fill this buffer to a size of MAX_NUM_SECTORS_PER_FILE + 2 = 32, or in other words you write 30 bytes beyond the end of the buffer.
A typical case of buffer overflow and subsequent undefined behaviour (an error message in your case).
Just finished putting this function together from some man documentation, it takes a char* and appends a const char* to it, if the size of the char* is too small it reallocates it to something a little bigger and finally appends it. Its been a long time since I used c, so just checking in.
// append with realloc
int append(char *orig_str, const char *append_str) {
int result = 0; // fail by default
// is there enough space to append our data?
int req_space = strlen(orig_str) + strlen(append_str);
if (req_space > strlen(orig_str)) {
// just reallocate enough + 4096
int new_size = req_space;
char *new_str = realloc(orig_str, req_space * sizeof(char));
// resize success..
if(new_str != NULL) {
orig_str = new_str;
result = 1; // success
} else {
// the resize failed..
fprintf(stderr, "Couldn't reallocate memory\n");
}
} else {
result = 1;
}
// finally, append the data
if (result) {
strncat(orig_str, append_str, strlen(append_str));
}
// return 0 if Ok
return result;
}
This is not usable because you never tell the caller where the memory is that you got back from realloc.
You will need to either return a pointer, or pass orig_str by reference.
Also (as pointed out in comments) you need to do realloc(orig_str, req_space + 1); to allow space for the null terminator.
Your code has a some inefficient logic , compare with this fixed version:
bool append(char **p_orig_str, const char *append_str)
{
// no action required if appending an empty string
if ( append_str[0] == 0 )
return true;
size_t orig_len = strlen(*p_orig_str);
size_t req_space = orig_len + strlen(append_str) + 1;
char *new_str = realloc(*p_orig_str, req_space);
// resize success..
if(new_str == NULL)
{
fprintf(stderr, "Couldn't reallocate memory\n");
return false;
}
*p_orig_str = new_str;
strcpy(new_str + orig_len, append_str);
return true;
}
This logic doesn't make any sense:
// is there enough space to append our data?
int req_space = strlen(orig_str) + strlen(append_str);
if (req_space > strlen(orig_str)) {
As long as append_str has non-zero length, you're always going to have to re-allocate.
The main problem is that you're trying to track the size of your buffers with strlen. If your string is NUL-terminated (as it should be), your perceived buffer size is always going to be the exact length of the data in it, ignoring any extra.
If you want to work with buffers like this, you need to track the size in a separate size_t, or keep some sort of descriptor like this:
struct buffer {
void *buf;
size_t alloc_size;
size_t used_amt; /* Omit if strings are NUL-terminated */
}
I get an
malloc: *** error for object 0x1001012f8: incorrect checksum for freed object
- object was probably modified after being freed.
*** set a breakpoint in malloc_error_break to debug
error in the following function:
char* substr(const char* source, const char* start, const char* end) {
char *path_start, *path_end, *path;
int path_len, needle_len = strlen(start);
path_start = strcasestr(source, start);
if (path_start != NULL) {
path_start += needle_len;
path_end = strcasestr(path_start, end);
path_len = path_end - path_start;
path = malloc(path_len + 1);
strncpy(path, path_start, path_len);
path[path_len] = '\0';
} else {
path = NULL;
}
return path;
}
How can I make this work? When I rewrite the function to allocate the memory using path[path_len + 1] it works just fine.
Now, the part I don't understand is, that I never even call free in any point of my application, as every allocated memory is needed for the program until it exists (which, AFAIK will invalidate every allocated memory anyway?!)
So, how can a freed object be corrupt if I never free one?
The function is called in this one:
char *read_response(int sock) {
int bytes_read;
char *buf = (char*)malloc(BUF_SIZE);
char *cur_position = buf;
while ((bytes_read = read(sock, cur_position, BUF_SIZE)) > 0) {
cur_position += bytes_read;
buf = realloc(buf, sizeof(buf) + BUF_SIZE);
}
int status = atoi(substr(buf, "HTTP/1.0 ", " "));
There is the realloc, am I using that wrong? I want to read the complete server response, so I have to reallocate after every iteration, don't I?
In read_response, you are probably overwriting the end of the buffer pointed to by buf.
The problem is that buf is a pointer, so sizeof(buf) will return the size of a pointer (probably 4 or 8 depending on your CPU). You are using sizeof as if buf were an array, which is not really the same thing as a pointer in C although they seem interchangeable in some contexts.
Instead of using sizeof, you need to be keeping track of the last size that you allocated for buf, and add BUF_SIZE to that each time you enlarge the buffer.
You should also consider that the read operation may be returning considerably fewer characters than BUF_SIZE on each call, so doing a realloc on buf in each iteration may be overkill. That probably won't cause any problems for you in terms of correctness, though; it will just use more memory than it needs to.
I would do something more like the code below.
#define MIN_BUF_SPACE_THRESHOLD (BUF_SIZE / 2)
char *read_response(int sock) {
int bytes_read;
char *buf = (char*)malloc(BUF_SIZE);
int cur_position = 0;
int space_left = BUF_SIZE;
if (buf == NULL) {
exit(1); /* or try to cope with out-of-memory situation */
}
while ((bytes_read = read(sock, buf + cur_position, space_left)) > 0) {
cur_position += bytes_read;
space_left -= bytes_read;
if (space_left < MIN_BUF_SPACE_THRESHOLD) {
buf = realloc(buf, cur_position + space_left + BUF_SIZE);
if (buf == NULL) {
exit(1); /* or try to cope with out-of-memory situation */
}
space_left += BUF_SIZE;
}
}
This version has the advantage of not trying to allocate more space if the read call comes back with only a few bytes of data.
This line
buf = realloc(buf, sizeof(buf) + BUF_SIZE);
is wrong. All reallocations are with the same size, BUF_SIZE + sizeof(char*). Then you are writing to unallocated memory when reading from the socket, overwriting memory previously freed by a realloc.
You have to keep track of the allocated size,
size_t current_buf_size = BUF_SIZE;
/* ... */
char *temp = realloc(buf, current_buf_size + BUF_SIZE);
if (temp == NULL) {
/* die or repair */
}
buf = temp;
I have a problem with an application I'm currently developing. In this program I have to read huge amounts (billions) of data from text files and manage them consequently, but since it's a two students project, the reading part will be developed by my mate. For testing reason I wrote a small procedures that generates pseudo-random structures to replace what my mate will do.
The problem is the following: a big amount of the generated data (due to redundancy) can be discarded in order to free its memory. But even invoking the free() function the memory usage keeps growing. So I tried to develop a debug application that simply generates a chunk of data and immediately frees it. And repeats that for thousands of times. Well, I can't grasp the reason, but the memory allocated to the process grows to ~1.8 GB ram and then crashes. Why? The strangest thing, that makes me thing there's a lot I'm not understanding well, is that when the process crashes the malloc does NOT return a NULL pointer, because the process always crashes when readCycles == 6008 and bypasses the NULL check.
I already read other related topics here on StackOverflow and I understood why free() doesn't reduce the memory allocated to my process. That's fine. But why the memory usage keeps growing? Shouldn't the malloc allocate previously freed memory instead of constantly requesting new one?
This is the most relevant part of my code:
#define NREAD 1000
#define READCYCLES 10000
#define N_ALPHA_ILLUMINA 7
#define N_ALPHA_SOLID 5
#define SEQLEN 76
typedef struct{
char* leftDNA;
char* leftQuality;
unsigned long int leftRow;
char* rightDNA;
char* rightQuality;
unsigned long int rightRow;
} MatePair;
unsigned long int readCycles = 0;
MatePair* readStream(MatePair* inputStream, short* eof, unsigned long int* inputSize){
double r;
unsigned long int i, j;
unsigned long int leftRow;
int alphabet[] = {'A', 'C', 'G', 'T', 'N'};
inputStream = (MatePair*) malloc (sizeof(MatePair) * (NREAD + 1));
printf("%d\n", readCycles);
if (inputStream == NULL){
(*eof) = 1;
return;
}
for (i = 0; i < NREAD; i++){
leftRow = readCycles * NREAD + i;
inputStream[i].leftDNA = (char*) malloc (SEQLEN);
inputStream[i].rightDNA = (char*) malloc (SEQLEN);
inputStream[i].leftQuality = (char*) malloc (SEQLEN);
inputStream[i].rightQuality = (char*) malloc (SEQLEN);
for (j = 0; j < SEQLEN; j++){
r = rand() / (RAND_MAX + 1);
inputStream[i].leftDNA[j] = alphabet[(int)(r * 5)];
inputStream[i].rightDNA[j] = alphabet[(int)(r * 5)];
inputStream[i].leftQuality[j] = (char) 64 + (int)(r * 60);
inputStream[i].rightQuality[j] = (char) 64 + (int)(r * 60);
}
inputStream[i].leftDNA[SEQLEN - 1] = '\0';
inputStream[i].rightDNA[SEQLEN - 1] = '\0';
inputStream[i].leftQuality[SEQLEN - 1] = '\0';
inputStream[i].rightQuality[SEQLEN - 1] = '\0';
inputStream[i].leftRow = leftRow;
inputStream[i].rightRow = leftRow;
}
inputStream[i].leftRow = -1;
readCycles++;
(*inputSize) = NREAD;
(*eof) = readCycles > READCYCLES;
return inputStream;
}
int main(int argc, char* argv[]){
short eof = 0;
unsigned long int inputSize = 0;
MatePair* inputStream = NULL;
while (!eof){
inputStream = readStream(inputStream, &eof, &inputSize);
free(inputStream);
inputStream = NULL;
}
return 0;
}
I forgot to mention that, but before posting here, instead of calling free(inputStream), I tried invoking freeMemory(inputStream). Not sure if it's the correct way of doing it, though.
void freeMemory(MatePair* memblock){
for ( ; memblock->leftRow != 1; memblock++){
free(memblock -> leftDNA);
free(memblock -> leftQuality);
free(memblock -> rightDNA);
free(memblock -> rightQuality);
}
}
Memory leaks. How many 'malloc()' you have called, how many 'free()' you must use to free all allocated memory on the heap.
Thus,
inputStream[i].leftDNA = (char*) malloc (SEQLEN);
inputStream[i].rightDNA = (char*) malloc (SEQLEN);
inputStream[i].leftQuality = (char*) malloc (SEQLEN);
inputStream[i].rightQuality = (char*) malloc (SEQLEN);
these 'malloc()' functions must be paired with free().
You're not freeing all members allocated within the read loop, hence you're losing memory eahc time. Remember, you have to free everything you allocate with a malloc, not just your array.
Ok, Just look at your edit, and your freeMemory is still wrong. Try this;
void freeMemory(MatePair* inputStream)
{
for (i = 0; i < NREAD; i++){
free(inputStream[i].leftDNA);
free(inputStream[i].leftQuality);
free(inputStream[i].rightDNA);
free(inputStream[i].rightQuality);
}
free (inputStream);
}
Your free(memblock) was in the loop, which it shouldn't have been, and I'd tend to use the same iteration sequence on freeing as mallocing. You also need to error check after each malloc, and decide what to do with a NULL at that point.
I'm looking for an efficient method for appending multiple strings.
The way it should work is C++ std::string::append or JAVA StringBuffer.append.
I wrote a function which actually reallocs previous source pointer and does strcat.
I believe this is not an efficient method as compiler may implement this free and malloc.
Other way I could think of (like std::vector) is allocate memory in bulk (1KB for eg) and do strcpy. In that case every append call will check if the total required allocation is more than (1200 bytes) the amount allocated in bulk, realloc to 2KB. But in that case there will be some memory wasted.
I'm looking for a balance between the above but the preference is performance.
What other approaches are possible. Please suggest.
I would add each string to a list, and add the length of each new string to a running total. Then, when you're done, allocate space for that total, walk the list and strcpy each string to the newly allocated space.
The classical approach is to double the buffer every time it is too small.
Start out with a "reasonable" buffer, so you don't need to do realloc()s for sizes 1, 2, 4, 8, 16 which are going to be hit by a large number of your strings.
Starting out at 1024 bytes means you will have one realloc() if you hit 2048, a second if you hit 4096, and so on. If rampant memory consumption scares you, cap the growth rate once it hits something suitably big, like 65536 bytes or whatever, it depends on your data and memory tolerance.
Also make sure you buffer the current length, so you can do strcpy() without having to walk the string to find the length, first.
Sample function to concatenate strings
void
addToBuffer(char **content, char *buf) {
int textlen, oldtextlen;
textlen = strlen(buf);
if (*content == NULL)
oldtextlen = 0;
else
oldtextlen = strlen(*content);
*content = (char *) realloc( (void *) *content, (sizeof(char)) * (oldtextlen+textlen+1));
if ( oldtextlen != 0 ) {
strncpy(*content + oldtextlen, buf, textlen + 1);
} else {
strncpy(*content, buf, textlen + 1);
}
}
int main(void) {
char *content = NULL;
addToBuffer(&content, "test");
addToBuffer(&content, "test1");
}
I would do something like this:
typedef struct Stringbuffer {
int capacity; /* Maximum capacity. */
int length; /* Current length (excluding null terminator). */
char* characters; /* Pointer to characters. */
} Stringbuffer;
BOOL StringBuffer_init(Stringbuffer* buffer) {
buffer->capacity = 0;
buffer->length = 0;
buffer->characters = NULL;
}
void StringBuffer_del(Stringbuffer* buffer) {
if (!buffer)
return;
free(buffer->characters);
buffer->capacity = 0;
buffer->length = 0;
buffer->characters = NULL;
}
BOOL StringBuffer_add(Stringbuffer* buffer, char* string) {
int len;
int new_length;
if (!buffer)
return FALSE;
len = string ? strlen(string) : 0;
if (len == 0)
return TRUE;
new_length = buffer->length + len;
if (new_length >= new_capacity) {
int new_capacity;
new_capacity = buffer->capacity;
if (new_capacity == 0)
new_capacity = 16;
while (new_length >= new_capacity)
new_capacity *= 2;
new_characters = (char*)realloc(buffer->characters, new_capacity);
if (!new_characters)
return FALSE;
buffer->capacity = new_capacity;
buffer->characters = new_characters;
}
memmove(buffer->characters + buffer->length, string, len);
buffer->length = new_length;
buffer->characters[buffer->length] = '\0';
return TRUE;
}