In implementation of list (Python 3.4) I saw the following:
static int
list_resize(PyListObject *self, Py_ssize_t newsize)
{
PyObject **items;
size_t new_allocated;
Py_ssize_t allocated = self->allocated;
/* Bypass realloc() when a previous overallocation is large enough
to accommodate the newsize. If the newsize falls lower than half
the allocated size, then proceed with the realloc() to shrink the list.
*/
if (allocated >= newsize && newsize >= (allocated >> 1)) {
assert(self->ob_item != NULL || newsize == 0);
Py_SIZE(self) = newsize;
return 0;
}
new_allocated = (newsize >> 3) + (newsize < 9 ? 3 : 6);
/* check for integer overflow */
if (new_allocated > PY_SIZE_MAX - newsize) {
PyErr_NoMemory();
return -1;
} else {
new_allocated += newsize;
}
if (newsize == 0)
new_allocated = 0;
items = self->ob_item;
if (new_allocated <= (PY_SIZE_MAX / sizeof(PyObject *)))
PyMem_RESIZE(items, PyObject *, new_allocated);
else
items = NULL;
if (items == NULL) {
PyErr_NoMemory();
return -1;
}
self->ob_item = items;
Py_SIZE(self) = newsize;
self->allocated = new_allocated;
return 0;
}
and concretely line:
PyMem_RESIZE(items, PyObject *, new_allocated);
How I know, realloc may return another pointer, different from the obtained. I don`t understand how it works stable.
This is the definition of PyMem_Resize:
#define PyMem_Resize(p, type, n) \
( (p) = (type *) PyMem_Realloc((p), (n) * sizeof(type)) )
So it modifies the pointer items in-place.
See also https://docs.python.org/3/c-api/memory.html#c.PyMem_Resize
Related
I've just encountered a very strange bug. I was doing unit-test for a simple function as below.
UPDATE: Thanks #Bodo, here's the minimal working example. You can simply compile and run tokenizer.c.
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <assert.h>
/* ============================= BOOL =============================== */
#ifndef _BOOL_
#define _BOOL_
typedef enum {
true, false
} bool;
#endif // _BOOL_
/* ============================= STACK =============================== */
#ifndef _STACK_
#define _STACK_
typedef void (*stack_freefn)(void *elemAddr);
typedef struct {
size_t size; // number of element allowed
int ite; // point to the current last element
size_t elemSize; // size of each element (how many bytes)
void *elems; // stockage of elements
stack_freefn freefn; // free memory allocated for each element if necessary
} stack;
/* constructor */
void new_stack(stack *s, const size_t size, const size_t elemSize, stack_freefn freefn) {
s->size = size;
s->ite = 0;
s->elemSize = elemSize;
s->elems = malloc(size * elemSize);
s->freefn = freefn;
}
/* free memory */
void dispose_stack(stack *s) {
if (s->freefn != NULL) {
while (s->ite > 0) {
void *elemAddr = (char *)s->elems + --s->ite * s->elemSize;
s->freefn(elemAddr);
}
}
free(s->elems);
s->elems = NULL;
}
/* push one new element on the top */
void push_stack(stack *s, const void *value, const size_t elemSize) {
if (s->ite == s->size) {
s->size *= 2;
s->elems = realloc(s->elems, s->size * s->elemSize);
}
void *elemAddr = (char *)s->elems + s->elemSize * s->ite++;
memcpy(elemAddr, value, s->elemSize);
}
/* pop our the element on the top */
void pop_stack(stack *s, void *res) {
if (s->ite > 0) {
void *elemAddr = (char *)s->elems + ((s->ite - 1) * s->elemSize);
memcpy(res, elemAddr, s->elemSize);
s->ite--;
}
}
void clear_stack(stack *s) {
if (s->freefn != NULL) {
while (s->ite > 0) {
void *elemAddr = (char *)s->elems + --s->ite * s->elemSize;
s->freefn(elemAddr);
}
} else {
s->ite = 0;
}
}
size_t stack_size(stack *s) {
return s->ite;
}
#endif // _STACK_
/* ============================= VECTOR =============================== */
#ifndef _VECTOR_
#define _VECTOR_
typedef int (*VectorCompareFunction)(const void *elemAddr1, const void *elemAddr2);
typedef void (*VectorFreeFunction)(void *elemAddr);
typedef struct {
int elemSize; //how many byte for each element
int elemNum; //number of current element in vector
int capacity; //maximum number of element vector can hold
void *elems; //pointer to data memory
VectorFreeFunction freefn; //pointer to the function used to free each element
} vector;
/**
* Reallocate a new memory of twice of original size
* return 1 if reallocation success, otherwise return -1.
*/
static void DoubleMemory(vector *v) {
void *tmp = realloc(v->elems, v->capacity * v->elemSize * 2);
assert(tmp != NULL);
v->elems = tmp;
v->capacity *= 2;
}
/**
* Constructor
*/
void VectorNew(vector *v, int elemSize, VectorFreeFunction freefn, int initialAllocation) {
v->elems = malloc(initialAllocation * elemSize);
assert(v->elems != NULL);
v->elemSize = elemSize;
v->elemNum = 0;
v->capacity = initialAllocation;
v->freefn = freefn;
}
/**
* Frees up all the memory of the specified vector.
*/
void VectorDispose(vector *v) {
if (v->freefn != NULL) {
for (; v->elemNum > 0; v->elemNum--) {
void *elemAddr = (char *)v->elems + (v->elemNum - 1) * v->elemSize;
v->freefn(elemAddr);
}
}
free(v->elems);
v->elems = NULL;
}
/**
* Returns the logical length of the vector.
*/
int VectorLength(const vector *v) {
return v->elemNum;
}
/**
* Appends a new element to the end of the specified vector.
*/
void VectorAppend(vector *v, const void *elemAddr) {
/* double size if neccessary */
if (v->elemNum == v->capacity) DoubleMemory(v);
memcpy((char *)v->elems + v->elemNum * v->elemSize, elemAddr, v->elemSize);
v->elemNum++;
}
/**
* Search the specified vector for an element whose contents match the element passed as the key.
*/
int VectorSearch(const vector *v, const void *key, VectorCompareFunction searchfn, int startIndex, bool isSorted) {
assert(key && searchfn);
if (v->elemNum == 0) return -1;
assert(startIndex >= 0 && startIndex < v->elemNum);
if (isSorted == true) {
/* binary search */
void *startAddr = (char *)v->elems + startIndex * v->elemSize;
int size = v->elemNum - startIndex;
void *resAddr = bsearch(key, startAddr, size, v->elemSize, searchfn);
return (resAddr != NULL)? ((char *)resAddr - (char *)v->elems) / v->elemSize : -1;
} else {
/* linear search */
for (int i = 0; i < v->elemNum; i++) {
if (searchfn((char *)v->elems + i * v->elemSize, key) == 0) {
return i;
}
}
return -1;
}
}
#endif // _VECTOR_
/* ============================= TOKENIZER =============================== */
/**
* Dump current string into vector as a new word.
* Strings are null-terminated.
*/
static void dumpstack(stack *s, vector *v) {
size_t len = stack_size(s);
char *word = (char *)malloc((len + 1) * sizeof(char)); // +1 for null-terminator
for (int i = len - 1; i >= 0; i--) {
pop_stack(s, word + i * sizeof(char));
}
word[len] = '\0';
VectorAppend(v, &word);
clear_stack(s);
}
static const size_t kTokenStackDefaultSize = 64;
static void tokenize(vector *words, char *stream) {
stack s;
new_stack(&s, kTokenStackDefaultSize, sizeof(char), NULL);
size_t len = strlen(stream);
bool begin = false;
char c;
for (int i = 0; i < len; i++) {
c = stream[i];
/* =============================== My printf() is here ============================== */
// printf("char c = [%c]\n", c);
/* =============================== My printf() is here ============================== */
if (isalpha(c) || isdigit(c)) {
if (begin == false) begin = true;
char lower = tolower(c);
push_stack(&s, &lower, sizeof(char));
} else if (c == '-') {
if (begin == true) { // case: covid-19
push_stack(&s, &c, sizeof(char));
} else {
if (i < len - 1 && isdigit(stream[i + 1])) { // case: -9
begin = true;
push_stack(&s, &c, sizeof(char));
} else {
if (begin == true) {
dumpstack(&s, words);
begin = false;
}
}
}
} else if (c == '.' && begin == true) { // case: 3.14
if (isdigit(stream[i - 1])) {
push_stack(&s, &c, sizeof(char));
} else {
if (begin == true) {
dumpstack(&s, words);
begin = false;
}
}
} else {
if (begin == true) {
dumpstack(&s, words);
begin = false;
}
}
}
if (begin == true) {
dumpstack(&s, words);
begin = false;
}
dispose_stack(&s);
}
/* ============================= UNIT-TEST =============================== */
/**
* HashSetFreeFunction<char *>
*/
static void freestr(void *elemAddr) {
char *str = *(char **)elemAddr;
free(str);
}
/**
* HashSetCompareFunction<char *>
*/
static int compstr(const void *elemAddr1, const void *elemAddr2) {
char *str1 = *(char **)elemAddr1;
char *str2 = *(char **)elemAddr2;
return strcmp(str1, str2);
}
static void test_tokenize(void) {
printf("Testing Tokenizer.c::tokenize() ...\n");
char *sentence = "Covid-19: Top adviser warns France at 'emergency' virus moment - BBC News\nPi = 3.14\n-1 is negative.";
vector words;
VectorNew(&words, sizeof(char *), freestr, 256);
tokenize(&words, sentence);
char *musthave[] = {"covid-19", "top", "3.14", "-1"};
char *musthavenot[] = {"-", "1"};
assert(VectorLength(&words) == 16);
for (int i = 0; i < sizeof(musthave)/sizeof(char *); i++) {
assert(VectorSearch(&words, &musthave[i], compstr, 0, false) != -1);
}
for (int i = 0; i < sizeof(musthavenot)/sizeof(char *); i++) {
assert(VectorSearch(&words, &musthavenot[i], compstr, 0, false) == -1);
}
VectorDispose(&words);
printf("[ALL PASS]\n");
}
int main(void) {
test_tokenize();
}
I've got segmentation fault at first.
[1] 4685 segmentation fault testtokenizer
But when I add a printf() to debug, the segmentation fault was gone and the test passed. After comment out the printf, the function still works. I was so confused.
Just recall that before this test, I tested some memory dispose function, and perhaps had left some unfreed blocks in memory. Will that be the reason for fleeting segmentation fault? Thx bros.
UPDATE:
Now I can't even reproduce this bug myself. tokenizer.c above can pass the unit-test. I thought it might caused by makefile prerequisite rules. gcc didn't re-compile some object files when source code is changed.
Thanks #Steve Summit, you make it clear that unfreed memory will not cause segmentation fault.
Thanks #schwern for code review, it's really helpful.
But when I add a printf() to debug, the segmentation fault was gone and the test passed. After comment out the printf, the function still works. I was so confused.
They call it undefined behavior, because its behavior is undefined. Seemingly unrelated operations might nudge things just a bit to make the code "work" but they're only tangentially related to the problem.
I tested some memory dispose function, and perhaps had left some unfreed blocks in memory. Will that be the reason for fleeting segmentation fault?
No. It does mean the memory is unreferencable and "leaked". The memory will be freed to the operating system when the process exits.
The problem must lie elsewhere. Without seeing your whole program we can't say for sure, but two fishy things stand out.
You're defining a fixed sized stack, but you're pushing onto it an indeterminate number of times. Unless push_stack has protection against this, you will walk off your allocated memory.
You're storing references to variables on the stack. lower, c
char lower = tolower(c);
push_stack(&s, &lower, sizeof(char));
Once lower goes out of scope it will automatically be freed and the memory reused. &lower is invalid once tokenize returns. This seems to be fine if your stack only lasts the length of the function, but it's worth noting.
And it's possible new_stack, push_stack, or dumpstack are doing something incorrect.
void put(char* key, int value)
{
int i = 0;
// Iterate through elements of hashtable
while (array[i].flag == 1)
{
// If key already exists, update the value
// and return
if (strcmp(array[i].data->key, key) == 0)
{
array[i].data->value = value;
return;
}
i = i + 1;
// Error Handling, when end of hashtable is reached
if (i == max)
{
i = 0;
// p rintf("\n Hash table is full, cannot insert any more item \n");
// return;
}
}
// Insert new item into the hashtable
array[i].flag = 1;
array[i].data = (struct item*) malloc(sizeof(struct item));
array[i].data->key = (char *)malloc((strlen(key)+1)*sizeof(char));
// so here I have to use snprintf instead of strcy because it's
// forbidden, and I don't know how
**strcpy(array[i].data->key, key);**
//snprintf(array[i].data->key,sizeof(key),"%s",key);
array[i].data->value = value;
}
Firstly, never cast malloc, and do not forget verify the return value of malloc.
array[i].data = malloc(sizeof(struct item));
if (!array[i].data) {
// handle error
return; // for example
}
array[i].data->key = malloc((strlen(key)+1)*sizeof(char));
if (!array[i].data->key) {
// handle error
}
Using strlen instead of sizeof that returns the size of pointer, not the length of string:
snprintf(array[i].data->key, strlen(key) + 1,"%s", key);
here I have to use snprintf instead of strcy because it's forbidden, and I don't know how
strcpy() can overflow is the destination is insufficient. That really does not apply here as the needed space is allocated.
// array[i].data = (struct item*) malloc(sizeof(struct item));
// array[i].data->key = (char *)malloc((strlen(key)+1)*sizeof(char));
// strcpy(array[i].data->key, key);
array[i].data = malloc(sizeof *(array[i].data));
size_t sz = strlen(key) + 1;
array[i].data->key = malloc(sz);
snprintf(array[i].data->key, sz, "%s", key);
// or
strcpy(array[i].data->key, key);
// or
memcpy(array[i].data->key, key, sz);
Cast not needed
size of of referenced data better to use that sizeof type.
Avoid 2 trips finding the length.
Returns check omitted for brevity.
With checks
array[i].data = malloc(sizeof *(array[i].data));
if (array[i].data == NULL) return fail; // or do something to indicate error
size_t sz = strlen(key) + 1;
array[i].data->key = malloc(sz);
if (array[i].data->key == NULL) return fail;
int count = snprintf(array[i].data->key, sz, "%s", key);
if (count < 0 || (unsigned) count >= sz) return fail;
Primary mistake
OP's attempt fails as sizeof(key) is the size of a pointer, perhaps 2, 4 or 8. Instead, what is needed is the size of the allocation for array[i].data->key as in sz (see above) or strlen(key) + 1.
snprintf(array[i].data->key,sizeof(key),"%s",key); // bad
I have an error using realloc to replace malloc.
This code below runs OK on my computer.
int vector_grow(Vector* vec) {
unsigned long newcap;
int * newarr;
if (0 == vec->cap) {
vec->arr = (int*)malloc(START_CAPACITY * sizeof(*vec->arr));
if (NULL == vec->arr)
return -1;
vec->cap = START_CAPACITY;
return 0;
}
newarr = malloc (newcap * sizeof(*vec->arr));
if (NULL == newarr)
return -1;
memcpy (newarr, vec->arr, vec->len * sizeof(*vec->arr));
free (vec->arr);
vec->arr = newarr;
vec->cap = newcap;
return 0;
}
I want to change the malloc to realloc, but the error occurs.
int vector_grow(Vector* vec) {
unsigned long newcap;
if (0 == vec->cap) {
vec->arr = (int*)malloc(START_CAPACITY * sizeof(*vec->arr));
if (NULL == vec->arr)
return -1;
vec->cap = START_CAPACITY;
return 0;
}
newcap = 2 * vec->cap;
if ((vec->arr = (int*)realloc(vec->arr, newcap * sizeof(int))) == NULL)
return -1;
return 0;
}
It says
malloc: *** error for object 0x7fca64c02598: incorrect checksum for freed object - object was probably modified after being freed.
I don't know any difference between those two snippets of code, if you know what causes the error, please tell me! Thank you very much!
Bug in missing vec->cap = in updated code certainly contribute to various calls to malloc() and calling code's misuse of data.
int vector_grow(Vector* vec) {
unsigned long newcap;
if (0 == vec->cap) {
... // not important to show the bug
}
newcap = 2 * vec->cap;
if ((vec->arr = (int*)realloc(vec->arr, newcap * sizeof(int))) == NULL)
return -1;
// Add missing update
vec->cap = newcap;
return 0;
}
Also better to test for allocation success
void *p = realloc(vec->arr, sizeof *(vec->arr) * newcap);
if (p == NULL) {
return -1;
}
vec->arr = p;
vec->cap = newcap;
The only scenario where I can imagine such error message is when you actually modify the pointer, for example
int *x = malloc(2 * sizeof *x);
if (x != NULL) {
x = x + 1;
free(x);
}
The pointer that MUST be passed to free() MUST had been returned by malloc()/calloc()/realloc(), passing any other pointer including a pointer to the same data but at a different position like x in the example above is undefined behavior.
Context: I have written a best-fit memory allocator. It allocates large blocks of memory, and serves best fitting chunks of it to programs upon request. If memory reserves are exhausted, its asks for OS for more. All free blocks are stored on a linked-list, ordered by increasing pointer value.
Problem:: When memory is released, the program is supposed to link it back into the free list of blocks for recycling, and more crucially merges the given block with its surrounding blocks if possible. Unfortunately, this only works well as long as I do not need to ask the OS for more then one super-block to serve. When this does occur, the new blocks I receive have nonsensical addressing and get inserted between other super-block addressing space. This leads to permanent fragmentation.
Tl;dr: I am being given new super-blocks of memory whose address is within the address space of another super-block, leading to fragmentation when sub-blocks are returned.
Illustration of problem:
Here is a diagram of the problem I described above.
Numbers: Memory addresses (from real execution).
Beige blocks: Free memory.
White blocks: Memory unlinked for use.
The diagram shows the progression of memory usage until the fragmentation catalyst occurs. You can see once the block gets inserted, merging will never be possible up for the busy blocks once they are checked back in.
Code: Reproducible Example:
The following includes the allocator and a small test program. It must be compiled with at least C99.
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
/*****************************************************************************/
/* SYMBOLIC CONSTANTS */
/*****************************************************************************/
#define NEXT(hp) ((hp)->key.next)
#define UNITS(hp) ((hp)->key.units)
#define UNIT_SIZE sizeof(Header)
#define MIN_UNITS_ALLOC 64
#define MAX(a,b) ((a) > (b) ? (a) : (b))
/*****************************************************************************/
/* TYPE DEFINITIONS */
/*****************************************************************************/
typedef union header {
intmax_t align;
struct {
union header *next;
unsigned units;
} key;
} Header;
/*****************************************************************************/
/* GLOBAL VARIABLES */
/*****************************************************************************/
Header base = {.key = { NULL, 0 }};
Header *list;
/*****************************************************************************/
/* PROTOTYPES */
/*****************************************************************************/
/* Allocates a 'bytes' size block of memory. On success, returns pointer to
* the block. On error, NULL is returned. */
void *alloc (size_t bytes);
/* Returns a 'bytes' size block of allocated memory for reuse */
void release (void *bytes);
/* Attempts to reserve memory from the operating system via a system call */
static Header *reserve (unsigned units);
/*****************************************************************************/
/* FUNCTION IMPLEMENTATIONS */
/*****************************************************************************/
void *alloc (size_t bytes) {
size_t units, diff;
Header *block, *lastBlock, *best, *lastBest;
if (bytes == 0) {
return NULL;
}
if (list == NULL) {
list = base.key.next = &base;
}
best = lastBest = NULL;
units = (bytes + UNIT_SIZE - 1) / UNIT_SIZE + 1;
diff = SIZE_MAX;
for (lastBlock = list, block = NEXT(list); ; lastBlock = block, block = NEXT(block)) {
/* Loop across list, find closest fitting block */
if (UNITS(block) >= units && UNITS(block) - units < diff) {
diff = UNITS(block) - units;
best = block;
lastBest = lastBlock;
}
/* Upon cycle completion */
if (block == list) {
/* If no block available, reserve some. */
if (best == NULL) {
if ((lastBest = reserve(units)) == NULL) {
return NULL;
} else {
fprintf(stderr, "\nalloc: Out of memory, linking new block %lld of size %u.\n\n", (long long)lastBest, UNITS(lastBest));
release((void *)(lastBest + 1));
}
/* If block of perfect size, return. Else slice and return */
} else {
if (diff == 0) {
NEXT(lastBest) = NEXT(best);
} else {
UNITS(best) = diff;
best += diff;
UNITS(best) = units;
}
fprintf(stderr, "alloc: Unlinked block %lld of %u units.\n", (long long)best, UNITS(best));
return (void *)(best + 1);
}
}
}
}
void release (void *bytes) {
Header *p, *block;
block = (Header *)bytes - 1;
/* Choose p such that: p -> block -> NEXT(p) */
for (p = list; !(p < block && NEXT(p) > block); p = NEXT(p)) {
if (p >= NEXT(p) && (block > p || block < NEXT(p))) {
break;
}
}
/* Merge block with NEXT(p) if adjacent */
if (block + UNITS(block) == NEXT(p)) {
NEXT(block) = NEXT(NEXT(p));
UNITS(block) += UNITS(NEXT(p));
} else {
NEXT(block) = NEXT(p);
}
/* Merge block with p if adjacent */
if (p + UNITS(p) == block) {
NEXT(p) = NEXT(block);
UNITS(p) += UNITS(block);
} else {
NEXT(p) = block;
}
}
static Header *reserve (unsigned units) {
char *bytes, *sbrk(int);
Header *block;
units = MAX(units, MIN_UNITS_ALLOC);
if ((bytes = sbrk(units)) == (char *)-1) {
return NULL;
} else {
block = (Header *)bytes;
UNITS(block) = units;
}
return block;
}
/*****************************************************************************/
/* TESTING FUNCTIONS (DELETE) */
/*****************************************************************************/
void printFreeList (unsigned byAddress) {
Header *lp = list;
if (lp == NULL) {
fprintf(stdout, "List is NULL\n");
return;
}
do {
fprintf(stdout, "[ %lld ] -> ", (byAddress ? (long long)lp : (long long)UNITS(lp)));
lp = NEXT(lp);
} while (lp != list);
putc('\n', stdout);
}
void releaseItemAtIndex (int i, int k, long long *p[]) {
fprintf(stderr, "Releasing block %d/%d\n", i, k);
release(p[i]);
for (int j = i; j < k; j++) {
if (j + 1 < k) {
p[j] = p[j + 1];
}
}
}
#define MAX_TEST_SIZE 5000
int main (int argc, const char *argv[]) {
/* Seed PRNG: For removed random deletion (now manual) */
srand(time(NULL));
/* Array of pointers to store allocated blocks, blockSize we want to allocate. */
long long *p[MAX_TEST_SIZE], blockSize = 256;
/* Number of blocks we choose to allocate */
int k = 6;
/* Allocate said blocks */
for (int i = 0; i < k; i++) {
p[i] = alloc(blockSize * sizeof(char));
printFreeList(0); printFreeList(1); putchar('\n');
}
fprintf(stderr, "\n\n");
int idx;
while (k > 0) {
fprintf(stderr, "Delete an index between 0 up to and including %d:\n", k - 1);
scanf("%d", &idx);
releaseItemAtIndex(idx, k, p);
printFreeList(0); printFreeList(1); putchar('\n');
k--;
}
return 0;
}
Miscellaneous Details:
I am running a 64 bit operating system.
I do not know if pointer comparison on the heap is guaranteed to be valid. This is not guaranteed by the standard according to K&R.
Well I eventually wrote one a couple years later that seems to work alright. Bonus: It's using static memory.
Header File
#if !defined(STATIC_ALLOCATOR_H)
#define STATIC_ALLOCATOR_H
/*
*******************************************************************************
* (C) Copyright 2020 *
* Created: 07/04/2020 *
* *
* Programmer(s): *
* - Jillian Oduber *
* - Charles Randolph *
* *
* Description: *
* Static first-fit memory allocator *
* *
*******************************************************************************
*/
#include <iostream>
#include <cstddef>
#include <cstdint>
#include "dx_types.h"
extern "C" {
#include "stddef.h"
}
/*
*******************************************************************************
* Type Definitions *
*******************************************************************************
*/
// Defines the header block used in the custom allocator
typedef union block_h {
struct {
union block_h *next;
size_t size; // Size is in units of sizeof(block_h)
} d;
max_align_t align;
} block_h;
/*
*******************************************************************************
* Class Definitions *
*******************************************************************************
*/
class Static_Allocator {
public:
/*\
* #brief Configures the class with the given static memory capacity
* #param memory Pointer to static array (must support address comparison)
* #param size Total number of bytes alloted for distribution
\*/
Static_Allocator(uint8_t *memory, size_t size);
/*\
* #brief Returns a memory block of the desired size
* #param size Number of bytes to allocate
* #return
* - valid pointer if memory is available
* - NULL otherwise
\*/
uint8_t *alloc (size_t size);
/*\
* #brief Releases the allocated block of memory
* #param ptr Pointer to allocated block of memory
* #return
* - STATUS_OK on success
* - STATUS_ERR if invalid block
* - STATUS_BAD_PARAM on NULL parameter
\*/
dx::status_t free (uint8_t *ptr);
/*\
* #brief Returns the amount of free memory
* #return size_t Bytes (of available free memory)
\*/
size_t free_memory_size ();
/*\
* #brief Debug utility which shows what is on the free-list
* #return None
\*/
void show ();
/*\
* #brief Debug utility which asserts there is only a single
* memory block
\*/
bool unified ();
private:
// Fixed allocator capacity
size_t d_capacity;
// Fixed memory
uint8_t *d_memory;
// Pointer to head of the linked list
block_h *d_free_list;
// Available memory
size_t d_free_memory_size;
// Unit size
static const size_t mem_unit_size = sizeof(block_h);
};
#endif
Source File
#include "static_allocator.h"
Static_Allocator::Static_Allocator (uint8_t *memory, size_t size):
d_capacity(0),
d_memory(NULL),
d_free_list(NULL),
d_free_memory_size(0)
{
// Assign memory array
d_memory = memory;
// Trim off two mem_unit_size for head + init-block + spillover
d_capacity = size - (size % mem_unit_size) - 2 * mem_unit_size;
// Ensure free memory is set to capacity
d_free_memory_size = d_capacity;
}
uint8_t * Static_Allocator::alloc (size_t size)
{
block_h *last, *curr;
// Number of blocks sized units to allocate
size_t nblocks = (size + mem_unit_size - 1) / mem_unit_size + 1;
// If larger than total capacity or invalid size; immediately return
if ((nblocks * sizeof(block_h)) > d_capacity || size == 0) {
return NULL;
}
// If uninitialized, create initial list units
if ((last = d_free_list) == NULL) {
block_h *head = reinterpret_cast<block_h *>(d_memory);
head->d.size = 0;
block_h *init = head + 1;
init->d.size = d_capacity / sizeof(block_h);
init->d.next = d_free_list = last = head;
head->d.next = init;
}
// Problem: You must not be allowed to merge the init block
// Look for space - stop if wrapped around
for (curr = last->d.next; ; last = curr, curr = curr->d.next) {
// If sufficient space available
if (curr->d.size >= nblocks) {
if (curr->d.size == nblocks) {
last->d.next = curr->d.next;
} else {
curr->d.size -= nblocks;
curr += curr->d.size; // Suspect
curr->d.size = nblocks;
}
// Reassign free list head
d_free_list = last;
// Update amount of free memory available
d_free_memory_size -= nblocks * sizeof(block_h);
return reinterpret_cast<uint8_t *>(curr + 1);
}
// Otherwise not sufficient space. If reached
// the head of the list again, there is no more
// memory left
if (curr == d_free_list) {
return NULL;
}
}
}
dx::status_t Static_Allocator::free (uint8_t *ptr)
{
block_h *b, *p;
// Check if parameter is valid
if (ptr == NULL) {
std::cerr << "Null pointer!";
return dx::STATUS_BAD_PARAM;
}
// Check if memory is in range
if (!(ptr >= (d_memory + 2 * mem_unit_size)
&& ptr < (d_memory + d_capacity + 2 * mem_unit_size))) {
std::cerr << "Pointer out of range!";
return dx::STATUS_ERR;
}
// Obtain block header (ptr - sizeof(block_h))
b = reinterpret_cast<block_h *>(ptr) - 1;
// Update available memory size
d_free_memory_size += b->d.size;
// Find insertion location for block
for (p = d_free_list; !(b >= p && b < p->d.next); p = p->d.next) {
// If the block comes at the end of the list - break
if (p >= p->d.next && b > p) {
break;
}
// If at the end of the list, but block comes before next link
if (p >= p->d.next && b < p->d.next) {
break;
}
}
// [p] <----b----> [p->next] ----- [X]
// Check if we can merge forwards
if (b + b->d.size == p->d.next) {
b->d.size += (p->d.next)->d.size;
b->d.next = (p->d.next)->d.next;
} else {
b->d.next = p->d.next;
}
// Check if we can merge backwards
if (p + p->d.size == b) {
p->d.size += b->d.size;
p->d.next = b->d.next;
} else {
p->d.next = b;
}
d_free_list = p;
return dx::STATUS_OK;
}
size_t Static_Allocator::free_memory_size ()
{
return d_free_memory_size;
}
void Static_Allocator::show ()
{
std::cout << "Block Size: " << sizeof(block_h) << "\n"
<< "Capacity: " << d_capacity << "\n"
<< "Free Size (B): " << d_free_memory_size << "\n";
if (d_free_list == NULL) {
std::cout << "<uninitialized>\n";
return;
}
// Show all blocks
block_h *p = d_free_list;
do {
std::cout << "-------------------------------\n";
std::cout << "Block Address: " << (uint64_t)(p) << "\n";
std::cout << "Blocks (32B): " << p->d.size << "\n";
std::cout << "Next: " << (uint64_t)(p->d.next) << "\n";
std::cout << "{The next block is " << (uint64_t)((p->d.next) - (p)) << " bytes away, but we claim the next " << p->d.size * mem_unit_size << " bytes\n";
p = p->d.next;
} while (p != d_free_list);
std::cout << "-------------------------------\n\n\n";
}
bool Static_Allocator::unified () {
if (d_free_list == NULL) {
return false;
}
return ((d_free_list->d.next)->d.next == d_free_list);
}
How do you resize a Lua 5.1 userdata object in C during runtime?
I would like to change the size of the NumArray structure described in Roberto Ierusalimschy's book Programming in Lua, 2nd edition, pp. 260, from the Lua 5.1 console.
My NumArray userdata can either store unsigned chars or lua_Numbers. I played around with calling the unmodified luaM_realloc_ function defined in lmem.c, but finally any call to C's realloc
(via l_alloc) function just returns NULL, so I get a not enough memory error message.
Would somebody please help me ?
--- lapi.c -------------------------------------------------------------------------------------------------
LUA_API void *agn_resizeud (lua_State *L, void *block, size_t osize, size_t nsize) {
Udata *u = (Udata *)luaM_realloc_(L, block, osize + sizeof(Udata), nsize + sizeof(Udata));
u->uv.len = nsize;
if (u == NULL)
luaL_error(L, "Error in " LUA_QS ": failed to allocate memory.", "API/agn_resizeud");
return u;
}
--- numarray.c ---------------------------------------------------------------------------------------------
#define checkarray(L, n) (NumArray *)luaL_checkudata(L, n, "numarray")
[...]
/* Auxiliary C function to finally call luaM_realloc_ via the Lua C API */
void *reallocud (lua_State *L, void *block, size_t o, size_t n, size_t sizeofelem, size_t sizeofnumarray) {
if (n + 1 > MAX_SIZET/sizeofelem)
luaL_error(L, "Error in " LUA_QS ": memory allocation error: block too big.", "(reallocud)", n);
else
return agn_resizeud(L, block, sizeofnumarray + o*sizeofelem, sizeofnumarray + n*sizeofelem);
}
/* function to resize a NumArray */
static int numarray_resize (lua_State *L) {
size_t i;
global_State *g = G(L);
NumArray *a = checkarray(L, 1);
int n = luaL_checkinteger(L, 2);
if (n < 1)
luaL_error(L, "Error in " LUA_QS ": new size %d is non-positive.", "numarray.resize", n);
if (n == a->size) { /* do nothing and do not complain */
lua_pushinteger(L, a->size);
return 1;
}
if (n > a->size) { /* extend */
a = reallocud(L, a, a->size, n, a->isnumber ? sizeof(lua_Number) : sizeof(char), sizeof(NumArray));
if (a->isnumber) {
for (i=a->size; i < n; i++) a->data.n[i] = 0;
} else {
for (i=a->size; i < n; i++) a->data.c[i] = 0;
}
} else { /* reduce */
if (a->isnumber) {
for (i=a->size - 1; i > n - 1; i--) a->data.n[i] = 0;
} else {
for (i=a->size - 1; i > n - 1; i--) a->data.c[i] = 0;
}
a = reallocud(L, a, a->size, n, a->isnumber ? sizeof(lua_Number) : sizeof(char), sizeof(NumArray));
}
a->size = n;
lua_pushnumber(L, a->size);
return 1;
}
[...]
typedef struct NumArray {
size_t size; /* number of slots */
char isnumber;
union data {
lua_Number n[1]; /* pointer to the various lua_Number values */
unsigned char c[1];
} data;
} NumArray;
There is no need to mess with the internals of Lua.
But you need to change how the userdata is created: instead of allocating the struct and the data in the same memory block, allocate them separately, using lua_newuserdata to allocate the struct and malloc to allocate the data.
Then, to resize the data part, just update the size in the struct and realloc the data part.
you have:
static int numarray_resize (lua_State *L) {
[...]
if (n > a->size) { /* extend */
a = realloc(L, a, a->size, n, a->isnumber ? sizeof(lua_Number) : sizeof(char), sizeof(NumArray));
if (a->isnumber) {
[...]
But realloc takes 2 arguments, you want reallocud.
reallocing L with size a is likely to be a giant number and L may not have ever been allocated by the alloc family so there a couple reasons to get NULL unexpectedly.