Ambiguous error "...undeclared (first use in this function)" - c

For whatever reason, MinGW's gcc can't seem to find the '#define' block for this variable. The full error is
.../mapped.c:396:28: error: '_zzip_strcasecmp' undeclared (first use in this function)
Contents of mapped.c :
/*
* NOTE: this is part of libzzipmmapped (i.e. it is not libzzip).
* ==================
*
* These routines are fully independent from the traditional zzip
* implementation. They assume a readonly mmapped sharedmem block
* representing a complete zip file. The functions show how to
* parse the structure, find files and return a decoded bytestream.
*
* These routines are a bit simple and really here for documenting
* the way to access a zip file. The complexity of zip access comes
* from staggered reading of bytes and reposition of a filepointer in
* a big archive with lots of files and long compressed datastreams.
* Plus varaints of drop-in stdio replacements, obfuscation routines,
* auto fileextensions, drop-in dirent replacements, and so on...
*
* Author:
* Guido Draheim <guidod#gmx.de>
*
* Copyright (c) 2003,2004,2006 Guido Draheim
* All rights reserved,
* use under the restrictions of the
* Lesser GNU General Public License
* or alternatively the restrictions
* of the Mozilla Public License 1.1
*/
#define _ZZIP_DISK_FILE_STRUCT 1
#ifdef __linux__
#define _GNU_SOURCE _glibc_developers_are_idiots_to_call_strndup_gnu_specific_
#endif
#include <zzip/mmapped.h>
#include <zzip/format.h>
#include <zzip/fetch.h>
#include <zzip/__mmap.h>
#include <zzip/__fnmatch.h>
#include <stdlib.h>
#include <sys/stat.h>
#if defined ZZIP_HAVE_UNISTD_H
#include <unistd.h>
#elif defined ZZIP_HAVE_IO_H
#include <io.h>
#endif
#if defined ZZIP_HAVE_STRING_H
#include <string.h>
#elif defined ZZIP_HAVE_STRINGS_H
#include <strings.h>
#endif
#if __STDC_VERSION__+0 > 199900L
#define ___
#define ____
#else
#define ___ {
#define ____ }
#endif
/** => zzip_disk_mmap
* This function does primary initialization of a disk-buffer struct.
*/
int
zzip_disk_init(ZZIP_DISK* disk, void* buffer, zzip_size_t buflen)
{
disk->buffer = (zzip_byte_t*) buffer;
disk->endbuf = (zzip_byte_t*) buffer + buflen;
disk->reserved = 0;
disk->flags = 0;
disk->mapped = 0;
/* do not touch disk->user */
/* do not touch disk->code */
return 0;
}
/** => zzip_disk_mmap
* This function allocates a new disk-buffer with => malloc(3)
*/
zzip__new__ ZZIP_DISK*
zzip_disk_new(void)
{
ZZIP_DISK* disk = malloc(sizeof(disk));
if (! disk) return disk;
zzip_disk_init (disk, 0, 0);
return disk;
}
/** turn a filehandle into a mmapped zip disk archive handle
*
* This function uses the given file-descriptor to detect the length of the
* file and calls the system => mmap(2) to put it in main memory. If it is
* successful then a newly allocated ZZIP_DISK* is returned with
* disk->buffer pointing to the mapview of the zipdisk content.
*/
zzip__new__ ZZIP_DISK*
zzip_disk_mmap(int fd)
{
struct stat st;
if (fstat (fd, &st) || ! st.st_size) return 0;
___ ZZIP_DISK* disk = zzip_disk_new (); if (! disk) return 0;
disk->buffer = _zzip_mmap (& disk->mapped, fd, 0, st.st_size);
if (disk->buffer == MAP_FAILED) { free (disk); return 0; }
disk->endbuf = disk->buffer + st.st_size;
return disk; ____;
}
/** => zzip_disk_mmap
* This function is the inverse of => zzip_disk_mmap and using the system
* munmap(2) on the buffer area and => free(3) on the ZZIP_DISK structure.
*/
int
zzip_disk_munmap(ZZIP_DISK* disk)
{
if (! disk) return 0;
_zzip_munmap (disk->mapped, disk->buffer, disk->endbuf-disk->buffer);
free (disk);
return 0;
}
/** => zzip_disk_mmap
*
* This function opens the given archive by name and turn the filehandle
* to => zzip_disk_mmap for bringing it to main memory. If it can not
* be => mmap(2)'ed then we slurp the whole file into a newly => malloc(2)'ed
* memory block. Only if that fails too then we return null. Since handling
* of disk->buffer is ambigous it should not be snatched away please.
*/
ZZIP_DISK* zzip__new__
zzip_disk_open(char* filename)
{
# ifndef O_BINARY
# define O_BINARY 0
# endif
struct stat st;
if (stat (filename, &st) || ! st.st_size) return 0;
___ int fd = open (filename, O_RDONLY|O_BINARY);
if (fd <= 0) return 0;
___ ZZIP_DISK* disk = zzip_disk_mmap (fd);
if (disk) return disk;
___ zzip_byte_t* buffer = malloc (st.st_size);
if (! buffer) return 0;
if ((st.st_size == read (fd, buffer, st.st_size)) &&
(disk = zzip_disk_new ()))
{
disk->buffer = buffer;
disk->endbuf = buffer+st.st_size;
disk->mapped = -1;
}else free (buffer);
return disk; ____;____;____;
}
/** => zzip_disk_mmap
*
* This function will release all data needed to access a (mmapped)
* zip archive, including any malloc()ed blocks, sharedmem mappings
* and it dumps the handle struct as well.
*/
int
zzip_disk_close(ZZIP_DISK* disk)
{
if (! disk) return 0;
if (disk->mapped != -1) return zzip_disk_munmap (disk);
free (disk->buffer);
free (disk);
return 0;
}
/* ====================================================================== */
/* helper functions */
#ifdef ZZIP_HAVE_STRNDUP
#define _zzip_strndup strndup
#else
/* if your system does not have strndup: */
zzip__new__ static char* _zzip_strndup(char* p, size_t maxlen)
{
if (! p) return 0;
___ zzip_byte_t* r = malloc (maxlen+1);
if (! r) return r;
strncpy (r, p, maxlen);
r[maxlen] = '\0';
return r; ____;
}
#endif
#if defined ZZIP_HAVE_STRCASECMP || defined strcasecmp
#define _zzip_strcasecmp strcasecmp
#else
/* if your system does not have strcasecmp: */
static int _zzip_strcasecmp(char* __zzip_restrict a, char* _zzip_restrict b)
{
if (! a) return (b) ? 1 : 0;
if (! b) return -1;
while (1)
{
int v = tolower(*a) - tolower(*b);
if (v) return v;
if (! *a) return 1;
if (! *b) return -1;
a++; b++;
}
}
#endif
/** helper functions for (mmapped) zip access api
*
* This function augments the other zzip_disk_entry_* helpers: here we move
* a disk_entry pointer (as returned by _find* functions) into a pointer to
* the data block right after the file_header. Only disk->buffer would be
* needed to perform the seek but we check the mmapped range end as well.
*/
zzip_byte_t*
zzip_disk_entry_to_data(ZZIP_DISK* disk, struct zzip_disk_entry* entry)
{
struct zzip_file_header* file =
zzip_disk_entry_to_file_header(disk, entry);
if (file) return zzip_file_header_to_data (file);
return 0;
}
/** => zzip_disk_entry_to_data
* This function does half the job of => zzip_disk_entry_to_data where it
* can augment with => zzip_file_header_to_data helper from format/fetch.h
*/
struct zzip_file_header*
zzip_disk_entry_to_file_header(ZZIP_DISK* disk, struct zzip_disk_entry* entry)
{
zzip_byte_t* file_header = /* (struct zzip_file_header*) */
(disk->buffer + zzip_disk_entry_fileoffset (entry));
if (disk->buffer > file_header || file_header >= disk->endbuf)
return 0;
return (struct zzip_file_header*) file_header;
}
/** => zzip_disk_entry_to_data
* This function is a big helper despite its little name: in a zip file the
* encoded filenames are usually NOT zero-terminated but for common usage
* with libc we need it that way. Secondly, the filename SHOULD be present
* in the zip central directory but if not then we fallback to the filename
* given in the file_header of each compressed data portion.
*/
zzip__new__ char*
zzip_disk_entry_strdup_name(ZZIP_DISK* disk, struct zzip_disk_entry* entry)
{
if (! disk || ! entry) return 0;
___ char* name; zzip_size_t len;
struct zzip_file_header* file;
if ((len = zzip_disk_entry_namlen (entry)))
name = zzip_disk_entry_to_filename (entry);
else if ((file = zzip_disk_entry_to_file_header (disk, entry)) &&
(len = zzip_file_header_namlen (file)))
name = zzip_file_header_to_filename (file);
else
return 0;
if ((zzip_byte_t*) name < disk->buffer ||
(zzip_byte_t*) name+len > disk->endbuf)
return 0;
return _zzip_strndup (name, len); ____;
}
/** => zzip_disk_entry_to_data
* This function is similar creating a reference to a zero terminated
* string but it can only exist in the zip central directory entry.
*/
zzip__new__ char*
zzip_disk_entry_strdup_comment(ZZIP_DISK* disk, struct zzip_disk_entry* entry)
{
if (! disk || ! entry) return 0;
___ char* text; zzip_size_t len;
if ((len = zzip_disk_entry_comment (entry)))
text = zzip_disk_entry_to_comment (entry);
else
return 0;
if ((zzip_byte_t*) text < disk->buffer ||
(zzip_byte_t*) text+len > disk->endbuf)
return 0;
return _zzip_strndup (text, len); ____;
}
/* ====================================================================== */
/** => zzip_disk_findfile
*
* This function is the first call of all the zip access functions here.
* It contains the code to find the first entry of the zip central directory.
* Here we require the mmapped block to represent a real zip file where the
* disk_trailer is _last_ in the file area, so that its position would be at
* a fixed offset from the end of the file area if not for the comment field
* allowed to be of variable length (which needs us to do a little search
* for the disk_tailer). However, in this simple implementation we disregard
* any disk_trailer info telling about multidisk archives, so we just return
* a pointer to the zip central directory.
*
* For an actual means, we are going to search backwards from the end
* of the mmaped block looking for the PK-magic signature of a
* disk_trailer. If we see one then we check the rootseek value to
* find the first disk_entry of the root central directory. If we find
* the correct PK-magic signature of a disk_entry over there then we
* assume we are done and we are going to return a pointer to that label.
*
* The return value is a pointer to the first zzip_disk_entry being checked
* to be within the bounds of the file area specified by the arguments. If
* no disk_trailer was found then null is returned, and likewise we only
* accept a disk_trailer with a seekvalue that points to a disk_entry and
* both parts have valid PK-magic parts. Beyond some sanity check we try to
* catch a common brokeness with zip archives that still allows us to find
* the start of the zip central directory.
*/
struct zzip_disk_entry*
zzip_disk_findfirst(ZZIP_DISK* disk)
{
if (disk->buffer > disk->endbuf-sizeof(struct zzip_disk_trailer))
return 0;
___ zzip_byte_t* p = disk->endbuf-sizeof(struct zzip_disk_trailer);
for (; p >= disk->buffer ; p--)
{
zzip_byte_t* root; /* (struct zzip_disk_entry*) */
if (zzip_disk_trailer_check_magic(p)) {
root = disk->buffer + zzip_disk_trailer_get_rootseek (
(struct zzip_disk_trailer*)p);
if (root > p)
{ /* the first disk_entry is after the disk_trailer? can't be! */
zzip_size_t rootsize = zzip_disk_trailer_get_rootsize (
(struct zzip_disk_trailer*)p);
if (disk->buffer+rootsize > p) continue;
/* a common brokeness that can be fixed: we just assume the
* central directory was written directly before the trailer:*/
root = p - rootsize;
}
} else if (zzip_disk64_trailer_check_magic(p)) {
if (sizeof(void*) < 8) return 0; /* EOVERFLOW */
root = disk->buffer + zzip_disk64_trailer_get_rootseek (
(struct zzip_disk64_trailer*)p);
if (root > p) continue;
} else continue;
if (root < disk->buffer) continue;
if (zzip_disk_entry_check_magic(root))
return (struct zzip_disk_entry*) root;
}____;
return 0;
}
/** => zzip_disk_findfile
*
* This function takes an existing disk_entry in the central root directory
* (e.g. from zzip_disk_findfirst) and returns the next entry within in
* the given bounds of the mmapped file area.
*/
struct zzip_disk_entry*
zzip_disk_findnext(ZZIP_DISK* disk, struct zzip_disk_entry* entry)
{
if ((zzip_byte_t*)entry < disk->buffer ||
(zzip_byte_t*)entry > disk->endbuf-sizeof(entry) ||
! zzip_disk_entry_check_magic (entry) ||
zzip_disk_entry_sizeto_end (entry) > 64*1024)
return 0;
entry = zzip_disk_entry_to_next_entry (entry);
if ((zzip_byte_t*)entry > disk->endbuf-sizeof(entry) ||
! zzip_disk_entry_check_magic (entry) ||
zzip_disk_entry_sizeto_end (entry) > 64*1024 ||
zzip_disk_entry_skipto_end (entry) + sizeof(entry) > disk->endbuf)
return 0;
else
return entry;
}
/** search for files in the (mmapped) zip central directory
*
* This function is given a filename as an additional argument, to find the
* disk_entry matching a given filename. The compare-function is usually
* strcmp or strcasecmp or perhaps strcoll, if null then strcmp is used.
* - use null as argument for "after"-entry when searching the first
* matching entry, otherwise the last returned value if you look for other
* entries with a special "compare" function (if null then a doubled search
* is rather useless with this variant of _findfile).
*/
struct zzip_disk_entry*
zzip_disk_findfile(ZZIP_DISK* disk, char* filename,
struct zzip_disk_entry* after, zzip_strcmp_fn_t compare)
{
struct zzip_disk_entry* entry = (! after ? zzip_disk_findfirst (disk)
: zzip_disk_findnext (disk, after));
if (! compare)
compare = (zzip_strcmp_fn_t)( (disk->flags&1) ?
(_zzip_strcasecmp) : (strcmp));
for (; entry ; entry = zzip_disk_findnext (disk, entry))
{
/* filenames within zip files are often not null-terminated! */
char* realname = zzip_disk_entry_strdup_name (disk, entry);
if (realname && ! compare(filename, realname))
{
free (realname);
return entry;
}
free (realname);
}
return 0;
}
/** => zzip_disk_findfile
*
* This function uses a compare-function with an additional argument
* and it is called just like fnmatch(3) from POSIX.2 AD:1993), i.e.
* the argument filespec first and the ziplocal filename second with
* the integer-flags put in as third to the indirect call. If the
* platform has fnmatch available then null-compare will use that one
* and otherwise we fall back to mere strcmp, so if you need fnmatch
* searching then please provide an implementation somewhere else.
* - use null as argument for "after"-entry when searching the first
* matching entry, or the last disk_entry return-value to find the
* next entry matching the given filespec.
*/
struct zzip_disk_entry*
zzip_disk_findmatch(ZZIP_DISK* disk, char* filespec,
struct zzip_disk_entry* after,
zzip_fnmatch_fn_t compare, int flags)
{
struct zzip_disk_entry* entry = (! after ? zzip_disk_findfirst (disk)
: zzip_disk_findnext (disk, after));
if (! compare) {
compare = (zzip_fnmatch_fn_t) _zzip_fnmatch;
if (disk->flags&1) disk->flags |= _zzip_fnmatch_CASEFOLD;
}
for (; entry ; entry = zzip_disk_findnext (disk, entry))
{
/* filenames within zip files are often not null-terminated! */
char* realname = zzip_disk_entry_strdup_name(disk, entry);
if (realname && ! compare(filespec, realname, flags))
{
free (realname);
return entry;
}
free (realname);
}
return 0;
}
/* ====================================================================== */
/** => zzip_disk_fopen
*
* the ZZIP_DISK_FILE* is rather simple in just encapsulating the
* arguments given to this function plus a zlib deflate buffer.
* Note that the ZZIP_DISK pointer does already contain the full
* mmapped file area of a zip disk, so open()ing a file part within
* that area happens to be a lookup of its bounds and encoding. That
* information is memorized on the ZZIP_DISK_FILE so that subsequent
* _read() operations will be able to get the next data portion or
* return an eof condition for that file part wrapped in the zip archive.
*/
zzip__new__ ZZIP_DISK_FILE*
zzip_disk_entry_fopen (ZZIP_DISK* disk, ZZIP_DISK_ENTRY* entry)
{
/* keep this in sync with zzip_mem_entry_fopen */
struct zzip_file_header* header =
zzip_disk_entry_to_file_header (disk, entry);
if (! header) return 0;
___ ZZIP_DISK_FILE* file = malloc(sizeof(ZZIP_DISK_FILE));
if (! file) return file;
file->buffer = disk->buffer;
file->endbuf = disk->endbuf;
file->avail = zzip_file_header_usize (header);
if (! file->avail || zzip_file_header_data_stored (header))
{ file->stored = zzip_file_header_to_data (header); return file; }
file->stored = 0;
file->zlib.opaque = 0;
file->zlib.zalloc = Z_NULL;
file->zlib.zfree = Z_NULL;
file->zlib.avail_in = zzip_file_header_csize (header);
file->zlib.next_in = zzip_file_header_to_data (header);
if (! zzip_file_header_data_deflated(header) ||
inflateInit2(&file->zlib, -MAX_WBITS) != Z_OK)
{ free (file); return 0; }
return file;
____;
}
/** openening a file part wrapped within a (mmapped) zip archive
*
* This function opens a file found by name, so it does a search into
* the zip central directory with => zzip_disk_findfile and whatever
* is found first is given to => zzip_disk_entry_fopen
*/
zzip__new__ ZZIP_DISK_FILE*
zzip_disk_fopen (ZZIP_DISK* disk, char* filename)
{
ZZIP_DISK_ENTRY* entry = zzip_disk_findfile (disk, filename, 0, 0);
if (! entry) return 0; else return zzip_disk_entry_fopen (disk, entry);
}
/** => zzip_disk_fopen
*
* This function reads more bytes into the output buffer specified as
* arguments. The return value is null on eof or error, the stdio-like
* interface can not distinguish between these so you need to check
* with => zzip_disk_feof for the difference.
*/
zzip_size_t
zzip_disk_fread (void* ptr, zzip_size_t sized, zzip_size_t nmemb,
ZZIP_DISK_FILE* file)
{
zzip_size_t size = sized*nmemb;
if (size > file->avail) size = file->avail;
if (file->stored)
{
memcpy (ptr, file->stored, size);
file->stored += size;
file->avail -= size;
return size;
}
file->zlib.avail_out = sized*nmemb;
file->zlib.next_out = ptr;
___ zzip_size_t total_old = file->zlib.total_out;
___ int err = inflate (& file->zlib, Z_NO_FLUSH);
if (err == Z_STREAM_END)
file->avail = 0;
else if (err == Z_OK)
file->avail -= file->zlib.total_out - total_old;
else
return 0;
return file->zlib.total_out - total_old;
____;____;
}
/** => zzip_disk_fopen
* This function releases any zlib decoder info needed for decompression
* and dumps the ZZIP_DISK_FILE* then.
*/
int
zzip_disk_fclose (ZZIP_DISK_FILE* file)
{
if (! file->stored)
inflateEnd (& file->zlib);
free (file);
return 0;
}
/** => zzip_disk_fopen
*
* This function allows to distinguish an error from an eof condition.
* Actually, if we found an error but we did already reach eof then we
* just keep on saying that it was an eof, so the app can just continue.
*/
int
zzip_disk_feof (ZZIP_DISK_FILE* file)
{
return ! file || ! file->avail;
}
Does anyone have any idea what's going on? Sorry if it may look obvious to some of you, I'm a newbie with this stuff.
Thanks!
Edit: Since I couldn't figure what's wrong, I decided to use the visual studio solution that came with the source code and I finally managed to build it successfully. Thanks to everyone who tried to help me with this.

The reason for your error is that in C language names enclosed in () are not treated as macros. That's actually a feature of the language that exists specifically for that purpose: to give you an opportunity to "ignore" macro definitions when you want to.
When you attempt to compile the code with GCC, it appears that strcasecmp is available. The true branch of the #if is taken by the compiler, which means that _zzip_strcasecmp gets defined as a macro
#define _zzip_strcasecmp strcasecmp
This will work fine as long as you call it as
_zzip_strcasecmp( /* whatever */ )
But this will not work when you refer to it as (_zzip_strcasecmp) here
compare = (zzip_strcmp_fn_t)( (disk->flags&1) ?
(_zzip_strcasecmp) : (strcmp));
The compiler sees (_zzip_strcasecmp) as a request to ignore macros when looking for _zzip_strcasecmp. The compiler will look for non-macro definition of _zzip_strcasecmp and fail with the error message, since such definition does not exist.
When you compile in Visual Siudio, strcasecmp is not available. The false branch of the #if is taken and _zzip_strcasecmp gets defined internally as an ordinary function. So, everything works fine.
One way to fix it for macro version is to remove these excessive () around function names in the problematic erroneous statement
compare = (zzip_strcmp_fn_t)( (disk->flags&1) ?
_zzip_strcasecmp : strcmp);
This should compile in GCC assuming strcasecmp is really available there.
I don't know why they put these superfluous () in the original version. Might be an instance of "rampant parenthesizing" and misguided "better safe than sorry" sentiment. Might actually be done intentionally to prevent use of macros in this context (in which case the bad guy is whoever wrote that #if).

Related

How to use scandir() in C to list sorted subdirectories recursively

I'm implementing parts of the Linux ls command in C. I want to sort the contents of directories lexicographically, which I've been doing using scandir(). This is easy enough for listing single directories, but I'm having trouble doing it for listing subdirectories recursively. My current code: (results in a segmentation faults once a directory type is reached)
void recursive(char* arg){
int i;
struct dirent **file_list;
int num;
char* next_dir;
num = scandir(arg, &file_list, NULL, alphasort);
for(i = 0; i < num; i++) {
if(file_list[i]->d_type == DT_DIR) {
if(strcmp(".", file_list[i]->d_name) != 0 && strcmp("..", file_list[i]->d_name) != 0) {
// Directories are printed with a colon to distinguish them from files
printf("%s: \n", file_list[i]->d_name);
strcpy(next_dir, arg);
strcat(next_dir, "/");
strcat(next_dir, file_list[i]->d_name);
printf("\n");
recursive(next_dir);
}
} else {
if(strcmp(".", file_list[i]->d_name) != 0 && strcmp("..", file_list[i]->d_name) != 0) {
printf("%s \n", file_list[i]->d_name);
}
}
}
}
int main(void) {
recursive(".");
return 0;
}
There are two recommended methods for traversing entire filesystem trees in Linux and other POSIXy systems:
nftw(): man 3 nftw
Given an initial path, a callback function, the maximum number of descriptors to use, and a set of flags, nftw() will call the callback function once for every filesystem object in the subtree. The order in which entries in the same directory is called is not specified, however.
This is the POSIX.1 (IEEE 1003) function.
fts_open()/fts_read()/fts_children()/fts_close(): man 3 fts
The fts interface provides a way to traverse filesystem hierarchies. The fts_children() provides a linked list of filesystem entries sorted by the comparison function specified in the fts_open() call. It is rather similar to how scandir() returns an array of filesystem entries, except that the two use very different structures to describe each filesystem entry.
Prior to glibc 2.23 (released in 2016), the Linux (glibc) fts implementation had bugs when using 64-bit file sizes (so on x86-64, or when compiling with -D_FILE_OFFSET_BITS=64).
These are BSD functions (FreeBSD/OpenBSD/macOS), but are available in Linux also.
Finally, there is also the atfile version of scandir(), scandirat(), that returns the filtered and sorted filesystem entries from a specific directory, but in addition to the pathname, it takes a file descriptor to the relative root directory to be used as a parameter. (If AT_FDCWD is used instead of a file descriptor, then scandirat() behaves like scandir().)
The simplest option here is to use nftw(), store all walked paths, and finally sort the paths. For example, walk.c:
// SPDX-License-Identifier: CC0-1.0
#define _POSIX_C_SOURCE 200809L
#define _GNU_SOURCE
#include <stdlib.h>
#include <locale.h>
#include <ftw.h>
#include <string.h>
#include <stdio.h>
#include <errno.h>
struct entry {
/* Insert additional properties like 'off_t size' here. */
char *name; /* Always points to name part of pathname */
char pathname[]; /* Full path and name */
};
struct listing {
size_t max; /* Number of entries allocated for */
size_t num; /* Number of entries in the array */
struct entry **ent; /* Array of pointers, one per entry */
};
#define STRUCT_LISTING_INITIALIZER { 0, 0, NULL }
/* Locale-aware sort for arrays of struct entry pointers.
*/
static int entrysort(const void *ptr1, const void *ptr2)
{
const struct entry *ent1 = *(const struct entry **)ptr1;
const struct entry *ent2 = *(const struct entry **)ptr2;
return strcoll(ent1->pathname, ent2->pathname);
}
/* Global variable used by nftw_add() to add to the listing */
static struct listing *nftw_listing = NULL;
static int nftw_add(const char *pathname, const struct stat *info, int typeflag, struct FTW *ftwbuf)
{
const char *name = pathname + ftwbuf->base;
/* These generate no code, just silences the warnings about unused parameters. */
(void)info;
(void)typeflag;
/* Ignore "." and "..". */
if (name[0] == '.' && !name[1])
return 0;
if (name[0] == '.' && name[1] == '.' && !name[2])
return 0;
/* Make sure there is room for at least one more entry in the listing. */
if (nftw_listing->num >= nftw_listing->max) {
const size_t new_max = nftw_listing->num + 1000;
struct entry **new_ent;
new_ent = realloc(nftw_listing->ent, new_max * sizeof (struct entry *));
if (!new_ent)
return -ENOMEM;
nftw_listing->max = new_max;
nftw_listing->ent = new_ent;
}
const size_t pathnamelen = strlen(pathname);
struct entry *ent;
/* Allocate memory for this entry.
Remember to account for the name, and the end-of-string terminator, '\0', at end of name. */
ent = malloc(sizeof (struct entry) + pathnamelen + 1);
if (!ent)
return -ENOMEM;
/* Copy other filesystem entry properties to ent here; say 'ent->size = info->st_size;'. */
/* Copy pathname, including the end-of-string terminator, '\0'. */
memcpy(ent->pathname, pathname, pathnamelen + 1);
/* The name pointer is always to within the pathname. */
ent->name = ent->pathname + ftwbuf->base;
/* Append. */
nftw_listing->ent[nftw_listing->num++] = ent;
return 0;
}
/* Scan directory tree starting at path, adding the entries to struct listing.
Note: the listing must already have been properly initialized!
Returns 0 if success, nonzero if error; -1 if errno is set to indicate error.
*/
int scan_tree_sorted(struct listing *list, const char *path)
{
if (!list) {
errno = EINVAL;
return -1;
}
if (!path || !*path) {
errno = ENOENT;
return -1;
}
nftw_listing = list;
int result = nftw(path, nftw_add, 64, FTW_DEPTH);
nftw_listing = NULL;
if (result < 0) {
errno = -result;
return -1;
} else
if (result > 0) {
errno = 0;
return result;
}
if (list->num > 2)
qsort(list->ent, list->num, sizeof list->ent[0], entrysort);
return 0;
}
int main(int argc, char *argv[])
{
struct listing list = STRUCT_LISTING_INITIALIZER;
setlocale(LC_ALL, "");
if (argc < 2 || !strcmp(argv[1], "-h") || !strcmp(argv[1], "--help")) {
const char *arg0 = (argc > 0 && argv && argv[0] && argv[0][0]) ? argv[0] : "(this)";
fprintf(stderr, "\n");
fprintf(stderr, "Usage: %s [ -h | --help ]\n", arg0);
fprintf(stderr, " %s .\n", arg0);
fprintf(stderr, " %s TREE [ TREE ... ]\n", arg0);
fprintf(stderr, "\n");
fprintf(stderr, "This program lists all files and directories starting at TREE,\n");
fprintf(stderr, "in sorted order.\n");
fprintf(stderr, "\n");
return EXIT_SUCCESS;
}
for (int arg = 1; arg < argc; arg++) {
if (scan_tree_sorted(&list, argv[arg])) {
fprintf(stderr, "%s: Error scanning directory tree: %s.\n", argv[arg], strerror(errno));
return EXIT_FAILURE;
}
}
printf("Found %zu entries:\n", list.num);
for (size_t i = 0; i < list.num; i++)
printf("\t%s\t(%s)\n", list.ent[i]->pathname, list.ent[i]->name);
return EXIT_SUCCESS;
}
Compile using gcc -Wall -Wextra -O2 walk.c -o walk, and run using e.g. ./walk ...
The scan_tree_sorted() function calls nftw() for the directory specified, updating the global variable nftw_listing so that the nftw_add() callback function can add each new directory entry to it. If the listing contains more that one entry afterwards, it is sorted using qsort() and a locale-aware comparison function (based on strcoll()).
nftw_add() skips . and .., and adds every other pathname to the listing structure nftw_listing. It automatically grows the array as needed in linear fashion; the new_max = nftw_listing->num + 1000; means we allocate in units of a thousand (pointers).
The scan_tree_sorted() can be called multiple times with the same listing as the target, if one wants to list disjoint subtrees in one listing. Note, however, that it does not check for duplicates, although those could easily be filtered out after the qsort.

What is the use of the type userptr_t in os161?

I'm trying to complete an assignment for an operating systems course, Here.
I had a question from the assignment:
What is the purpose of userptr_t?
When I searched the source code for userptr_t, Here, I found this:
/*
* Define userptr_t as a pointer to a one-byte struct, so it won't mix
* with other pointers.
*/
struct __userptr { char _dummy; };
typedef struct __userptr *userptr_t;
typedef const struct __userptr *const_userptr_t;
I can't get to understand completely the use of it, can anyone explain what is the purpose of this type?
It's used here for example in the file copyinout.c in the functions copyin, copyout, copyinstr, copyoutstr and other functions:
#include <types.h>
#include <kern/errno.h>
#include <lib.h>
#include <setjmp.h>
#include <thread.h>
#include <current.h>
#include <vm.h>
#include <copyinout.h>
/*
* User/kernel memory copying functions.
*
* These are arranged to prevent fatal kernel memory faults if invalid
* addresses are supplied by user-level code. This code is itself
* machine-independent; it uses the machine-dependent C setjmp/longjmp
* facility to perform recovery.
*
* However, it assumes things about the memory subsystem that may not
* be true on all platforms.
*
* (1) It assumes that user memory is mapped into the current address
* space while running in the kernel, and can be accessed by just
* dereferencing a pointer in the ordinary way. (And not, for example,
* with special instructions or via special segment registers.)
*
* (2) It assumes that the user-space region of memory is contiguous
* and extends from 0 to some virtual address USERSPACETOP, and so if
* a user process passes a kernel address the logic in copycheck()
* will trap it.
*
* (3) It assumes that access to user memory from the kernel behaves
* the same way as access to user memory from user space: for
* instance, that the processor honors read-only bits on memory pages
* when in kernel mode.
*
* (4) It assumes that if a proper user-space address that is valid
* but not present, or not valid at all, is touched from the kernel,
* that the correct faults will occur and the VM system will load the
* necessary pages and whatnot.
*
* (5) It assumes that the machine-dependent trap logic provides and
* honors a tm_badfaultfunc field in the thread_machdep structure.
* This feature works as follows: if an otherwise fatal fault occurs
* in kernel mode, and tm_badfaultfunc is set, execution resumes in
* the function pointed to by tm_badfaultfunc.
*
* This code works by setting tm_badfaultfunc and then copying memory
* in an ordinary fashion. If these five assumptions are satisfied,
* which is the case for many ordinary CPU types, this code should
* function correctly. If the assumptions are not satisfied on some
* platform (for instance, certain old 80386 processors violate
* assumption 3), this code cannot be used, and cpu- or platform-
* specific code must be written.
*
* To make use of this code, in addition to tm_badfaultfunc the
* thread_machdep structure should contain a jmp_buf called
* "tm_copyjmp".
*/
/*
* Recovery function. If a fatal fault occurs during copyin, copyout,
* copyinstr, or copyoutstr, execution resumes here. (This behavior is
* caused by setting t_machdep.tm_badfaultfunc and is implemented in
* machine-dependent code.)
*
* We use the C standard function longjmp() to teleport up the call
* stack to where setjmp() was called. At that point we return EFAULT.
*/
static
void
copyfail(void)
{
longjmp(curthread->t_machdep.tm_copyjmp, 1);
}
/*
* Memory region check function. This checks to make sure the block of
* user memory provided (an address and a length) falls within the
* proper userspace region. If it does not, EFAULT is returned.
*
* stoplen is set to the actual maximum length that can be copied.
* This differs from len if and only if the region partially overlaps
* the kernel.
*
* Assumes userspace runs from 0 through USERSPACETOP-1.
*/
static
int
copycheck(const_userptr_t userptr, size_t len, size_t *stoplen)
{
vaddr_t bot, top;
*stoplen = len;
bot = (vaddr_t) userptr;
top = bot+len-1;
if (top < bot) {
/* addresses wrapped around */
return EFAULT;
}
if (bot >= USERSPACETOP) {
/* region is within the kernel */
return EFAULT;
}
if (top >= USERSPACETOP) {
/* region overlaps the kernel. adjust the max length. */
*stoplen = USERSPACETOP - bot;
}
return 0;
}
/*
* copyin
*
* Copy a block of memory of length LEN from user-level address USERSRC
* to kernel address DEST. We can use memcpy because it's protected by
* the tm_badfaultfunc/copyfail logic.
*/
int
copyin(const_userptr_t usersrc, void *dest, size_t len)
{
int result;
size_t stoplen;
result = copycheck(usersrc, len, &stoplen);
if (result) {
return result;
}
if (stoplen != len) {
/* Single block, can't legally truncate it. */
return EFAULT;
}
curthread->t_machdep.tm_badfaultfunc = copyfail;
result = setjmp(curthread->t_machdep.tm_copyjmp);
if (result) {
curthread->t_machdep.tm_badfaultfunc = NULL;
return EFAULT;
}
memcpy(dest, (const void *)usersrc, len);
curthread->t_machdep.tm_badfaultfunc = NULL;
return 0;
}
/*
* copyout
*
* Copy a block of memory of length LEN from kernel address SRC to
* user-level address USERDEST. We can use memcpy because it's
* protected by the tm_badfaultfunc/copyfail logic.
*/
int
copyout(const void *src, userptr_t userdest, size_t len)
{
int result;
size_t stoplen;
result = copycheck(userdest, len, &stoplen);
if (result) {
return result;
}
if (stoplen != len) {
/* Single block, can't legally truncate it. */
return EFAULT;
}
curthread->t_machdep.tm_badfaultfunc = copyfail;
result = setjmp(curthread->t_machdep.tm_copyjmp);
if (result) {
curthread->t_machdep.tm_badfaultfunc = NULL;
return EFAULT;
}
memcpy((void *)userdest, src, len);
curthread->t_machdep.tm_badfaultfunc = NULL;
return 0;
}
/*
* Common string copying function that behaves the way that's desired
* for copyinstr and copyoutstr.
*
* Copies a null-terminated string of maximum length MAXLEN from SRC
* to DEST. If GOTLEN is not null, store the actual length found
* there. Both lengths include the null-terminator. If the string
* exceeds the available length, the call fails and returns
* ENAMETOOLONG.
*
* STOPLEN is like MAXLEN but is assumed to have come from copycheck.
* If we hit MAXLEN it's because the string is too long to fit; if we
* hit STOPLEN it's because the string has run into the end of
* userspace. Thus in the latter case we return EFAULT, not
* ENAMETOOLONG.
*/
static
int
copystr(char *dest, const char *src, size_t maxlen, size_t stoplen,
size_t *gotlen)
{
size_t i;
for (i=0; i<maxlen && i<stoplen; i++) {
dest[i] = src[i];
if (src[i] == 0) {
if (gotlen != NULL) {
*gotlen = i+1;
}
return 0;
}
}
if (stoplen < maxlen) {
/* ran into user-kernel boundary */
return EFAULT;
}
/* otherwise just ran out of space */
return ENAMETOOLONG;
}
/*
* copyinstr
*
* Copy a string from user-level address USERSRC to kernel address
* DEST, as per copystr above. Uses the tm_badfaultfunc/copyfail
* logic to protect against invalid addresses supplied by a user
* process.
*/
int
copyinstr(const_userptr_t usersrc, char *dest, size_t len, size_t *actual)
{
int result;
size_t stoplen;
result = copycheck(usersrc, len, &stoplen);
if (result) {
return result;
}
curthread->t_machdep.tm_badfaultfunc = copyfail;
result = setjmp(curthread->t_machdep.tm_copyjmp);
if (result) {
curthread->t_machdep.tm_badfaultfunc = NULL;
return EFAULT;
}
result = copystr(dest, (const char *)usersrc, len, stoplen, actual);
curthread->t_machdep.tm_badfaultfunc = NULL;
return result;
}
/*
* copyoutstr
*
* Copy a string from kernel address SRC to user-level address
* USERDEST, as per copystr above. Uses the tm_badfaultfunc/copyfail
* logic to protect against invalid addresses supplied by a user
* process.
*/
int
copyoutstr(const char *src, userptr_t userdest, size_t len, size_t *actual)
{
int result;
size_t stoplen;
result = copycheck(userdest, len, &stoplen);
if (result) {
return result;
}
curthread->t_machdep.tm_badfaultfunc = copyfail;
result = setjmp(curthread->t_machdep.tm_copyjmp);
if (result) {
curthread->t_machdep.tm_badfaultfunc = NULL;
return EFAULT;
}
result = copystr((char *)userdest, src, len, stoplen, actual);
curthread->t_machdep.tm_badfaultfunc = NULL;
return result;
}
This looks like a strong typedef, i.e. a typedef intended to increase type safety by avoiding unintended uses/conversions of the wrapped data.
In your context, most likely intended to differentiate kernel pointers from user space pointers (usually mapped via the MMU).

Identifying the start and end of functions in a C program

I am currently programming in C to find the complexity of functions in a program based on the number of lines in the functions. I will have to fopen an existing C file and proceed with the calculation. I know that there maybe some builtin tools for finding it. But still I want it to be programmed manually. Is there any specific method to find the start and end of the various functions in a C file?
Run this through C preprocessor. This way you strip comments, unroll macros, include #includes etc. Unless you want complexity of the user-readable code, this will produce results much more true.
Remove fixed strings. Anything between "" goes, note escaped quote \" doesn't close the string.
Scan the file. First { increases count of functions and begins scanning the body of a function. Observe depth. { increases depth, } decreases, as depth reaches 0 another } is the end of the function. Next { will be a new function, but as you scan the outside, if before reaching next { or EOF you encounter a ; - cancel any data collected on the last piece. That wasn't a function, it was a struct, an union or something like that.
I would recommend a 2-pass approach.
Pass 1: Remove any open or close braces inside comments (and optionally those in preprocessor directives).
Pass 2: Count open and close braces and whenever they match up (#open == #close) a function ends. The next open brace denotes the start of a new function.
This approach is not fail-safe. It may fail if the code contains preprocessor statements that violate good programming practice. If you encounter such code you may want to run your tool on the code after it has passed through the preprocessor stage.
I finally found a nice way to do this!
doxygen already does a lot of things to process functions and other things nicely.
generate doxygen conf like doxygen -g doxygen_conf
open the conf file with your favorite editor and set GENERATE_XML = YES. You might also wanna set RECURSIVE = YES and others needed for your project, and run doxygen. set also INPUT = [PATH_TO_PROJECT_BASE].
In your doxygen build directory, you will find html/ and xml/.
cd80#cd80 ~/lab/VulnVizOnLinux/linux-5.4.109 » cd build_doc
cd80#cd80 ~/lab/VulnVizOnLinux/linux-5.4.109/build_doc » ls
ExtractFunctions.ipynb html xml
cd80#cd80 ~/lab/VulnVizOnLinux/linux-5.4.109/build_doc »
(ignore ExtractFunctions.ipynb, that's mine)
cd to xml and open any of xml files and analyze it for a while.
Here's how I did it.
import os
import xml.etree.ElementTree as ET
base_path = '/home/cd80/lab/VulnVizOnLinux/linux-5.4.109/'
open_files = {}
doc = ET.parse('/home/cd80/lab/VulnVizOnLinux/linux-5.4.109/build_doc/xml/4_2kernel_2module-plts_8c.xml')
root = doc.getroot()
for func in root.findall(".//memberdef/[#kind='function']"):
name = func.find('./name').text
location = func.find('./location')
if 'bodyend' not in location.keys():
continue # this memberdef is not a definition of function
bodystart = int(location.attrib.get('bodystart'))
bodyend = int(location.attrib.get('bodyend'))
file_path = location.attrib.get('file')
file_path = os.path.join(base_path, file_path)
if file_path not in open_files.keys():
with open(file_path, 'rb') as f:
code = f.read().decode('utf-8')
open_files[file_path] = code
else:
code = open_files[file_path]
func_def = '\n'.join(code.split("\n")[bodystart-1:bodyend])
print(func_def)
print('='*30)
Result:
static struct plt_entry __get_adrp_add_pair(u64 dst, u64 pc,
enum aarch64_insn_register reg)
{
u32 adrp, add;
adrp = aarch64_insn_gen_adr(pc, dst, reg, AARCH64_INSN_ADR_TYPE_ADRP);
add = aarch64_insn_gen_add_sub_imm(reg, reg, dst % SZ_4K,
AARCH64_INSN_VARIANT_64BIT,
AARCH64_INSN_ADSB_ADD);
return (struct plt_entry){ cpu_to_le32(adrp), cpu_to_le32(add) };
}
==============================
struct plt_entry get_plt_entry(u64 dst, void *pc)
{
struct plt_entry plt;
static u32 br;
if (!br)
br = aarch64_insn_gen_branch_reg(AARCH64_INSN_REG_16,
AARCH64_INSN_BRANCH_NOLINK);
plt = __get_adrp_add_pair(dst, (u64)pc, AARCH64_INSN_REG_16);
plt.br = cpu_to_le32(br);
return plt;
}
==============================
bool plt_entries_equal(const struct plt_entry *a, const struct plt_entry *b)
{
u64 p, q;
/*
* Check whether both entries refer to the same target:
* do the cheapest checks first.
* If the 'add' or 'br' opcodes are different, then the target
* cannot be the same.
*/
if (a->add != b->add || a->br != b->br)
return false;
p = ALIGN_DOWN((u64)a, SZ_4K);
q = ALIGN_DOWN((u64)b, SZ_4K);
/*
* If the 'adrp' opcodes are the same then we just need to check
* that they refer to the same 4k region.
*/
if (a->adrp == b->adrp && p == q)
return true;
return (p + aarch64_insn_adrp_get_offset(le32_to_cpu(a->adrp))) ==
(q + aarch64_insn_adrp_get_offset(le32_to_cpu(b->adrp)));
}
==============================
static bool in_init(const struct module *mod, void *loc)
{
return (u64)loc - (u64)mod->init_layout.base < mod->init_layout.size;
}
==============================
u64 module_emit_plt_entry(struct module *mod, Elf64_Shdr *sechdrs,
void *loc, const Elf64_Rela *rela,
Elf64_Sym *sym)
{
struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core :
&mod->arch.init;
struct plt_entry *plt = (struct plt_entry *)sechdrs[pltsec->plt_shndx].sh_addr;
int i = pltsec->plt_num_entries;
int j = i - 1;
u64 val = sym->st_value + rela->r_addend;
if (is_forbidden_offset_for_adrp(&plt[i].adrp))
i++;
plt[i] = get_plt_entry(val, &plt[i]);
/*
* Check if the entry we just created is a duplicate. Given that the
* relocations are sorted, this will be the last entry we allocated.
* (if one exists).
*/
if (j >= 0 && plt_entries_equal(plt + i, plt + j))
return (u64)&plt[j];
pltsec->plt_num_entries += i - j;
if (WARN_ON(pltsec->plt_num_entries > pltsec->plt_max_entries))
return 0;
return (u64)&plt[i];
}
==============================
static int cmp_rela(const void *a, const void *b)
{
const Elf64_Rela *x = a, *y = b;
int i;
/* sort by type, symbol index and addend */
i = cmp_3way(ELF64_R_TYPE(x->r_info), ELF64_R_TYPE(y->r_info));
if (i == 0)
i = cmp_3way(ELF64_R_SYM(x->r_info), ELF64_R_SYM(y->r_info));
if (i == 0)
i = cmp_3way(x->r_addend, y->r_addend);
return i;
}
==============================
static bool duplicate_rel(const Elf64_Rela *rela, int num)
{
/*
* Entries are sorted by type, symbol index and addend. That means
* that, if a duplicate entry exists, it must be in the preceding
* slot.
*/
return num > 0 && cmp_rela(rela + num, rela + num - 1) == 0;
}
==============================
static unsigned int count_plts(Elf64_Sym *syms, Elf64_Rela *rela, int num,
Elf64_Word dstidx, Elf_Shdr *dstsec)
{
unsigned int ret = 0;
Elf64_Sym *s;
int i;
for (i = 0; i < num; i++) {
u64 min_align;
switch (ELF64_R_TYPE(rela[i].r_info)) {
case R_AARCH64_JUMP26:
case R_AARCH64_CALL26:
if (!IS_ENABLED(CONFIG_RANDOMIZE_BASE))
break;
/*
* We only have to consider branch targets that resolve
* to symbols that are defined in a different section.
* This is not simply a heuristic, it is a fundamental
* limitation, since there is no guaranteed way to emit
* PLT entries sufficiently close to the branch if the
* section size exceeds the range of a branch
* instruction. So ignore relocations against defined
* symbols if they live in the same section as the
* relocation target.
*/
s = syms + ELF64_R_SYM(rela[i].r_info);
if (s->st_shndx == dstidx)
break;
/*
* Jump relocations with non-zero addends against
* undefined symbols are supported by the ELF spec, but
* do not occur in practice (e.g., 'jump n bytes past
* the entry point of undefined function symbol f').
* So we need to support them, but there is no need to
* take them into consideration when trying to optimize
* this code. So let's only check for duplicates when
* the addend is zero: this allows us to record the PLT
* entry address in the symbol table itself, rather than
* having to search the list for duplicates each time we
* emit one.
*/
if (rela[i].r_addend != 0 || !duplicate_rel(rela, i))
ret++;
break;
case R_AARCH64_ADR_PREL_PG_HI21_NC:
case R_AARCH64_ADR_PREL_PG_HI21:
if (!IS_ENABLED(CONFIG_ARM64_ERRATUM_843419) ||
!cpus_have_const_cap(ARM64_WORKAROUND_843419))
break;
/*
* Determine the minimal safe alignment for this ADRP
* instruction: the section alignment at which it is
* guaranteed not to appear at a vulnerable offset.
*
* This comes down to finding the least significant zero
* bit in bits [11:3] of the section offset, and
* increasing the section's alignment so that the
* resulting address of this instruction is guaranteed
* to equal the offset in that particular bit (as well
* as all less signficant bits). This ensures that the
* address modulo 4 KB != 0xfff8 or 0xfffc (which would
* have all ones in bits [11:3])
*/
min_align = 2ULL << ffz(rela[i].r_offset | 0x7);
/*
* Allocate veneer space for each ADRP that may appear
* at a vulnerable offset nonetheless. At relocation
* time, some of these will remain unused since some
* ADRP instructions can be patched to ADR instructions
* instead.
*/
if (min_align > SZ_4K)
ret++;
else
dstsec->sh_addralign = max(dstsec->sh_addralign,
min_align);
break;
}
}
if (IS_ENABLED(CONFIG_ARM64_ERRATUM_843419) &&
cpus_have_const_cap(ARM64_WORKAROUND_843419))
/*
* Add some slack so we can skip PLT slots that may trigger
* the erratum due to the placement of the ADRP instruction.
*/
ret += DIV_ROUND_UP(ret, (SZ_4K / sizeof(struct plt_entry)));
return ret;
}
==============================
int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
char *secstrings, struct module *mod)
{
unsigned long core_plts = 0;
unsigned long init_plts = 0;
Elf64_Sym *syms = NULL;
Elf_Shdr *pltsec, *tramp = NULL;
int i;
/*
* Find the empty .plt section so we can expand it to store the PLT
* entries. Record the symtab address as well.
*/
for (i = 0; i < ehdr->e_shnum; i++) {
if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt"))
mod->arch.core.plt_shndx = i;
else if (!strcmp(secstrings + sechdrs[i].sh_name, ".init.plt"))
mod->arch.init.plt_shndx = i;
else if (!strcmp(secstrings + sechdrs[i].sh_name,
".text.ftrace_trampoline"))
tramp = sechdrs + i;
else if (sechdrs[i].sh_type == SHT_SYMTAB)
syms = (Elf64_Sym *)sechdrs[i].sh_addr;
}
if (!mod->arch.core.plt_shndx || !mod->arch.init.plt_shndx) {
pr_err("%s: module PLT section(s) missing\n", mod->name);
return -ENOEXEC;
}
if (!syms) {
pr_err("%s: module symtab section missing\n", mod->name);
return -ENOEXEC;
}
for (i = 0; i < ehdr->e_shnum; i++) {
Elf64_Rela *rels = (void *)ehdr + sechdrs[i].sh_offset;
int numrels = sechdrs[i].sh_size / sizeof(Elf64_Rela);
Elf64_Shdr *dstsec = sechdrs + sechdrs[i].sh_info;
if (sechdrs[i].sh_type != SHT_RELA)
continue;
/* ignore relocations that operate on non-exec sections */
if (!(dstsec->sh_flags & SHF_EXECINSTR))
continue;
/* sort by type, symbol index and addend */
sort(rels, numrels, sizeof(Elf64_Rela), cmp_rela, NULL);
if (!str_has_prefix(secstrings + dstsec->sh_name, ".init"))
core_plts += count_plts(syms, rels, numrels,
sechdrs[i].sh_info, dstsec);
else
init_plts += count_plts(syms, rels, numrels,
sechdrs[i].sh_info, dstsec);
}
pltsec = sechdrs + mod->arch.core.plt_shndx;
pltsec->sh_type = SHT_NOBITS;
pltsec->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
pltsec->sh_addralign = L1_CACHE_BYTES;
pltsec->sh_size = (core_plts + 1) * sizeof(struct plt_entry);
mod->arch.core.plt_num_entries = 0;
mod->arch.core.plt_max_entries = core_plts;
pltsec = sechdrs + mod->arch.init.plt_shndx;
pltsec->sh_type = SHT_NOBITS;
pltsec->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
pltsec->sh_addralign = L1_CACHE_BYTES;
pltsec->sh_size = (init_plts + 1) * sizeof(struct plt_entry);
mod->arch.init.plt_num_entries = 0;
mod->arch.init.plt_max_entries = init_plts;
if (tramp) {
tramp->sh_type = SHT_NOBITS;
tramp->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
tramp->sh_addralign = __alignof__(struct plt_entry);
tramp->sh_size = sizeof(struct plt_entry);
}
return 0;
}
==============================
Dirty but works just as how I wanted

Explain this implementation of malloc from the K&R book

This is an excerpt from the book on C by Kernighan and Ritchie. It shows how to implement a version of malloc. Although well commented, I am having great difficulty in understanding it. Can somebody please explain it?
typedef long Align; /* for alignment to long boundary */
union header { /* block header */
struct {
union header *ptr; /* next block if on free list */
unsigned size; /* size of this block */
} s;
Align x; /* force alignment of blocks */
};
typedef union header Header;
static Header base; /* empty list to get started */
static Header *freep = NULL; /* start of free list */
/* malloc: general-purpose storage allocator */
void *malloc(unsigned nbytes)
{
Header *p, *prevp;
Header *morecore(unsigned);
unsigned nunits;
nunits = (nbytes+sizeof(Header)-1)/sizeof(header) + 1;
if ((prevp = freep) == NULL) { /* no free list yet */
base.s.ptr = freeptr = prevptr = &base;
base.s.size = 0;
}
for (p = prevp->s.ptr; ; prevp = p, p = p->s.ptr) {
if (p->s.size >= nunits) { /* big enough */
if (p->s.size == nunits) /* exactly */
prevp->s.ptr = p->s.ptr;
else { /* allocate tail end */
p->s.size -= nunits;
p += p->s.size;
p->s.size = nunits
}
freep = prevp;
return (void *)(p+1);
}
if (p == freep) /* wrapped around free list */
if ((p = morecore(nunits)) == NULL)
return NULL; /* none left */
}
}
#define NALLOC 1024 /* minimum #units to request */
/* morecore: ask system for more memory */
static Header *morecore(unsigned nu)
{
char *cp, *sbrk(int);
Header *up;
if (nu < NALLOC)
nu = NALLOC;
cp = sbrk(nu * sizeof(Header));
if (cp == (char *) -1) /* no space at all */
return NULL;
up = (Header *) cp;
up->s.size = nu;
free((void *)(up+1));
return freep;
}
/* free: put block ap in free list */
void free(void *ap) {
Header *bp, *p;
bp = (Header *)ap - 1; /* point to block header */
for (p = freep; !(bp > p && bp < p->s.ptr); p = p->s.ptr)
if (p >= p->s.ptr && (bp > p || bp < p->s.ptr))
break; /* freed block at start or end of arena */
if (bp + bp->size == p->s.ptr) {
bp->s.size += p->s.ptr->s.size;
bp->s.ptr = p->s.ptr->s.ptr;
} else
bp->s.ptr = p->s.ptr;
if (p + p->size == bp) {
p->s.size += bp->s.size;
p->s.ptr = bp->s.ptr;
} else
p->s.ptr = bp;
freep = p;
}
Ok, what we have here is a chunk of really poorly written code. What I will do in this post could best be described as software archaeology.
Step 1: fix the formatting.
The indention and compact format doesn't do anyone any good. Various spaces and empty rows need to be inserted. The comments could be written in more readable ways. I'll start by fixing that.
At the same time I'm changing the brace style from K&R style - please note that the K&R brace style is acceptable, this is merely a personal preference of mine. Another personal preference is to write the * for pointers next to the type pointed at. I'll not argue about (subjective) style matters here.
Also, the type definition of Header is completely unreadable, it needs a drastic fix.
And I spotted something completely obscure: they seem to have declared a function prototype inside the function. Header* morecore(unsigned);. This is very old and very poor style, and I'm not sure if C even allows it any longer. Lets just remove that line, whatever that function does, it will have to be defined elsewhere.
typedef long Align; /* for alignment to long boundary */
typedef union header /* block header */
{
struct
{
union header *ptr; /* next block if on free list */
unsigned size; /* size of this block */
} s;
Align x; /* force alignment of blocks */
} Header;
static Header base; /* empty list to get started */
static Header* freep = NULL; /* start of free list */
/* malloc: general-purpose storage allocator */
void* malloc (unsigned nbytes)
{
Header* p;
Header* prevp;
unsigned nunits;
nunits = (nbytes + sizeof(Header) - 1) / sizeof(header) + 1;
if ((prevp = freep) == NULL) /* no free list yet */
{
base.s.ptr = freeptr = prevptr = &base;
base.s.size = 0;
}
for (p = prevp->s.ptr; ; prevp = p, p = p->s.ptr)
{
if (p->s.size >= nunits) /* big enough */
{
if (p->s.size == nunits) /* exactly */
prevp->s.ptr = p->s.ptr;
else /* allocate tail end */
{
p->s.size -= nunits;
p += p->s.size;
p->s.size = nunits
}
freep = prevp;
return (void *)(p+1);
}
if (p == freep) /* wrapped around free list */
if ((p = morecore(nunits)) == NULL)
return NULL; /* none left */
}
}
Ok now we might actually be able to read the code.
Step 2: weed out widely-recognized bad practice.
This code is filled with things that are nowadays regarded as bad practice. They need to be removed, since they jeopardize the safety, readability and maintenance of the code. If you want a reference to an authority preaching the same practices as me, check out the widely-recognized coding standard MISRA-C.
I have spotted and removed the following bad practices:
1) Just typing unsigned in the code could lead to be confusion: was this a typo by the programmer or was the intention to write unsigned int? We should replace all unsigned with unsigned int. But as we do that, we find that it is used in this context to give the size of various binary data. The correct type to use for such matters is the C standard type size_t. This is essentially just an unsigned int as well, but it is guaranteed to be "large enough" for the particular platform. The sizeof operator returns a result of type size_t and if we look at the C standard's definition of the real malloc, it is void *malloc(size_t size);. So size_t is the most correct type to use.
2) It is a bad idea to use the same name for our own malloc function as the one residing in stdlib.h. Should we need to include stdlib.h, things will get messy. As a rule of thumb, never use identifier names of C standard library functions in your own code. I'll change the name to kr_malloc.
3) The code is abusing the fact that all static variables are guaranteed to be initialized to zero. This is well-defined by the C standard, but a rather subtle rule. Lets initialize all statics explicitly, to show that we haven't forgotten to init them by accident.
4) Assignment inside conditions is dangerous and hard to read. This should be avoided if possible, since it can also lead to bugs, such as the classic = vs == bug.
5) Multiple assignments on the same row is hard to read, and also possibly dangerous, because of the order of evaluation.
6) Multiple declarations on the same row is hard to read, and dangerous, since it could lead to bugs when mixing data and pointer declarations. Always declare each variable on a row of its own.
7) Always uses braces after every statement. Not doing so will lead to bugs bugs bugs.
8) Never type cast from a specific pointer type to void*. It is unnecessary in C, and could hide away bugs that the compiler would otherwise have detected.
9) Avoid using multiple return statements inside a function. Sometimes they lead to clearer code, but in most cases they lead to spaghetti. As the code stands, we can't change that without rewriting the loop though, so I will fix this later.
10) Keep for loops simple. They should contain one init statement, one loop condition and one iteration, nothing else. This for loop, with the comma operator and everything, is very obscure. Again, we spot a need to rewrite this loop into something sane. I'll do this next, but for now we have:
typedef long Align; /* for alignment to long boundary */
typedef union header /* block header */
{
struct
{
union header *ptr; /* next block if on free list */
size_t size; /* size of this block */
} s;
Align x; /* force alignment of blocks */
} Header;
static Header base = {0}; /* empty list to get started */
static Header* freep = NULL; /* start of free list */
/* malloc: general-purpose storage allocator */
void* kr_malloc (size_t nbytes)
{
Header* p;
Header* prevp;
size_t nunits;
nunits = (nbytes + sizeof(Header) - 1) / sizeof(header) + 1;
prevp = freep;
if (prevp == NULL) /* no free list yet */
{
base.s.ptr = &base;
freeptr = &base;
prevptr = &base;
base.s.size = 0;
}
for (p = prevp->s.ptr; ; prevp = p, p = p->s.ptr)
{
if (p->s.size >= nunits) /* big enough */
{
if (p->s.size == nunits) /* exactly */
{
prevp->s.ptr = p->s.ptr;
}
else /* allocate tail end */
{
p->s.size -= nunits;
p += p->s.size;
p->s.size = nunits
}
freep = prevp;
return p+1;
}
if (p == freep) /* wrapped around free list */
{
p = morecore(nunits);
if (p == NULL)
{
return NULL; /* none left */
}
}
} /* for */
}
Step 3: rewrite the obscure loop.
For the reasons mentioned earlier. We can see that this loop goes on forever, it terminates by returning from the function, either when the allocation is done, or when there is no memory left. So lets create that as a loop condition, and lift out the return to the end of the function where it should be. And lets get rid of that ugly comma operator.
I'll introduce two new variables: one result variable to hold the resulting pointer, and another to keep track of whether the loop should continue or not. I'll blow K&R's minds by using the bool type, which is part of the C language since 1999.
(I hope I haven't altered the algorithm with this change, I believe I haven't)
#include <stdbool.h>
typedef long Align; /* for alignment to long boundary */
typedef union header /* block header */
{
struct
{
union header *ptr; /* next block if on free list */
size_t size; /* size of this block */
} s;
Align x; /* force alignment of blocks */
} Header;
static Header base = {0}; /* empty list to get started */
static Header* freep = NULL; /* start of free list */
/* malloc: general-purpose storage allocator */
void* kr_malloc (size_t nbytes)
{
Header* p;
Header* prevp;
size_t nunits;
void* result;
bool is_allocating;
nunits = (nbytes + sizeof(Header) - 1) / sizeof(header) + 1;
prevp = freep;
if (prevp == NULL) /* no free list yet */
{
base.s.ptr = &base;
freeptr = &base;
prevptr = &base;
base.s.size = 0;
}
is_allocating = true;
for (p = prevp->s.ptr; is_allocating; p = p->s.ptr)
{
if (p->s.size >= nunits) /* big enough */
{
if (p->s.size == nunits) /* exactly */
{
prevp->s.ptr = p->s.ptr;
}
else /* allocate tail end */
{
p->s.size -= nunits;
p += p->s.size;
p->s.size = nunits
}
freep = prevp;
result = p+1;
is_allocating = false; /* we are done */
}
if (p == freep) /* wrapped around free list */
{
p = morecore(nunits);
if (p == NULL)
{
result = NULL; /* none left */
is_allocating = false;
}
}
prevp = p;
} /* for */
return result;
}
Step 4: make this crap compile.
Since this is from K&R, it is filled with typos. sizeof(header) should be sizeof(Header). There are missing semi-colons. They use different names freep, prevp versus freeptr, prevptr, but clearly mean the same variable. I believe the latter were actually better names, so lets use those.
#include <stdbool.h>
typedef long Align; /* for alignment to long boundary */
typedef union header /* block header */
{
struct
{
union header *ptr; /* next block if on free list */
size_t size; /* size of this block */
} s;
Align x; /* force alignment of blocks */
} Header;
static Header base = {0}; /* empty list to get started */
static Header* freeptr = NULL; /* start of free list */
/* malloc: general-purpose storage allocator */
void* kr_malloc (size_t nbytes)
{
Header* p;
Header* prevptr;
size_t nunits;
void* result;
bool is_allocating;
nunits = (nbytes + sizeof(Header) - 1) / sizeof(Header) + 1;
prevptr = freeptr;
if (prevptr == NULL) /* no free list yet */
{
base.s.ptr = &base;
freeptr = &base;
prevptr = &base;
base.s.size = 0;
}
is_allocating = true;
for (p = prevptr->s.ptr; is_allocating; p = p->s.ptr)
{
if (p->s.size >= nunits) /* big enough */
{
if (p->s.size == nunits) /* exactly */
{
prevptr->s.ptr = p->s.ptr;
}
else /* allocate tail end */
{
p->s.size -= nunits;
p += p->s.size;
p->s.size = nunits;
}
freeptr = prevptr;
result = p+1;
is_allocating = false; /* we are done */
}
if (p == freeptr) /* wrapped around free list */
{
p = morecore(nunits);
if (p == NULL)
{
result = NULL; /* none left */
is_allocating = false;
}
}
prevptr = p;
} /* for */
return result;
}
And now we have somewhat readable, maintainable code, without numerous dangerous practices, that will even compile! So now we could actually start to ponder about what the code is actually doing.
The struct "Header" is, as you might have guessed, the declaration of a node in a linked list. Each such node contains a pointer to the next one. I don't quite understand the morecore function, nor the "wrap-around", I have never used this function, nor sbrk. But I assume that it allocates a header as specified in this struct, and also some chunk of raw data following that header. If so, that explains why there is no actual data pointer: the data is assumed to follow the header, adjacently in memory. So for each node, we get the header, and we get a chunk of raw data following the header.
The iteration itself is pretty straight-forward, they are going through a single-linked list, one node at a time.
At the end of the loop, they set the pointer to point one past the end of the "chunk", then store that in a static variable, so that the program will remember where it previously allocated memory, next time the function is called.
They are using a trick to make their header end up on an aligned memory address: they store all the overhead info in a union together with a variable large enough to correspond to the platform's alignment requirement. So if the size of "ptr" plus the size of "size" are too small to give the exact alignment, the union guarantees that at least sizeof(Align) bytes are allocated. I believe that this whole trick is obsolete today, since the C standard mandates automatic struct/union padding.
I'm studying K&R as I'd imagine OP was when he asked this question, and I came here because I also found these implementations to be confusing. While the accepted answer is very detailed and helpful, I tried to take a different tack which was to understand the code as it was originally written - I've gone through the code and added comments to the sections of the code that were difficult to me. This includes code for the other routines in the section (which are the functions free and memcore - I've renamed them kandr_malloc and kandr_free to avoid conflicts with the stdlib). I thought I would leave this here as a supplement to the accepted answer, for other students who may find it helpful.
I acknowledge that the comments in this code are excessive. Please know that I am only doing this as a learning exercise and I am not proposing that this is a good way to actually write code.
I took the liberty of changing some variable names to ones that seemed more intuitive to me; other than that the code is essentially left intact. It seems to compile and run fine for the test programs that I used, although valgrind had complaints for some applications.
Also: some of the text in the comments is lifted directly from K&R or the man pages - I do not intend to take any credit for these sections.
#include <unistd.h> // sbrk
#define NALLOC 1024 // Number of block sizes to allocate on call to sbrk
#ifdef NULL
#undef NULL
#endif
#define NULL 0
// long is chosen as an instance of the most restrictive alignment type
typedef long Align;
/* Construct Header data structure. To ensure that the storage returned by
* kandr_malloc is aligned properly for the objects that are stored in it, all
* blocks are multiples of the header size, and the header itself is aligned
* properly. This is achieved through the use of a union; this data type is big
* enough to hold the "widest" member, and the alignment is appropriate for all
* of the types in the union. Thus by including a member of type Align, which
* is an instance of the most restrictive type, we guarantee that the size of
* Header is aligned to the worst-case boundary. The Align field is never used;
* it just forces each header to the desired alignment.
*/
union header {
struct {
union header *next;
unsigned size;
} s;
Align x;
};
typedef union header Header;
static Header base; // Used to get an initial member for free list
static Header *freep = NULL; // Free list starting point
static Header *morecore(unsigned nblocks);
void kandr_free(void *ptr);
void *kandr_malloc(unsigned nbytes) {
Header *currp;
Header *prevp;
unsigned nunits;
/* Calculate the number of memory units needed to provide at least nbytes of
* memory.
*
* Suppose that we need n >= 0 bytes and that the memory unit sizes are b > 0
* bytes. Then n / b (using integer division) yields one less than the number
* of units needed to provide n bytes of memory, except in the case that n is
* a multiple of b; then it provides exactly the number of units needed. It
* can be verified that (n - 1) / b provides one less than the number of units
* needed to provide n bytes of memory for all values of n > 0. Thus ((n - 1)
* / b) + 1 provides exactly the number of units needed for n > 0.
*
* The extra sizeof(Header) in the numerator is to include the unit of memory
* needed for the header itself.
*/
nunits = ((nbytes + sizeof(Header) - 1) / sizeof(Header)) + 1;
// case: no free list yet exists; we have to initialize.
if (freep == NULL) {
// Create degenerate free list; base points to itself and has size 0
base.s.next = &base;
base.s.size = 0;
// Set free list starting point to base address
freep = &base;
}
/* Initialize pointers to two consecutive blocks in the free list, which we
* call prevp (the previous block) and currp (the current block)
*/
prevp = freep;
currp = prevp->s.next;
/* Step through the free list looking for a block of memory large enough to
* fit nunits units of memory into. If the whole list is traversed without
* finding such a block, then morecore is called to request more memory from
* the OS.
*/
for (; ; prevp = currp, currp = currp->s.next) {
/* case: found a block of memory in free list large enough to fit nunits
* units of memory into. Partition block if necessary, remove it from the
* free list, and return the address of the block (after moving past the
* header).
*/
if (currp->s.size >= nunits) {
/* case: block is exactly the right size; remove the block from the free
* list by pointing the previous block to the next block.
*/
if (currp->s.size == nunits) {
/* Note that this line wouldn't work as intended if we were down to only
* 1 block. However, we would never make it here in that scenario
* because the block at &base has size 0 and thus the conditional will
* fail (note that nunits is always >= 1). It is true that if the block
* at &base had combined with another block, then previous statement
* wouldn't apply - but presumably since base is a global variable and
* future blocks are allocated on the heap, we can be sure that they
* won't border each other.
*/
prevp->s.next = currp->s.next;
}
/* case: block is larger than the amount of memory asked for; allocate
* tail end of the block to the user.
*/
else {
// Changes the memory stored at currp to reflect the reduced block size
currp->s.size -= nunits;
// Find location at which to create the block header for the new block
currp += currp->s.size;
// Store the block size in the new header
currp->s.size = nunits;
}
/* Set global starting position to the previous pointer. Next call to
* malloc will start either at the remaining part of the partitioned block
* if a partition occurred, or at the block after the selected block if
* not.
*/
freep = prevp;
/* Return the location of the start of the memory, i.e. after adding one
* so as to move past the header
*/
return (void *) (currp + 1);
} // end found a block of memory in free list case
/* case: we've wrapped around the free list without finding a block large
* enough to fit nunits units of memory into. Call morecore to request that
* at least nunits units of memory are allocated.
*/
if (currp == freep) {
/* morecore returns freep; the reason that we have to assign currp to it
* again (since we just tested that they are equal), is that there is a
* call to free inside of morecore that can potentially change the value
* of freep. Thus we reassign it so that we can be assured that the newly
* added block is found before (currp == freep) again.
*/
if ((currp = morecore(nunits)) == NULL) {
return NULL;
}
} // end wrapped around free list case
} // end step through free list looking for memory loop
}
static Header *morecore(unsigned nunits) {
void *freemem; // The address of the newly created memory
Header *insertp; // Header ptr for integer arithmatic and constructing header
/* Obtaining memory from OS is a comparatively expensive operation, so obtain
* at least NALLOC blocks of memory and partition as needed
*/
if (nunits < NALLOC) {
nunits = NALLOC;
}
/* Request that the OS increment the program's data space. sbrk changes the
* location of the program break, which defines the end of the process's data
* segment (i.e., the program break is the first location after the end of the
* uninitialized data segment). Increasing the program break has the effect
* of allocating memory to the process. On success, brk returns the previous
* break - so if the break was increased, then this value is a pointer to the
* start of the newly allocated memory.
*/
freemem = sbrk(nunits * sizeof(Header));
// case: unable to allocate more memory; sbrk returns (void *) -1 on error
if (freemem == (void *) -1) {
return NULL;
}
// Construct new block
insertp = (Header *) freemem;
insertp->s.size = nunits;
/* Insert block into the free list so that it is available for malloc. Note
* that we add 1 to the address, effectively moving to the first position
* after the header data, since of course we want the block header to be
* transparent for the user's interactions with malloc and free.
*/
kandr_free((void *) (insertp + 1));
/* Returns the start of the free list; recall that freep has been set to the
* block immediately preceeding the newly allocated memory (by free). Thus by
* returning this value the calling function can immediately find the new
* memory by following the pointer to the next block.
*/
return freep;
}
void kandr_free(void *ptr) {
Header *insertp, *currp;
// Find address of block header for the data to be inserted
insertp = ((Header *) ptr) - 1;
/* Step through the free list looking for the position in the list to place
* the insertion block. In the typical circumstances this would be the block
* immediately to the left of the insertion block; this is checked for by
* finding a block that is to the left of the insertion block and such that
* the following block in the list is to the right of the insertion block.
* However this check doesn't check for one such case, and misses another. We
* still have to check for the cases where either the insertion block is
* either to the left of every other block owned by malloc (the case that is
* missed), or to the right of every block owned by malloc (the case not
* checked for). These last two cases are what is checked for by the
* condition inside of the body of the loop.
*/
for (currp = freep; !((currp < insertp) && (insertp < currp->s.next)); currp = currp->s.next) {
/* currp >= currp->s.ptr implies that the current block is the rightmost
* block in the free list. Then if the insertion block is to the right of
* that block, then it is the new rightmost block; conversely if it is to
* the left of the block that currp points to (which is the current leftmost
* block), then the insertion block is the new leftmost block. Note that
* this conditional handles the case where we only have 1 block in the free
* list (this case is the reason that we need >= in the first test rather
* than just >).
*/
if ((currp >= currp->s.next) && ((currp < insertp) || (insertp < currp->s.next))) {
break;
}
}
/* Having found the correct location in the free list to place the insertion
* block, now we have to (i) link it to the next block, and (ii) link the
* previous block to it. These are the tasks of the next two if/else pairs.
*/
/* case: the end of the insertion block is adjacent to the beginning of
* another block of data owned by malloc. Absorb the block on the right into
* the block on the left (i.e. the previously existing block is absorbed into
* the insertion block).
*/
if ((insertp + insertp->s.size) == currp->s.next) {
insertp->s.size += currp->s.next->s.size;
insertp->s.next = currp->s.next->s.next;
}
/* case: the insertion block is not left-adjacent to the beginning of another
* block of data owned by malloc. Set the insertion block member to point to
* the next block in the list.
*/
else {
insertp->s.next = currp->s.next;
}
/* case: the end of another block of data owned by malloc is adjacent to the
* beginning of the insertion block. Absorb the block on the right into the
* block on the left (i.e. the insertion block is absorbed into the preceeding
* block).
*/
if ((currp + currp->s.size) == insertp) {
currp->s.size += insertp->s.size;
currp->s.next = insertp->s.next;
}
/* case: the insertion block is not right-adjacent to the end of another block
* of data owned by malloc. Set the previous block in the list to point to
* the insertion block.
*/
else {
currp->s.next = insertp;
}
/* Set the free pointer list to start the block previous to the insertion
* block. This makes sense because calls to malloc start their search for
* memory at the next block after freep, and the insertion block has as good a
* chance as any of containing a reasonable amount of memory since we've just
* added some to it. It also coincides with calls to morecore from
* kandr_malloc because the next search in the iteration looks at exactly the
* right memory block.
*/
freep = currp;
}
The basic of malloc()
In Linux, there are two typical ways to request memory: sbrk and mmap. These system calls have severe limitations on frequent small allocations. malloc() is a library function to address this issue. It requests large chunks of memory with sbrk/mmap and returns small memory blocks inside large chunks. This is much more efficient and flexible than directly calling sbrk/mmap.
K&R malloc()
In the K&R implementation, a core (more commonly called arena) is a large chunk of memory. morecore() requests a core from system via sbrk(). When you call malloc()/free() multiple times, some blocks in the cores are used/allocated while others are free. K&R malloc stores the addresses of free blocks in a circular single linked list. In this list, each node is a block of free memory. The first sizeof(Header) bytes keep the size of the block and the pointer to the next free block. The rest of bytes in the free block are uninitialized. Different from typical lists in textbooks, nodes in the free list are just pointers to some unused areas in cores; you don't actually allocate each node except for cores. This list is the key to the understanding of the algorithm.
The following diagram shows an example memory layout with two cores/arenas. In the diagram, each character takes sizeof(Header) bytes. # is a Header, + marks allocated memory and - marks free memory inside cores. In the example, there are three allocated blocks and three free blocks. The three free blocks are stored in the circular list. For the three allocated blocks, only their sizes are stored in Header.
This is core 1 This is core 2
#---------#+++++++++#++++++++++++ #----------#+++++++++++++++++#------------
| | |
p->ptr->ptr p = p->ptr->ptr->ptr p->ptr
In your code, freep is an entry point to the free list. If you repeatedly follow freep->ptr, you will come back to freep – it is circular. Once you understand the circular single-linked list, the rest is relatively easy. malloc() finds a free block and possibly splits it. free() adds a free block back to the list and may merge it to adjacent free blocks. They both try to maintain the structure of the list.
Other comments on the implementation
The code comments mentioned "wrapped around" in malloc(). That line happens when you have traversed the entire free list but can't find a free block larger than the requested length. In this case, you have to add a new core with morecore().
base is a zero-sized block that is always included in the free list. It is a trick to avoid special casing. It is not strictly necessary.
free() may look a little complex because it has to consider four different cases to merge a newly freed block to other free blocks in the list. This detail is not that important unless you want to reimplement by yourself.
This blog post explains K&R malloc in more details.
PS: K&R malloc is one of the most elegant pieces of code in my view. It was really eye opening when I first understood the code. It makes me sad that some modern programmers, not even understanding the basic of this implementation, are calling the masterpiece crap solely based on its coding style.
I also found this exercise great and interesting.
In my opinion visualizing the structure may help a lot with understanding the logic - or at least this worked for me. Below is my code, which prints as much as possible about the flow of the K&R malloc.
The most significant change I made in the K&R malloc is the change of 'free' to make sure some old pointer will not be used again.
Other than that I added comments and fixed some small typos.
Experimenting with NALLOC, MAXMEM and the test variables in 'main' could be also of help.
On my computer (Ubuntu 16.04.3) this compiled without errors with:
gcc -g -std=c99 -Wall -Wextra -pedantic-errors krmalloc.c
krmalloc.c :
#include <stdio.h>
#include <unistd.h>
typedef long Align; /* for alignment to long boundary */
union header { /* block header */
struct {
union header *ptr; /* next block if on free list */
size_t size; /* size of this block */
/* including the Header itself */
/* measured in count of Header chunks */
/* not less than NALLOC Header's */
} s;
Align x; /* force alignment of blocks */
};
typedef union header Header;
static Header *morecore(size_t);
void *mmalloc(size_t);
void _mfree(void **);
void visualize(const char*);
size_t getfreem(void);
size_t totmem = 0; /* total memory in chunks */
static Header base; /* empty list to get started */
static Header *freep = NULL; /* start of free list */
#define NALLOC 1 /* minimum chunks to request */
#define MAXMEM 2048 /* max memory available (in bytes) */
#define mfree(p) _mfree((void **)&p)
void *sbrk(__intptr_t incr);
int main(void)
{
char *pc, *pcc, *pccc, *ps;
long *pd, *pdd;
int dlen = 100;
int ddlen = 50;
visualize("start");
/* trying to fragment as much as possible to get a more interesting view */
/* claim a char */
if ((pc = (char *) mmalloc(sizeof(char))) == NULL)
return -1;
/* claim a string */
if ((ps = (char *) mmalloc(dlen * sizeof(char))) == NULL)
return -1;
/* claim some long's */
if ((pd = (long *) mmalloc(ddlen * sizeof(long))) == NULL)
return -1;
/* claim some more long's */
if ((pdd = (long *) mmalloc(ddlen * 2 * sizeof(long))) == NULL)
return -1;
/* claim one more char */
if ((pcc = (char *) mmalloc(sizeof(char))) == NULL)
return -1;
/* claim the last char */
if ((pccc = (char *) mmalloc(sizeof(char))) == NULL)
return -1;
/* free and visualize */
printf("\n");
mfree(pccc);
/* bugged on purpose to test free(NULL) */
mfree(pccc);
visualize("free(the last char)");
mfree(pdd);
visualize("free(lot of long's)");
mfree(ps);
visualize("free(string)");
mfree(pd);
visualize("free(less long's)");
mfree(pc);
visualize("free(first char)");
mfree(pcc);
visualize("free(second char)");
/* check memory condition */
size_t freemem = getfreem();
printf("\n");
printf("--- Memory claimed : %ld chunks (%ld bytes)\n",
totmem, totmem * sizeof(Header));
printf(" Free memory now : %ld chunks (%ld bytes)\n",
freemem, freemem * sizeof(Header));
if (freemem == totmem)
printf(" No memory leaks detected.\n");
else
printf(" (!) Leaking memory: %ld chunks (%ld bytes).\n",
(totmem - freemem), (totmem - freemem) * sizeof(Header));
printf("// Done.\n\n");
return 0;
}
/* visualize: print the free list (educational purpose) */
void visualize(const char* msg)
{
Header *tmp;
printf("--- Free list after \"%s\":\n", msg);
if (freep == NULL) { /* does not exist */
printf("\tList does not exist\n\n");
return;
}
if (freep == freep->s.ptr) { /* self-pointing list = empty */
printf("\tList is empty\n\n");
return;
}
printf(" ptr: %10p size: %-3lu --> ", (void *) freep, freep->s.size);
tmp = freep; /* find the start of the list */
while (tmp->s.ptr > freep) { /* traverse the list */
tmp = tmp->s.ptr;
printf("ptr: %10p size: %-3lu --> ", (void *) tmp, tmp->s.size);
}
printf("end\n\n");
}
/* calculate the total amount of available free memory */
size_t getfreem(void)
{
if (freep == NULL)
return 0;
Header *tmp;
tmp = freep;
size_t res = tmp->s.size;
while (tmp->s.ptr > tmp) {
tmp = tmp->s.ptr;
res += tmp->s.size;
}
return res;
}
/* mmalloc: general-purpose storage allocator */
void *mmalloc(size_t nbytes)
{
Header *p, *prevp;
size_t nunits;
/* smallest count of Header-sized memory chunks */
/* (+1 additional chunk for the Header itself) needed to hold nbytes */
nunits = (nbytes + sizeof(Header) - 1) / sizeof(Header) + 1;
/* too much memory requested? */
if (((nunits + totmem + getfreem())*sizeof(Header)) > MAXMEM) {
printf("Memory limit overflow!\n");
return NULL;
}
if ((prevp = freep) == NULL) { /* no free list yet */
/* set the list to point to itself */
base.s.ptr = freep = prevp = &base;
base.s.size = 0;
}
/* traverse the circular list */
for (p = prevp->s.ptr; ; prevp = p, p = p->s.ptr) {
if (p->s.size >= nunits) { /* big enough */
if (p->s.size == nunits) /* exactly */
prevp->s.ptr = p->s.ptr;
else { /* allocate tail end */
/* adjust the size */
p->s.size -= nunits;
/* find the address to return */
p += p->s.size;
p->s.size = nunits;
}
freep = prevp;
return (void *)(p+1);
}
/* back where we started and nothing found - we need to allocate */
if (p == freep) /* wrapped around free list */
if ((p = morecore(nunits)) == NULL)
return NULL; /* none left */
}
}
/* morecore: ask system for more memory */
/* nu: count of Header-chunks needed */
static Header *morecore(size_t nu)
{
char *cp;
Header *up;
/* get at least NALLOC Header-chunks from the OS */
if (nu < NALLOC)
nu = NALLOC;
cp = (char *) sbrk(nu * sizeof(Header));
if (cp == (char *) -1) /* no space at all */
return NULL;
printf("... (sbrk) claimed %ld chunks.\n", nu);
totmem += nu; /* keep track of allocated memory */
up = (Header *) cp;
up->s.size = nu;
/* add the free space to the circular list */
void *n = (void *)(up+1);
mfree(n);
return freep;
}
/* mfree: put block ap in free list */
void _mfree(void **ap)
{
if (*ap == NULL)
return;
Header *bp, *p;
bp = (Header *)*ap - 1; /* point to block header */
if (bp->s.size == 0 || bp->s.size > totmem) {
printf("_mfree: impossible value for size\n");
return;
}
/* the free space is only marked as free, but 'ap' still points to it */
/* to avoid reusing this address and corrupt our structure set it to '\0' */
*ap = NULL;
/* look where to insert the free space */
/* (bp > p && bp < p->s.ptr) => between two nodes */
/* (p > p->s.ptr) => this is the end of the list */
/* (p == p->p.str) => list is one element only */
for (p = freep; !(bp > p && bp < p->s.ptr); p = p->s.ptr)
if (p >= p->s.ptr && (bp > p || bp < p->s.ptr))
/* freed block at start or end of arena */
break;
if (bp + bp->s.size == p->s.ptr) { /* join to upper nbr */
/* the new block fits perfect up to the upper neighbor */
/* merging up: adjust the size */
bp->s.size += p->s.ptr->s.size;
/* merging up: point to the second next */
bp->s.ptr = p->s.ptr->s.ptr;
} else
/* set the upper pointer */
bp->s.ptr = p->s.ptr;
if (p + p->s.size == bp) { /* join to lower nbr */
/* the new block fits perfect on top of the lower neighbor */
/* merging below: adjust the size */
p->s.size += bp->s.size;
/* merging below: point to the next */
p->s.ptr = bp->s.ptr;
} else
/* set the lower pointer */
p->s.ptr = bp;
/* reset the start of the free list */
freep = p;
}

problem using rebar to compile c source

I tried to use rebar to compile an iconv port extracted from ejabberd (http://www.ejabberd.im/), it successfully generated an "iconv_erl.so".
but when I use erl_ddll:load_driver("priv/", iconv_erl)
to load it, it returns {error,{open_error,-10}}.
Is there anything I am missing? thx in advance.
p.s. The c source is as follows:
/*
* ejabberd, Copyright (C) 2002-2010 ProcessOne
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
* 02111-1307 USA
*
*/
#include <stdio.h>
#include <string.h>
#include <erl_driver.h>
#include <ei.h>
#include <iconv.h>
typedef struct {
ErlDrvPort port;
iconv_t cd;
} iconv_data;
static ErlDrvData iconv_erl_start(ErlDrvPort port, char *buff)
{
iconv_data* d = (iconv_data*)driver_alloc(sizeof(iconv_data));
d->port = port;
d->cd = NULL;
set_port_control_flags(port, PORT_CONTROL_FLAG_BINARY);
return (ErlDrvData)d;
}
static void iconv_erl_stop(ErlDrvData handle)
{
driver_free((char*)handle);
}
static int iconv_erl_control(ErlDrvData drv_data,
unsigned int command,
char *buf, int len,
char **rbuf, int rlen)
{
int i;
int size;
int index = 0;
int avail;
size_t inleft, outleft;
ErlDrvBinary *b;
char *from, *to, *string, *stmp, *rstring, *rtmp;
iconv_t cd;
int invalid_utf8_as_latin1 = 0;
ei_decode_version(buf, &index, &i);
ei_decode_tuple_header(buf, &index, &i);
ei_get_type(buf, &index, &i, &size);
from = driver_alloc(size + 1);
ei_decode_string(buf, &index, from);
ei_get_type(buf, &index, &i, &size);
to = driver_alloc(size + 1);
ei_decode_string(buf, &index, to);
ei_get_type(buf, &index, &i, &size);
stmp = string = driver_alloc(size + 1);
ei_decode_string(buf, &index, string);
/* Special mode: parse as UTF-8 if possible; otherwise assume it's
Latin-1. Makes no difference when encoding. */
if (strcmp(from, "utf-8+latin-1") == 0) {
from[5] = '\0';
invalid_utf8_as_latin1 = 1;
}
if (strcmp(to, "utf-8+latin-1") == 0) {
to[5] = '\0';
}
cd = iconv_open(to, from);
if (cd == (iconv_t) -1) {
cd = iconv_open("ascii", "ascii");
if (cd == (iconv_t) -1) {
*rbuf = (char*)(b = driver_alloc_binary(size));
memcpy(b->orig_bytes, string, size);
driver_free(from);
driver_free(to);
driver_free(string);
return size;
}
}
outleft = avail = 4*size;
inleft = size;
rtmp = rstring = driver_alloc(avail);
while (inleft > 0) {
if (iconv(cd, &stmp, &inleft, &rtmp, &outleft) == (size_t) -1) {
if (invalid_utf8_as_latin1 && (*stmp & 0x80) && outleft >= 2) {
/* Encode one byte of (assumed) Latin-1 into two bytes of UTF-8 */
*rtmp++ = 0xc0 | ((*stmp & 0xc0) >> 6);
*rtmp++ = 0x80 | (*stmp & 0x3f);
outleft -= 2;
}
stmp++;
inleft--;
}
}
size = rtmp - rstring;
*rbuf = (char*)(b = driver_alloc_binary(size));
memcpy(b->orig_bytes, rstring, size);
driver_free(from);
driver_free(to);
driver_free(string);
driver_free(rstring);
iconv_close(cd);
return size;
}
ErlDrvEntry iconv_driver_entry = {
NULL, /* F_PTR init, N/A */
iconv_erl_start, /* L_PTR start, called when port is opened */
iconv_erl_stop, /* F_PTR stop, called when port is closed */
NULL, /* F_PTR output, called when erlang has sent */
NULL, /* F_PTR ready_input, called when input descriptor ready */
NULL, /* F_PTR ready_output, called when output descriptor ready */
"iconv_erl", /* char *driver_name, the argument to open_port */
NULL, /* F_PTR finish, called when unloaded */
NULL, /* handle */
iconv_erl_control, /* F_PTR control, port_command callback */
NULL, /* F_PTR timeout, reserved */
NULL /* F_PTR outputv, reserved */
};
DRIVER_INIT(iconv_erl) /* must match name in driver_entry */
{
return &iconv_driver_entry;
}
When you get an error from erl_ddll:load_driver/2, try passing the error to the erl_ddll:format_error/1 function to get more details. For example, I'm currently seeing the same error you are — {open_error, -10} — so I pass that to erl_ddll:format_error/1 in an Erlang shell, as shown below:
1> erl_ddll:format_error({open_error,-10}).
"undefined symbol: _ZTVN10__cxxabiv117__class_type_infoE"
The output shows that my driver won't load because it has an undefined symbol.

Resources