How to implement a dynamic–grow stack with pthread? - c

Question

From APUE, I learned about guardsize and stackaddr of a stack. If a rsp pointer is going to be lower than stackaddr, the rsp enters the guard stack area, and a signal emits to notify the program.
I am wondering if it is possible to implement dynamic–grow(which grows dynamically) stack using this feature. Can you show
how?

Answer 1

The stack can be grown "dynamically"
Set up an alternate signal stack.
Set up signal handler for SIGSEGV with SA_ONSTACK to use the alternate stack.
When the program runs out of stack, it will be issued a SIGSEGV signal.
The signal handler can use getrlimit/setrlimit with RLIMIT_STACK to change the stack size.
getrlimit and setrlimit are not explicitly mentioned in man signal-safety but I don't see why they would be a problem in a signal handler.
Here is a sample test program. It is annotated. Invoke with -d to dynamically increase the stack size from within the signal handler:
// stacktest.c -- test dynamically growing stack
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <signal.h>
#include <assert.h>
#include <setjmp.h>
#include <sys/time.h>
#include <sys/resource.h>
volatile int opt_t = 0;
volatile int opt_d = 0;
volatile int opt_i = 0;
void *top; // initial top of stack
// signal handler
volatile int may_grow = 1; // grow stack within signal handler
volatile int err1 = 0; // error on get
volatile int err2 = 0; // error on set
volatile int hitno; // number of signals
jmp_buf jbuf;
// alternate signal stack
unsigned char altstk[64 * 4096] __attribute__((aligned(4096)));
// xprtstr -- print string in signal handler
void
xprtstr(const char *str)
{
size_t len = strlen(str);
write(1,str,len);
}
// xprtstr -- print number in signal handler
void
xprtnum(unsigned long val,const char *sym)
{
static const char *hex = "0123456789ABCDEF";
xprtstr(" ");
xprtstr(sym);
xprtstr("=");
char buf[100];
char *bp = &buf[50];
*bp-- = 0;
for (int idx = 0; idx < 16; ++idx, --bp) {
*bp = hex[val & 0x0F];
val >>= 4;
}
++bp;
xprtstr(bp);
}
// sigfault -- SIGSEGV handler
void
sigfault(int signo,siginfo_t *info,void *vp)
{
++hitno;
xprtstr("ISR hit");
xprtnum(signo,"signo");
xprtnum(hitno,"hitno");
xprtstr("\n");
struct rlimit rlim;
do {
if (opt_t) {
xprtstr("ISR test\n");
siglongjmp(jbuf,1);
break;
}
// dynamically grow the stack
if (may_grow) {
xprtstr("ISR grow\n");
may_grow = 0;
err1 = getrlimit(RLIMIT_STACK,&rlim);
xprtnum(rlim.rlim_cur,"rlim_cur");
xprtnum(rlim.rlim_cur / 1024,"rlim_cur");
xprtstr("\n");
rlim.rlim_cur += 8 * 1024 * 1024;
err2 = setrlimit(RLIMIT_STACK,&rlim);
getrlimit(RLIMIT_STACK,&rlim);
xprtnum(rlim.rlim_cur,"rlim_cur");
xprtnum(rlim.rlim_cur / 1024,"rlim_cur");
xprtstr("\n");
break;
}
// stop the program
xprtstr("ISR stop\n");
siglongjmp(jbuf,2);
} while (0);
}
// loop -- recursive function to overflow stack
void
loop(unsigned char *old)
{
// get some space on the stack
unsigned char cur[4096];
// get current stack frame address
void *frame = __builtin_frame_address(0);
// get amount of space used on stack
size_t dif = top - frame;
// show where we are
printf("loop: top=%p frame=%p dif=%8.8zX/%zu may_grow=%d\n",
top,frame,dif,dif / 1024,may_grow);
// keep consuming more stack
// NOTE: we don't actually use cur/old but ensure that the compiler won't
// optimize it away
loop(cur);
}
#define SHOWFLAGS(_msk) \
if (flags & (_msk)) \
printf(" " #_msk)
void
showsa(const struct sigaction *sa,const char *who)
{
int flags = sa->sa_flags;
// show the signal flags
printf("showflags: %s flags=%8.8X",who,flags);
SHOWFLAGS(SA_NOCLDSTOP);
SHOWFLAGS(SA_NOCLDWAIT);
SHOWFLAGS(SA_NODEFER);
SHOWFLAGS(SA_ONSTACK);
SHOWFLAGS(SA_RESETHAND);
SHOWFLAGS(SA_RESTART);
SHOWFLAGS(SA_SIGINFO);
// show the signal mask
for (int signo = 1; signo < 32; ++signo) {
if (sigismember(&sa->sa_mask,signo))
printf(" S%d",signo);
}
printf("\n");
}
// setup_signal -- set up signal handler
void
setup_signal(int signo)
{
struct sigaction sa;
assert(sigaction(signo,NULL,&sa) == 0);
sa.sa_sigaction = sigfault;
showsa(&sa,"BEF");
sa.sa_flags |= SA_SIGINFO;
sa.sa_flags |= SA_ONSTACK;
sa.sa_flags &= ~SA_RESETHAND;
assert(sigaction(signo,&sa,NULL) == 0);
assert(sigaction(signo,NULL,&sa) == 0);
showsa(&sa,"AFT");
}
// setup_altstack -- set up alternate signal stack
void
setup_altstack(void)
{
stack_t ss;
sigaltstack(NULL,&ss);
printf("setup_altstack: ss_sp=%p ss_flags=%8.8X ss_size=%zu\n",
ss.ss_sp,ss.ss_flags,ss.ss_size);
void *sp = altstk;
sp += 16 * 4096;
printf("altstk=%p sp=%p\n",altstk,sp);
ss.ss_sp = sp;
ss.ss_size = sizeof(altstk) / 2;
ss.ss_flags = SS_ONSTACK;
assert(sigaltstack(&ss,NULL) == 0);
sigaltstack(NULL,&ss);
printf("setup_altstack: ss_sp=%p ss_flags=%8.8X ss_size=%zu\n",
ss.ss_sp,ss.ss_flags,ss.ss_size);
if (ss.ss_sp != sp) {
printf("setup_altstack: failed\n");
exit(99);
}
}
#define SHOWOPT(_opt,_reason) \
if (_opt) \
printf(#_opt " -- " _reason)
int
main(int argc,char **argv)
{
setlinebuf(stdout);
--argc;
++argv;
for (; argc > 0; --argc, ++argv) {
char *cp = *argv;
if (*cp != '-')
break;
cp += 2;
switch (cp[-1]) {
case 'd':
opt_d = ! opt_d;
break;
case 'i':
opt_i = ! opt_i;
break;
case 't':
opt_t = ! opt_t;
break;
}
}
// show the options
SHOWOPT(opt_d,"dynamically grow stack within signal handler");
SHOWOPT(opt_i,"grow stack initially");
SHOWOPT(opt_t,"test sigsetjmp/siglongjmp");
// set up alternate signal stack and signal handler
if (opt_d || opt_t) {
setup_altstack();
setup_signal(SIGSEGV);
}
// test our sigsetjmp/siglongjmp
if (opt_t) {
for (int try = 1; try <= 2; ++try) {
if (! sigsetjmp(jbuf,1)) {
printf("main: ptr try=%d\n",try);
unsigned int *ptr = NULL;
*ptr = 23;
}
else
printf("main: resume try=%d\n",try);
}
opt_t = 0;
}
// set up large stack outside of signal handler
if (opt_i) {
struct rlimit rlim;
getrlimit(RLIMIT_STACK,&rlim);
printf("rlim_cur=%lu/%lu\n",rlim.rlim_cur,rlim.rlim_cur / 1024);
rlim.rlim_cur = 32 * 1024 * 1024;
setrlimit(RLIMIT_STACK,&rlim);
getrlimit(RLIMIT_STACK,&rlim);
printf("rlim_cur=%lu/%lu\n",rlim.rlim_cur,rlim.rlim_cur / 1024);
}
printf("main: loop\n");
// dynamically grow the stack
may_grow = opt_d;
// top of stack
top = __builtin_frame_address(0);
if (! sigsetjmp(jbuf,1))
loop(NULL);
else
printf("main: stop\n");
return 0;
}
UPDATE:
This describes how to make "the" stack grow dynamically (very interesting!), but I take the question to be about the stacks of threads other than the initial one, or at least inclusive of those stacks. I don't think this answer addresses them. – 
John Bollinger
Although the question was tagged with pthreads, I'm not sure if OP actually was talking about subthreads doing this.
I'm not sure that this can be done for subthreads (created via pthread_create). At least not by using a default pthread_create call.
Without special pthread_attr_t values, the default for pthread_create is to malloc a stack [of a default size]. With attributes, the caller might set a larger stack size. And/or the caller will [usually] do an explicit malloc and pass the address along (with the size).
AFAICT, from reading glibc source, pthread_create will not set up a guard area if the user provides a stack pointer. Setting up the guard area [if there is to be one] is the responsibility of the caller.
But, in either case, if the stack overflows (with a guard area), a signal will be generated (SIGSEGV ?).
But, what can one do at that point???
The [pthread internal] function that calls the user's start_routine, the start_routine itself, and any functions that the start routine has called, already have pointers to things on that "old" stack [we must assume this].
So, the per-thread stack can not be moved (i.e. no realloc).
The only way to [possibly] do this is for the caller to provide an explicit stack pointer [and size]. As mentioned, the caller must set up the guard pages (via mprotect, I assume).
Although userfaultfd et. al. might be usable/preferable, I'm going to assume that the caller must use an explicit mmap call (vs. using the heap malloc/realloc).
The main thread's stack [generally] grows downward from the top of the virtual memory space. It can grow until all physical memory is used up and the paging disk is full.
However, for a per-thread (subthread) stack, the caller must decide on the maximum per-thread stack size before creating the thread. More on this below.
It can set up a stack (via mmap) of a smaller initial size. Once thread is created, the base/top stack address must remain constant.
Edit: The following is modified by additional thoughts below.
If a fault occurs, the signal handler could try an extension of the stack. There may be a few ways to do this. My best guess for this is:
It may have to copy/save the existing stack data [somewhere ;-)] if the remap below does not copy the existing data the way realloc does.
Temporarily undo any guard pages.
unmap/remap the stack at the same address but with a larger size (via mremap and/or mmap using MREMAP_FIXED/MAP_FIXED).
If necessary, copy back the stack data onto the "new" stack.
Set up new guard pages.
return from the signal handler [and hope ;-)].
As I said, subthread stacks can't grow "infinitely" as the main thread stack can (via setrlimit).
The sizes/addresses in the example below are not "to scale" ...
Consider two threads (e.g. tA and tB) that start with a small size:
tA's stack at xxxx1000 with size 1000
tB's stack at xxxx2000 with size 1000
If tB hits its guard page and faults, what happens? It has no room the extend its stack downward without colliding with the top of tA's stack.
So, we must map the stacks with sufficient space to grow to the "maximum" and we must know that beforehand.
We need to "space out" the stack addresses so they have room to grow [even if not all pages are mapped initially].
Let's assume that the "maximum" size is 10000. A mapping that would work is:
tA's stack at xxxx1000 with size 1000
tB's stack at xxx11000 with size 1000
Now, if tB hits its guard page, it can extend its stack up to the maximum of 10000
Additional thoughts ...
We probably must initially map the entire maximum region. Otherwise, other unrelated mmap calls may grab space in the proposed extension area. Either mmap from the heap manager, shared memory mappings, or even mmap calls done when setting up other threads.
So, we don't need to use mremap to ever increase the size of the region. Rather, we should use madvise with MADV_DONTNEED on the area that we're not currently using.
If we do this, we don't need a signal handler or guard pages to invoke it to extend the area. Just doing MADV_DONTNEED will keep resource usage low(er).
The thread function can (after popping the stack a bit) release the "popped" area with MADV_DONTNEED.
Other MADV_* options might be better.
All of the above strikes me as craziness! The only use would be a [hugely] multithreaded app that is doing deeply recursive functions.
But, if that's the case, it would be better to convert the recursive functions into ones that don't use actual program stack based recursion. Rather, the function manages an array of structs as a software controlled [pseudo] "stack". Each struct has all the variables that were function/automatic scope in the program stack frame.
The size of this array can be controlled with realloc. Or, we can implement the "stack" as a linked list of these structs with a "slab" allocation scheme. With the slab scheme, all threads could share the same slab allocator. This has the advantage that the amount of memory required can be smaller than if each thread had its own/private slab allocator.
So, IMO, don't do this on the real, per-thread program stack!
Okay, so that's what I've come up with. I've not written code for it or tested it. That's an exercise I think I'll leave to the reader ;-)