How to get fullstacktrace using _Unwind_Backtrace on SIGSEGV - c

I handle SIGSEGV by code:
int C()
{
int *i = NULL;
*i = 10; // Crash there
}
int B()
{
return C();
}
int A()
{
return B();
}
int main(void)
{
struct sigaction handler;
memset(&handler,0,sizeof(handler));
handler.sa_sigaction = handler_func;
handler.sa_flags = SA_SIGINFO;
sigaction(SIGSEGV,&handler,NULL);
return(C());
}
Where handler code are:
static int handler_func(int signal, siginfo_t info, void* rserved)
{
const void* stack[MAX_DEPTH];
StackCrowlState state;
state.addr = stack;
state.count = MAX_DEPTH;
_Unwind_Reason_Code code = _Unwind_Backtrace(trace_func,&state);
printf("Stack trace count: %d, code: %d\n",MAX_DEPTH - state.count, code);
kill(getpid(),SIGKILL);
}
static _Unwind_Reason_Code trace_func(void* context, void* arg)
{
StackCrowlState *state = (StackCrowlState *)arg;
if(state->count>0)
{
void *ip = (void *)_Unwind_GetIP(context);
if(ip)
{
state->addr[0] = ip;
state->count--;
state->addr++;
}
}
return(_URC_NO_REASON);
}
But trace_func where called only once, and shows only on _Unwind_Backtrace calls. Is it possible to get stacktrace of code which cause SIGSEGV signal using _Unwind_Backtrace?
thnx

If you want to use particularly _Unwind_Context(), you can do it like this (the code is 32-bit ARM specific):
struct BacktraceState {
const ucontext_t* signal_ucontext;
size_t address_count = 0;
static const size_t address_count_max = 30;
uintptr_t addresses[address_count_max] = {};
BacktraceState(const ucontext_t* ucontext) : signal_ucontext(ucontext) {}
bool AddAddress(uintptr_t ip) {
// No more space in the storage. Fail.
if (address_count >= address_count_max)
return false;
// Reset the Thumb bit, if it is set.
const uintptr_t thumb_bit = 1;
ip &= ~thumb_bit;
// Ignore null addresses.
// They sometimes happen when using _Unwind_Backtrace()
// with the compiler optimizations,
// when the Link Register is overwritten by the inner
// stack frames.
if (ip == 0)
return true;
// Ignore duplicate addresses.
// They sometimes happen when using _Unwind_Backtrace()
// with the compiler optimizations,
// because we both add the second address from the Link Register
// in ProcessRegisters() and receive the same address
// in UnwindBacktraceCallback().
if (address_count > 0 && ip == addresses[address_count - 1])
return true;
// Finally add the address to the storage.
addresses[address_count++] = ip;
return true;
}
};
void ProcessRegisters(
_Unwind_Context* unwind_context, BacktraceState* state) {
assert(state);
assert(unwind_context);
const ucontext_t* signal_ucontext = state->signal_ucontext;
assert(signal_ucontext);
const sigcontext* signal_mcontext = &(signal_ucontext->uc_mcontext);
assert(signal_mcontext);
_Unwind_SetGR(unwind_context, REG_R0, signal_mcontext->arm_r0);
_Unwind_SetGR(unwind_context, REG_R1, signal_mcontext->arm_r1);
_Unwind_SetGR(unwind_context, REG_R2, signal_mcontext->arm_r2);
_Unwind_SetGR(unwind_context, REG_R3, signal_mcontext->arm_r3);
_Unwind_SetGR(unwind_context, REG_R4, signal_mcontext->arm_r4);
_Unwind_SetGR(unwind_context, REG_R5, signal_mcontext->arm_r5);
_Unwind_SetGR(unwind_context, REG_R6, signal_mcontext->arm_r6);
_Unwind_SetGR(unwind_context, REG_R7, signal_mcontext->arm_r7);
_Unwind_SetGR(unwind_context, REG_R8, signal_mcontext->arm_r8);
_Unwind_SetGR(unwind_context, REG_R9, signal_mcontext->arm_r9);
_Unwind_SetGR(unwind_context, REG_R10, signal_mcontext->arm_r10);
_Unwind_SetGR(unwind_context, REG_R11, signal_mcontext->arm_fp);
_Unwind_SetGR(unwind_context, REG_R12, signal_mcontext->arm_ip);
_Unwind_SetGR(unwind_context, REG_R13, signal_mcontext->arm_sp);
_Unwind_SetGR(unwind_context, REG_R14, signal_mcontext->arm_lr);
_Unwind_SetGR(unwind_context, REG_R15, signal_mcontext->arm_pc);
// Program Counter register aka Instruction Pointer will contain
// the address of the instruction where the crash happened.
// UnwindBacktraceCallback() will not supply us with it.
state->AddAddress(signal_mcontext->arm_pc);
// UnwindBacktraceCallback() does not always supply us with
// the return address of the frame where the crash happened.
// Sometimes Link Register will contain this address
// (noticed when compiling with Clang without optimization),
// but LR may also contain address of some previously visitied frame
// (noticed when compiling with GCC without optimization),
// or LR may contain null address
// (noticed when compiling with Clang with optimization).
// These heuristics are unreliable.
#if __clang__
state->AddAddress(signal_mcontext->arm_lr);
#endif
}
_Unwind_Reason_Code UnwindBacktraceCallback(
struct _Unwind_Context* unwind_context, void* state_voidp) {
assert(unwind_context);
assert(state_voidp);
BacktraceState* state = (BacktraceState*)state_voidp;
assert(state);
// On the first UnwindBacktraceCallback() call,
// set registers to _Unwind_Context and BacktraceState.
if (state->address_count == 0) {
ProcessRegisters(unwind_context, state);
return _URC_NO_REASON;
}
uintptr_t ip = _Unwind_GetIP(unwind_context);
bool ok = state->AddAddress(ip);
if (!ok)
return _URC_END_OF_STACK;
return _URC_NO_REASON;
}
void CaptureBacktrace(BacktraceState* state) {
assert(state);
_Unwind_Backtrace(UnwindBacktraceCallback, state);
}
void SigActionHandler(int sig, siginfo_t* info, void* ucontext) {
const ucontext_t* signal_ucontext = (const ucontext_t*)ucontext;
assert(signal_ucontext);
BacktraceState backtrace_state(signal_ucontext);
CaptureBacktrace(&backtrace_state);
// Do something with the backtrace - print, save to file, etc.
}
But I am advising you to not use _Unwind_Context(), but instead use precompiled libunwind for 32-bit ARM, bundled with modern Android NDKs (at sources/cxx-stl/llvm-libc++/libs/armeabi-v7a/libunwind.a) and with all LLVM. You will have to use use libc++ (LLVM STL). How to do it, is demonstrated in this my answer, you'd have to combine the examples here.
https://stackoverflow.com/a/50027799/1016580
If you use libstdc++ (GNU STL), you could use the Dar Hoo's solution:
https://stackoverflow.com/a/48593413/1016580

You want to backtrace from the signal triggering function, but you backtrace from the signal handler function. That's two different stacks. (Note, the SA_ONSTACK flag in sigaction is irrelevant to your question.)
To find the stack pointer of the of the triggering function, use the third parameter of the handler, i.e. void *rserved. You can reference to the answer in this question: Getting the saved instruction pointer address from a signal handler

better you use backtrace and backtrace_symbols_fd to get a stacktrace from a signal handler.

You may use __gnu_Unwind_Backtrace instead. Example for ARM32:
typedef struct
{
uintptr_t r[16];
} core_regs;
typedef struct
{
uintptr_t demand_save_flags;
core_regs core;
} phase2_vrs;
extern "C" _Unwind_Reason_Code __gnu_Unwind_Backtrace(_Unwind_Trace_Fn trace, void * trace_argument, phase2_vrs * entry_vrs);
int AndroidGetBackTraceWithContext(VOID **stack, UINT32 size, ucontext_t *ctx)
{
ANDROID_UNWIND_STATE state;
state.count = size;
state.stack = stack;
// First call stack is current pc
state.stack[0] = (VOID *)ctx->uc_mcontext.arm_pc;
state.stack++;
state.count--;
phase2_vrs pre_signal_state;
pre_signal_state.demand_save_flags = 0;
pre_signal_state.core = *reinterpret_cast<const core_regs*>(&(ctx->uc_mcontext.arm_r0));
// Return value is of no use and might be wrong on some systems
__gnu_Unwind_Backtrace(DmpAndroidUnwindCallback, &state, &pre_signal_state);
return size - state.count;
}

Related

How to get module image name from an arbitrary address in Windows kernel space?

I'm trying to see how I can get a loaded module image name from an arbitrary address from the kernel code.
In user mode I would do this:
void* pAddr;
VOID* pBase;
WCHAR buff[MAX_PATH] = {0};
//Get address of some function in some module (just to test it)
pAddr = GetProcAddress(GetModuleHandle(L"kernel32.dll"), "GetCurrentProcess");
//Get module base address
RtlPcToFileHeader(pAddr, &pBase);
//Get module image file name
GetModuleFileNameEx(GetCurrentProcess(), (HMODULE)pBase, buff, SIZEOF(buff));
Is there a way to do the same in kernel mode if I have pAddr that can point to some address in kernel or user space?
EDIT: While waiting for the answer I came up with my own code (using undocumented way of traversing PEB):
#ifdef CALLING_FROM_KERNEL_MODE
//Kernel mode
TEB* pTEB = (TEB*)PsGetCurrentThreadTeb();
#else
//User mode
#if defined(_M_X64)
//64-bit
TEB* pTEB = reinterpret_cast<TEB*>(__readgsqword(reinterpret_cast<DWORD_PTR>(&static_cast<NT_TIB*>(nullptr)->Self)));
#else
//32-bit
TEB* pTEB = reinterpret_cast<TEB*>(__readfsdword(reinterpret_cast<DWORD_PTR>(&static_cast<NT_TIB*>(nullptr)->Self)));
#endif
#endif
PEB* p_PEB = pTEB->ProcessEnvironmentBlock;
PEB_LDR_DATA* pPLD = p_PEB->Ldr;
const WCHAR* pModName = NULL;
LIST_ENTRY* pLE = &pPLD->InMemoryOrderModuleList;
LIST_ENTRY* pLE_Head = pLE;
while(pLE_Head != pLE->Flink)
{
PLDR_DATA_TABLE_ENTRY pLDTE = CONTAINING_RECORD(pLE, LDR_DATA_TABLE_ENTRY, InMemoryOrderLinks);
size_t szcbSizeOfImg = (size_t)pLDTE->Reserved3[1];
if((size_t)pAddr - (size_t)pLDTE->DllBase < szcbSizeOfImg)
{
pModName = pLDTE->FullDllName.Buffer;
break;
}
pLE = pLE->Flink;
}
The problem is that although it works from a user-mode, from a kernel mode PsGetCurrentThreadTeb() seems to return NULL. Does this mean kernel threads do not have a TEB?
this can be done by creating list of all loaded modules via ZwQuerySystemInformation with SystemModuleInformation
void fgt(PVOID *Callers, ULONG Count)
{
NTSTATUS status;
ULONG cb = 0x10000;
do
{
status = STATUS_INSUFFICIENT_RESOURCES;
if (PRTL_PROCESS_MODULES prpm = (PRTL_PROCESS_MODULES)ExAllocatePool(PagedPool, cb))
{
if (0 <= (status = NtQuerySystemInformation(SystemModuleInformation, prpm, cb, &cb)))
{
do
{
PVOID Caller = *Callers++;
if (ULONG NumberOfModules = prpm->NumberOfModules)
{
PRTL_PROCESS_MODULE_INFORMATION Modules = prpm->Modules;
do
{
if ((SIZE_T)Caller - (SIZE_T)Modules->ImageBase < Modules->ImageSize)
{
DbgPrint("%p> %s\n", Caller, Modules->FullPathName);
break;
}
} while (Modules++, --NumberOfModules);
}
} while (--Count);
}
ExFreePool(prpm);
}
} while (status == STATUS_INFO_LENGTH_MISMATCH);
}

CreateThread wrapper function

I am currently working on a project where we have a C thread implementation for UNIX systems using pthreads. Now we want to be able to run this entire project on Windows as well, and I am translating all the threading for WIN32. Now I encountered a problem for which I could not come up with a decent solution.
I have the thrd_create() function:
static inline int thrd_create(thrd_t *thr, thrd_start_t func, void *arg) {
Args* args = malloc(sizeof(Args));
args->arg = arg;
args->function = func;
*thr = CreateThread(NULL, 0, wrapper_function, (LPVOID) args, 0, NULL);
if (!*thr) {
free (args);
return thrd_error;
}
return thrd_success;
}
This function is supposed to create a new thread, and the user provides a start function. For convenience, I would like to leave the implementation that calls thrd_create() untouched if possible. For this reason, I created a wrapper_function:
static inline DWORD wrapper_function(LPVOID arg) {
Args * args;
args = (Args*) arg;
DWORD res = args->function(args->arg); //This does obviously not work
return res;
}
My question is: What DWORD should my wrapper function return? The function provided by the user for the pthread implementation has void return type, so I won't get any result from that. Any suggestions?
EDIT
Args looks like this:
struct Args {
void (*function)(void * aArg);
void* arg;
};
typedef struct Args Args;
According to manuals it is better to stick to a correct signature and use return value:
Windows
Pthreads
The other matter of concern would be the lifetime of args, I'd say the best way is for a caller to clean up, so they need to be tracked with your thread until it terminates.
An approximate API could be something along the lines of the following:
/* Your general error codes enumeration
* which should probably reside in a general
* header
*/
typedef enum {
OK = 0,
// Your application specific error codes
} error_t;
#ifdef _WIN32
#include <Windows.h>
typedef HANDLE thread_handle_t;
#else // assume pthreads
#include <pthread.h>
typedef pthread_t thread_handle_t;
#endif
typedef error_t(*entry_func_t)(void*);
typedef struct {
entry_func_t func;
void *args;
error_t _result;
thread_handle_t _handle;
} thread_t;
// returns OK(0) on success
// returns error code indicating a problem
error_t thread_create(thread_t *t);
An aproximate implementation would be:
#ifdef _WIN32
DWORD _win_entry_f(void *args) {
thread_t *t = args;
t->_result = t->func(t->args);
return 0; // Or some other Windows-specific value
}
error_t thread_create(thread_t *t) {
error_t err = OK;
if(!(t->_handle = ThreadCreate(NULL, 0, _win_entry_f, t, 0, NULL))) {
switch (GetLastError()) {
// Populate error with code
}
}
return err;
}
#else
void * _pthread_entry_f(void *args) {
thread_t *t = args;
t->_result = t->func(t->args);
return NULL; // Or some other pthreads specific value
}
error_t thread_create(thread_t *t, entry_func_t func, void *args) {
error_t err = OK;
switch(pthread_create(&t->_handle, NULL, _pthread_entry_f, t)) {
case 0: break;
// other cases populate err
}
return err;
}
#endif
Invokation would look somewhat like this.
error_t func(void* args) {
return OK;
}
.....................
thread_t t = { .func = func, .args = NULL };
thread_create(&t);
Obviously you'll need to implement your own cancelation, result collection, join, ...

Linux DMA: Using the DMAengine for scatter-gather transactions

I try to use the DMAengine API from a custom kernel driver to perform a scatter-gather operation. I have a contiguous memory region as source and I want to copy its data in several distributed buffers through a scatterlist structure. The DMA controller is the PL330 one that supports the DMAengine API (see PL330 DMA controller).
My test code is the following:
In my driver header file (test_driver.h):
#ifndef __TEST_DRIVER_H__
#define __TEST_DRIVER_H__
#include <linux/platform_device.h>
#include <linux/device.h>
#include <linux/scatterlist.h>
#include <linux/dma-mapping.h>
#include <linux/dmaengine.h>
#include <linux/of_dma.h>
#define SG_ENTRIES 3
#define BUF_SIZE 16
#define DEV_BUF 0x10000000
struct dma_block {
void * data;
int size;
};
struct dma_private_info {
struct sg_table sgt;
struct dma_block * blocks;
int nblocks;
int dma_started;
struct dma_chan * dma_chan;
struct dma_slave_config dma_config;
struct dma_async_tx_descriptor * dma_desc;
dma_cookie_t cookie;
};
struct test_platform_device {
struct platform_device * pdev;
struct dma_private_info dma_priv;
};
#define _get_devp(tdev) (&((tdev)->pdev->dev))
#define _get_dmapip(tdev) (&((tdev)->dma_priv))
int dma_stop(struct test_platform_device * tdev);
int dma_start(struct test_platform_device * tdev);
int dma_start_block(struct test_platform_device * tdev);
int dma_init(struct test_platform_device * tdev);
int dma_exit(struct test_platform_device * tdev);
#endif
In my source that contains the dma functions (dma_functions.c):
#include <linux/slab.h>
#include "test_driver.h"
#define BARE_RAM_BASE 0x10000000
#define BARE_RAM_SIZE 0x10000000
struct ram_bare {
uint32_t * __iomem map;
uint32_t base;
uint32_t size;
};
static void dma_sg_check(struct test_platform_device * tdev)
{
struct dma_private_info * dma_priv = _get_dmapip(tdev);
struct device * dev = _get_devp(tdev);
uint32_t * buf;
unsigned int bufsize;
int nwords;
int nbytes_word = sizeof(uint32_t);
int nblocks;
struct ram_bare ramb;
uint32_t * p;
int i;
int j;
ramb.map = ioremap(BARE_RAM_BASE,BARE_RAM_SIZE);
ramb.base = BARE_RAM_BASE;
ramb.size = BARE_RAM_SIZE;
dev_info(dev,"nblocks: %d \n",dma_priv->nblocks);
p = ramb.map;
nblocks = dma_priv->nblocks;
for( i = 0 ; i < nblocks ; i++ ) {
buf = (uint32_t *) dma_priv->blocks[i].data;
bufsize = dma_priv->blocks[i].size;
nwords = dma_priv->blocks[i].size/nbytes_word;
dev_info(dev,"block[%d],size %d: ",i,bufsize);
for ( j = 0 ; j < nwords; j++, p++) {
dev_info(dev,"DMA: 0x%x, RAM: 0x%x",buf[j],ioread32(p));
}
}
iounmap(ramb.map);
}
static int dma_sg_exit(struct test_platform_device * tdev)
{
struct dma_private_info * dma_priv = _get_dmapip(tdev);
int ret = 0;
int i;
for( i = 0 ; i < dma_priv->nblocks ; i++ ) {
kfree(dma_priv->blocks[i].data);
}
kfree(dma_priv->blocks);
sg_free_table(&(dma_priv->sgt));
return ret;
}
int dma_stop(struct test_platform_device * tdev)
{
struct dma_private_info * dma_priv = _get_dmapip(tdev);
struct device * dev = _get_devp(tdev);
int ret = 0;
dma_unmap_sg(dev,dma_priv->sgt.sgl,\
dma_priv->sgt.nents, DMA_FROM_DEVICE);
dma_sg_exit(tdev);
dma_priv->dma_started = 0;
return ret;
}
static void dma_callback(void * param)
{
enum dma_status dma_stat;
struct test_platform_device * tdev = (struct test_platform_device *) param;
struct dma_private_info * dma_priv = _get_dmapip(tdev);
struct device * dev = _get_devp(tdev);
dev_info(dev,"Checking the DMA state....\n");
dma_stat = dma_async_is_tx_complete(dma_priv->dma_chan,\
dma_priv->cookie, NULL, NULL);
if(dma_stat == DMA_COMPLETE) {
dev_info(dev,"DMA complete! \n");
dma_sg_check(tdev);
dma_stop(tdev);
} else if (unlikely(dma_stat == DMA_ERROR)) {
dev_info(dev,"DMA error! \n");
dma_stop(tdev);
}
}
static void dma_busy_loop(struct test_platform_device * tdev)
{
struct dma_private_info * dma_priv = _get_dmapip(tdev);
struct device * dev = _get_devp(tdev);
enum dma_status status;
int status_change = -1;
do {
status = dma_async_is_tx_complete(dma_priv->dma_chan, dma_priv->cookie, NULL, NULL);
switch(status) {
case DMA_COMPLETE:
if(status_change != 0)
dev_info(dev,"DMA status: COMPLETE\n");
status_change = 0;
break;
case DMA_PAUSED:
if (status_change != 1)
dev_info(dev,"DMA status: PAUSED\n");
status_change = 1;
break;
case DMA_IN_PROGRESS:
if(status_change != 2)
dev_info(dev,"DMA status: IN PROGRESS\n");
status_change = 2;
break;
case DMA_ERROR:
if (status_change != 3)
dev_info(dev,"DMA status: ERROR\n");
status_change = 3;
break;
default:
dev_info(dev,"DMA status: UNKNOWN\n");
status_change = -1;
break;
}
} while(status != DMA_COMPLETE);
dev_info(dev,"DMA transaction completed! \n");
}
static int dma_sg_init(struct test_platform_device * tdev)
{
struct dma_private_info * dma_priv = _get_dmapip(tdev);
struct scatterlist *sg;
int ret = 0;
int i;
ret = sg_alloc_table(&(dma_priv->sgt), SG_ENTRIES, GFP_ATOMIC);
if(ret)
goto out_mem2;
dma_priv->nblocks = SG_ENTRIES;
dma_priv->blocks = (struct dma_block *) kmalloc(dma_priv->nblocks\
*sizeof(struct dma_block), GFP_ATOMIC);
if(dma_priv->blocks == NULL)
goto out_mem1;
for( i = 0 ; i < dma_priv->nblocks ; i++ ) {
dma_priv->blocks[i].size = BUF_SIZE;
dma_priv->blocks[i].data = kmalloc(dma_priv->blocks[i].size, GFP_ATOMIC);
if(dma_priv->blocks[i].data == NULL)
goto out_mem3;
}
for_each_sg(dma_priv->sgt.sgl, sg, dma_priv->sgt.nents, i)
sg_set_buf(sg,dma_priv->blocks[i].data,dma_priv->blocks[i].size);
return ret;
out_mem3:
i--;
while(i >= 0)
kfree(dma_priv->blocks[i].data);
kfree(dma_priv->blocks);
out_mem2:
sg_free_table(&(dma_priv->sgt));
out_mem1:
ret = -ENOMEM;
return ret;
}
static int _dma_start(struct test_platform_device * tdev,int block)
{
struct dma_private_info * dma_priv = _get_dmapip(tdev);
struct device * dev = _get_devp(tdev);
int ret = 0;
int sglen;
/* Step 1: Allocate and initialize the SG list */
dma_sg_init(tdev);
/* Step 2: Map the SG list */
sglen = dma_map_sg(dev,dma_priv->sgt.sgl,\
dma_priv->sgt.nents, DMA_FROM_DEVICE);
if(! sglen)
goto out2;
/* Step 3: Configure the DMA */
(dma_priv->dma_config).direction = DMA_DEV_TO_MEM;
(dma_priv->dma_config).src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
(dma_priv->dma_config).src_maxburst = 1;
(dma_priv->dma_config).src_addr = (dma_addr_t) DEV_BUF;
dmaengine_slave_config(dma_priv->dma_chan, \
&(dma_priv->dma_config));
/* Step 4: Prepare the SG descriptor */
dma_priv->dma_desc = dmaengine_prep_slave_sg(dma_priv->dma_chan, \
dma_priv->sgt.sgl, dma_priv->sgt.nents, DMA_DEV_TO_MEM, \
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (dma_priv->dma_desc == NULL) {
dev_err(dev,"DMA could not assign a descriptor! \n");
goto out1;
}
/* Step 5: Set the callback method */
(dma_priv->dma_desc)->callback = dma_callback;
(dma_priv->dma_desc)->callback_param = (void *) tdev;
/* Step 6: Put the DMA descriptor in the queue */
dma_priv->cookie = dmaengine_submit(dma_priv->dma_desc);
/* Step 7: Fires the DMA transaction */
dma_async_issue_pending(dma_priv->dma_chan);
dma_priv->dma_started = 1;
if(block)
dma_busy_loop(tdev);
return ret;
out1:
dma_stop(tdev);
out2:
ret = -1;
return ret;
}
int dma_start(struct test_platform_device * tdev) {
return _dma_start(tdev,0);
}
int dma_start_block(struct test_platform_device * tdev) {
return _dma_start(tdev,1);
}
int dma_init(struct test_platform_device * tdev)
{
int ret = 0;
struct dma_private_info * dma_priv = _get_dmapip(tdev);
struct device * dev = _get_devp(tdev);
dma_priv->dma_chan = dma_request_slave_channel(dev, \
"dma_chan0");
if (dma_priv->dma_chan == NULL) {
dev_err(dev,"DMA channel busy! \n");
ret = -1;
}
dma_priv->dma_started = 0;
return ret;
}
int dma_exit(struct test_platform_device * tdev)
{
int ret = 0;
struct dma_private_info * dma_priv = _get_dmapip(tdev);
if(dma_priv->dma_started) {
dmaengine_terminate_all(dma_priv->dma_chan);
dma_stop(tdev);
dma_priv->dma_started = 0;
}
if(dma_priv->dma_chan != NULL)
dma_release_channel(dma_priv->dma_chan);
return ret;
}
In my driver source file (test_driver.c):
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/version.h>
#include <linux/device.h>
#include <linux/platform_device.h>
#include <linux/of_device.h>
#include <linux/of_address.h>
#include <linux/of_irq.h>
#include <linux/interrupt.h>
#include "test_driver.h"
static int dma_block=0;
module_param_named(dma_block, dma_block, int, 0444);
static struct test_platform_device tdev;
static struct of_device_id test_of_match[] = {
{ .compatible = "custom,test-driver-1.0", },
{}
};
static int test_probe(struct platform_device *op)
{
int ret = 0;
struct device * dev = &(op->dev);
const struct of_device_id *match = of_match_device(test_of_match, &op->dev);
if (!match)
return -EINVAL;
tdev.pdev = op;
dma_init(&tdev);
if(dma_block)
ret = dma_start_block(&tdev);
else
ret = dma_start(&tdev);
if(ret) {
dev_err(dev,"Error to start DMA transaction! \n");
} else {
dev_info(dev,"DMA OK! \n");
}
return ret;
}
static int test_remove(struct platform_device *op)
{
dma_exit(&tdev);
return 0;
}
static struct platform_driver test_platform_driver = {
.probe = test_probe,
.remove = test_remove,
.driver = {
.name = "test-driver",
.owner = THIS_MODULE,
.of_match_table = test_of_match,
},
};
static int test_init(void)
{
platform_driver_register(&test_platform_driver);
return 0;
}
static void test_exit(void)
{
platform_driver_unregister(&test_platform_driver);
}
module_init(test_init);
module_exit(test_exit);
MODULE_AUTHOR("klyone");
MODULE_DESCRIPTION("DMA SG test module");
MODULE_LICENSE("GPL");
However, the DMA never calls my callback function and I do not have any idea why it happens. Maybe, I am misunderstanding something...
Could anyone help me?
Thanks in advance.
Caveat: I don't have a definitive solution for you, but merely some observations and suggestions on how to debug this [based on many years of experience writing/debugging linux device drivers].
I presume you believe the callback is not being done because you don't get any printk messages. But, the callback is the only place that has them. But, is the printk level set high enough to see the messages? I'd add a dev_info to your module init, to prove it prints as expected.
Also, you [probably] won't get a callback if dma_start doesn't work as expected, so I'd add some dev_info calls there, too (e.g. before and after the call in step 7). I also notice that not all calls in dma_start check error returns [may be fine or void return, just mentioning in case you missed one]
At this point, it should be noted that there are really two questions here: (1) Did your DMA request start successfully [and complete]? (2) Did you get a callback?
So, I'd split off some code from dma_complete into (e.g.) dma_test_done. The latter does the same checking but only prints the "complete" message. You can call this in a poll mode to verify DMA completion.
So, if you [eventually] get a completion, then the problem reduces to why you didn't get the callback. If, however, you don't [even] get a completion, that's an even more fundamental problem.
This reminds me. You didn't show any code that calls dma_start or how you wait for the completion. I presume that if your callback were working, it would issue a wakeup of some sort that the base level would wait on. Or, the callback would do the request deallocate/cleanup (i.e. more code you'd write)
At step 7, you're calling dma_async_issue_pending, which should call pl330_issue_pending. pl330_issue_pending will call pl330_tasklet.
pl330_tasklet is a tasklet function, but it can also be called directly [to kick off DMA when there are no active requests].
pl330_tasklet will loop on its "work" queue and move any completed items to its "completed" queue. It then tries to start new requests. It then loops on its completed queue and issues the callbacks.
pl330_tasklet grabs the callback pointer, but if it's null it is silently ignored. You've set a callback, but it might be good to verify that where you set the callback is the same place [or propagates to] the place where pl330_tasklet will fetch it from.
When you make the call, everything may be busy, so there are no completed requests, no room to start a new request, so nothing to complete. In that case, pl330_tasklet will be called again later.
So, when dma_async_issue_pending returns, nothing may have happened yet. This is quite probable for your case.
pl330_tasklet tries to start new DMA by calling fill_queue. It will check that a descriptor is not [already] busy by looking at status != BUSY. So, you may wish to verify that yours has the correct value. Otherwise, you'd never get a callback [or even any DMA start].
Then, fill_queue will try to start the request via pl330_submit_req. But, that can return an error (e.g. queue already full), so, again, things are deferred.
For reference, notice the following comment at the top of pl330_submit_req:
Submit a list of xfers after which the client wants notification.
Client is not notified after each xfer unit, just once after all
xfer units are done or some error occurs.
What I'd do is start hacking up pl330.c and add debug messages and cross-checking. If your system is such that pl330 is servicing many other requests, you might limit the debug messages by checking that the device's private data pointer matches yours.
In particular, you'd like to get a message when your request actually gets started, so you could add a debug message to the end of pl330_submit_req
Then, adding messages within pl330_tasklet for requests will help, too.
Those are two good starting points. But, don't be afraid to add more printk calls as needed. You may be surprised by what gets called [or doesn't get called] or in what order.
UPDATE:
If I install the kernel module with the blocking behaviour, everything is initialized well. However, the dma_busy_loop function shows that the DMA descriptor is always IN PROGESS and the DMA transaction never completes. For this reason, the callback function is not executed. What could be happening?
Did a little more research. Cookies are just sequence numbers that increment. For example, if you issue a request that gets broken up into [say] 10 separate scatter/gather operations [descriptors], each one gets a unique cookie value. The cookie return value is the latest/last of the bunch (e.g. 10).
When you're calling (1) dma_async_is_tx_complete, (2) it calls chan->device->device_tx_status, (3) which is pl330_tx_status, (4) which calls dma_cookie_status
Side note/tip: When I was tracking this down, I just kept flipping back and forth between dmaengine.h and pl330.c. It was like: Look at (1), it calls (2). Where is that set? In pl330.c, I presume. So, I grepped for the string and got the name of pl330's function (i.e. (3)). So, I go there, and see that it does (4). So ... Back to dmaengine.h ...
However, when you make the outer call, you're ignoring [setting to NULL] the last two arguments. These can be useful because they return the "last" and "used" cookies. So, even if you don't get full completion, these values could change and show partial progress.
One of them should eventually be >= to the "return" cookie value. (i.e.) The entire operation should be complete. So, this will help differentiate what may be happening.
Also, note that in dmaengine.h, right below dma_async_is_tx_complete, there is dma_async_is_complete. This function is what decides whether to return DMA_COMPLETE or DMA_IN_PROGRESS, based on the cookie value you pass and the "last" and "used" cookie values. It's passive, and not used in the code path [AFAICT], but it does show how to calculate completion yourself.

kprobe, function scheduling - processor lockup - Linux kernel

I've a function I wrote in order to run a given function on all processors. It works perfectly well in all cases except the following case:
When I try to use it within a kprobe that I registered.
Here's some code:
static DEFINE_MUTEX(entryMutex);
static struct kretprobe my_kprobe = {
.entry_handler = (kprobe_opcode_t *) NULL,
.handler = (kprobe_opcode_t *) process_entry_callback,
.maxactive = 1000,
.data_size = 0
};
static int driver_init(void)
{
my_kprobe.kp.addr = (kprobe_opcode_t*)kallsyms_lookup_name("sys_execve");
if ((ret = register_kretprobe(&my_kprobe)) < 0)
return -1;
return 0;
}
void foo(void* nothing)
{
printk("In foo\n");
}
static int process_entry_callback(struct kretprobe_instance* instance, struct pt_regs* regs)
{
mutex_lock(&entryMutex);
for(int i = 0; i < 4; ++i) // assumes there are 4 processors
run_func(foo, NULL, i);
mutex_unlock(&entryMutex);
return 0;
}
void run_func_wrap(struct function_data* data)
{
data->func(data->context);
wake_up_process(data->waiting_task);
*(data->condition) = TRUE;
}
void run_func(SCHEDULED_FUNC func, void *context, int processor)
{
struct function_data data;
struct task_struct* th;
BOOLEAN condition = FALSE;
wait_queue_head_t queue;
init_waitqueue_head(&queue);
data.func = func;
data.waiting_task = current;
data.context = context;
data.condition = &condition;
th = kthread_create(sched_func_wrap, &data, "th");
kthread_bind(th, processor);
wake_up_process(th);
wait_event(queue, condition);
}
F
After the call to 'run_func' in process_entry_callback I can no longer run any programs. Every time I start a new program it just stuck. After a while I get 'processor lockup' warning in the system log.
I suspect that it has something to do with the IRQ levels.
Any suggestions ?
EDIT:
It also happens when using the following function:
smp_call_function_single
which can be found in smp.c # the Linux kernel source code.
instead of my function:
run_func

Overload symbols of running process (LD_PRELOAD attachment)

I'm working on a heap profiler for Linux, called heaptrack. Currently, I rely on LD_PRELOAD to overload various (de-)allocation functions, and that works extremely well.
Now I would like to extend the tool to allow runtime attaching to an existing process, which was started without LD_PRELOADing my tool. I can dlopen my library via GDB just fine, but that won't overwrite malloc etc. I think, this is because at that point the linker already resolved the position dependent code of the already running process - correct?
So what do I do instead to overload malloc and friends?
I am not proficient with assembler code. From what I've read so far, I guess I'll somehow have to patch malloc and the other functions, such that they first call back to my trace function and then continue with their actual implementation? Is that correct? How do I do that?
I hope there are existing tools out there, or that I can leverage GDB/ptrace for that.
Just for the lulz, another solution without ptracing your own process or touching a single line of assembly or playing around with /proc. You only have to load the library in the context of the process and let the magic happen.
The solution I propose is to use the constructor feature (brought from C++ to C by gcc) to run some code when a library is loaded. Then this library just patch the GOT (Global Offset Table) entry for malloc. The GOT stores the real addresses for the library functions so that the name resolution happen only once. To patch the GOT you have to play around with the ELF structures (see man 5 elf). And Linux is kind enough to give you the aux vector (see man 3 getauxval) that tells you where to find in memory the program headers of the current program. However, better interface is provided by dl_iterate_phdr, which is used below.
Here is an example code of library that does exactly this when the init function is called. Although the same could probably be achieved with a gdb script.
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <dlfcn.h>
#include <sys/auxv.h>
#include <elf.h>
#include <link.h>
#include <sys/mman.h>
struct strtab {
char *tab;
ElfW(Xword) size;
};
struct jmpreltab {
ElfW(Rela) *tab;
ElfW(Xword) size;
};
struct symtab {
ElfW(Sym) *tab;
ElfW(Xword) entsz;
};
/* Backup of the real malloc function */
static void *(*realmalloc)(size_t) = NULL;
/* My local versions of the malloc functions */
static void *mymalloc(size_t size);
/*************/
/* ELF stuff */
/*************/
static const ElfW(Phdr) *get_phdr_dynamic(const ElfW(Phdr) *phdr,
uint16_t phnum, uint16_t phentsize) {
int i;
for (i = 0; i < phnum; i++) {
if (phdr->p_type == PT_DYNAMIC)
return phdr;
phdr = (ElfW(Phdr) *)((char *)phdr + phentsize);
}
return NULL;
}
static const ElfW(Dyn) *get_dynentry(ElfW(Addr) base, const ElfW(Phdr) *pdyn,
uint32_t type) {
ElfW(Dyn) *dyn;
for (dyn = (ElfW(Dyn) *)(base + pdyn->p_vaddr); dyn->d_tag; dyn++) {
if (dyn->d_tag == type)
return dyn;
}
return NULL;
}
static struct jmpreltab get_jmprel(ElfW(Addr) base, const ElfW(Phdr) *pdyn) {
struct jmpreltab table;
const ElfW(Dyn) *dyn;
dyn = get_dynentry(base, pdyn, DT_JMPREL);
table.tab = (dyn == NULL) ? NULL : (ElfW(Rela) *)dyn->d_un.d_ptr;
dyn = get_dynentry(base, pdyn, DT_PLTRELSZ);
table.size = (dyn == NULL) ? 0 : dyn->d_un.d_val;
return table;
}
static struct symtab get_symtab(ElfW(Addr) base, const ElfW(Phdr) *pdyn) {
struct symtab table;
const ElfW(Dyn) *dyn;
dyn = get_dynentry(base, pdyn, DT_SYMTAB);
table.tab = (dyn == NULL) ? NULL : (ElfW(Sym) *)dyn->d_un.d_ptr;
dyn = get_dynentry(base, pdyn, DT_SYMENT);
table.entsz = (dyn == NULL) ? 0 : dyn->d_un.d_val;
return table;
}
static struct strtab get_strtab(ElfW(Addr) base, const ElfW(Phdr) *pdyn) {
struct strtab table;
const ElfW(Dyn) *dyn;
dyn = get_dynentry(base, pdyn, DT_STRTAB);
table.tab = (dyn == NULL) ? NULL : (char *)dyn->d_un.d_ptr;
dyn = get_dynentry(base, pdyn, DT_STRSZ);
table.size = (dyn == NULL) ? 0 : dyn->d_un.d_val;
return table;
}
static void *get_got_entry(ElfW(Addr) base, struct jmpreltab jmprel,
struct symtab symtab, struct strtab strtab, const char *symname) {
ElfW(Rela) *rela;
ElfW(Rela) *relaend;
relaend = (ElfW(Rela) *)((char *)jmprel.tab + jmprel.size);
for (rela = jmprel.tab; rela < relaend; rela++) {
uint32_t relsymidx;
char *relsymname;
relsymidx = ELF64_R_SYM(rela->r_info);
relsymname = strtab.tab + symtab.tab[relsymidx].st_name;
if (strcmp(symname, relsymname) == 0)
return (void *)(base + rela->r_offset);
}
return NULL;
}
static void patch_got(ElfW(Addr) base, const ElfW(Phdr) *phdr, int16_t phnum,
int16_t phentsize) {
const ElfW(Phdr) *dphdr;
struct jmpreltab jmprel;
struct symtab symtab;
struct strtab strtab;
void *(**mallocgot)(size_t);
dphdr = get_phdr_dynamic(phdr, phnum, phentsize);
jmprel = get_jmprel(base, dphdr);
symtab = get_symtab(base, dphdr);
strtab = get_strtab(base, dphdr);
mallocgot = get_got_entry(base, jmprel, symtab, strtab, "malloc");
/* Replace the pointer with our version. */
if (mallocgot != NULL) {
/* Quick & dirty hack for some programs that need it. */
/* Should check the returned value. */
void *page = (void *)((intptr_t)mallocgot & ~(0x1000 - 1));
mprotect(page, 0x1000, PROT_READ | PROT_WRITE);
*mallocgot = mymalloc;
}
}
static int callback(struct dl_phdr_info *info, size_t size, void *data) {
uint16_t phentsize;
data = data;
size = size;
printf("Patching GOT entry of \"%s\"\n", info->dlpi_name);
phentsize = getauxval(AT_PHENT);
patch_got(info->dlpi_addr, info->dlpi_phdr, info->dlpi_phnum, phentsize);
return 0;
}
/*****************/
/* Init function */
/*****************/
__attribute__((constructor)) static void init(void) {
realmalloc = malloc;
dl_iterate_phdr(callback, NULL);
}
/*********************************************/
/* Here come the malloc function and sisters */
/*********************************************/
static void *mymalloc(size_t size) {
printf("hello from my malloc\n");
return realmalloc(size);
}
And an example program that just loads the library between two malloc calls.
#include <stdio.h>
#include <stdlib.h>
#include <dlfcn.h>
void loadmymalloc(void) {
/* Should check return value. */
dlopen("./mymalloc.so", RTLD_LAZY);
}
int main(void) {
void *ptr;
ptr = malloc(42);
printf("malloc returned: %p\n", ptr);
loadmymalloc();
ptr = malloc(42);
printf("malloc returned: %p\n", ptr);
return EXIT_SUCCESS;
}
The call to mprotect is usually useless. However I found that gvim (which is compiled as a shared object) needs it. If you also want to catch the references to malloc as pointers (which may allow to later call the real function and bypass yours), you can apply the very same process to the symbol table pointed to by the DT_RELA dynamic entry.
If the constructor feature is not available for you, all you have to do is resolve the init symbol from the newly loaded library and call it.
Note that you may also want to replace dlopen so that libraries loaded after yours gets patched as well. Which may happen if you load your library quite early or if the application has dynamically loaded plugins.
This can not be done without tweaking with assembler a bit. Basically, you will have to do what gdb and ltrace do: find malloc and friends virtual addresses in the process image and put breakpoints at their entry. This process usually involves temporary rewriting the executable code, as you need to replace normal instructions with "trap" ones (such as int 3 on x86).
If you want to avoid doing this yourself, there exists linkable wrapper around gdb (libgdb) or you can build ltrace as a library (libltrace). As ltrace is much smaller, and the library variety of it is available out of the box, it will probably allow you to do what you want at lower effort.
For example, here's the best part of the "main.c" file from the ltrace package:
int
main(int argc, char *argv[]) {
ltrace_init(argc, argv);
/*
ltrace_add_callback(callback_call, EVENT_SYSCALL);
ltrace_add_callback(callback_ret, EVENT_SYSRET);
ltrace_add_callback(endcallback, EVENT_EXIT);
But you would probably need EVENT_LIBCALL and EVENT_LIBRET
*/
ltrace_main();
return 0;
}
http://anonscm.debian.org/cgit/collab-maint/ltrace.git/tree/?id=0.7.3

Resources