How can I invoke system functions from native code on AIX? - c

My purpose is invoke a system call or libc functions from native code.
The native code is a executable memory block which I insert some machine code into it. In the native code I try to invoke puts() or printf() functions to output a string. Then I call the native code. This sequence model is the Mono tries to do. But when I porting the Mono into AIX, I met a segment fault when invokes system library functions such as printf() or abs(). I guess it must break calling stack somewhere. But I could not resolve such problem I am not familiar with IBM powerpc platform.
I have wrote a simple program to demonstrate the sequence, it also lead to segment fault at calling puts(). It takes so much time, please give me some advises, thank you in advance!
#include <stdio.h>
#include <string.h>
#include <sys/types.h>
#include <sys/mman.h>
typedef uint8_t guint8;
typedef int16_t gint16;
typedef uint16_t guint16;
typedef int32_t gint32;
typedef uint32_t guint32;
typedef int64_t gint64;
typedef uint64_t guint64;
typedef float gfloat;
typedef double gdouble;
typedef int32_t gboolean;
typedef void * gpointer;
typedef enum {
ppc_r0 = 0,
ppc_r1,
ppc_sp = ppc_r1,
ppc_r2,
ppc_r3,
ppc_r4,
ppc_r5,
ppc_r6,
ppc_r7,
ppc_r8,
ppc_r9,
ppc_r10,
ppc_r11,
ppc_r12,
ppc_r13,
ppc_r14,
ppc_r15,
ppc_r16,
ppc_r17,
ppc_r18,
ppc_r19,
ppc_r20,
ppc_r21,
ppc_r22,
ppc_r23,
ppc_r24,
ppc_r25,
ppc_r26,
ppc_r27,
ppc_r28,
ppc_r29,
ppc_r30,
ppc_r31
} PPCIntRegister;
typedef enum {
ppc_lr = 256,
ppc_ctr = 256 + 32,
ppc_xer = 32
} PPCSpecialRegister;
#define G_STMT_START do
#define G_STMT_END while (0)
#define ppc_load32(c,D,v) G_STMT_START { \
ppc_lis ((c), (D), (guint32)(v) >> 16); \
ppc_ori ((c), (D), (D), (guint32)(v) & 0xffff); \
} G_STMT_END
#define ppc_emit32(c,x) do { *((guint32 *) (c)) = (guint32) (x); (c) = (gpointer)((guint8 *)(c) + sizeof ( guint32));} while (0)
#define ppc_stwux(c,S,A,B) ppc_emit32(c, (31 << 26) | (S << 21) | (A << 16) | (B << 11) | (183 << 1) | 0)
#define ppc_or(c,a,s,b) ppc_emit32 (c, (31 << 26) | ((s) << 21) | ((a) << 16) | ((b) << 11) | 888)
#define ppc_mr(c,a,s) ppc_or (c, a, s, s)
#define ppc_ori(c,S,A,ui) ppc_emit32 (c, (24 << 26) | ((S) << 21) | ((A) << 16) | (guint16)(ui))
#define ppc_addis(c,D,A,i) ppc_emit32 (c, (15 << 26) | ((D) << 21) | ((A) << 16) | (guint16)(i))
#define ppc_lis(c,D,v) ppc_addis (c, D, 0, (guint16)(v))
#define ppc_load_sequence(c,D,v) ppc_load32 ((c), (D), (guint32)(v))
#define ppc_load_func(c,D,V) ppc_load_sequence ((c), (D), (V))
#define ppc_mtspr(c,spr,S) ppc_emit32 (c, (31 << 26) | ((S) << 21) | ((spr) << 11) | (467 << 1))
#define ppc_mtlr(c,S) ppc_mtspr (c, ppc_lr, S)
#define ppc_blrl(c) ppc_emit32 (c, 0x4e800021)
#define ppc_mfspr(c,D,spr) ppc_emit32 (c, (31 << 26) | ((D) << 21) | ((spr) << 11) | (339 << 1))
#define ppc_mflr(c,D) ppc_mfspr (c, D, ppc_lr)
#define ppc_stw(c,S,d,A) ppc_emit32 (c, (36 << 26) | ((S) << 21) | ((A) << 16) | (guint16)(d))
#define ppc_stwu(c,s,d,A) ppc_emit32 (c, (37 << 26) | ((s) << 21) | ((A) << 16) | (guint16)(d))
#define ppc_addi(c,D,A,i) ppc_emit32 (c, (14 << 26) | ((D) << 21) | ((A) << 16) | (guint16)(i))
#define ppc_lwz(c,D,d,A) ppc_emit32 (c, (32 << 26) | ((D) << 21) | ((A) << 16) | (guint16)(d))
#define ppc_blr(c) ppc_emit32 (c, 0x4e800020)
#define ppc_bl(c,li) ppc_emit32 (c, (18 << 26) | ((li) << 2) | 1)
#define PPC_CALL_REG ppc_r12
void foo()
{
puts("Hello");
}
int main()
{
unsigned char codebuf [1024];
unsigned char* code;
void * mem;
unsigned char* codest;
void *values[1];
int rc;
foo();
code = codest = codebuf;
ppc_load_func(code, PPC_CALL_REG, *((void **)foo));
ppc_mtlr(code, PPC_CALL_REG);
ppc_blrl(code);
mem = mmap(NULL, code - codest, PROT_WRITE | PROT_EXEC,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
memcpy(mem, codest, code - codest);
void (*func) () = &mem;
func();
return 0;
}
I must use gcc. the macros were copied from Mono. func() is good when not call system library functions.
It could step into func(), but when invokes puts, the register R2 value somehow was manipulated. I don't know why? following is debug step on my test box :
127 func();
=> 0x10000760 <main+304>: 80 1f 00 40 lwz r0,64(r31)
0x10000764 <main+308>: 7c 0b 03 78 mr r11,r0
0x10000768 <main+312>: 81 2b 00 00 lwz r9,0(r11)
0x1000076c <main+316>: 90 41 00 14 stw r2,20(r1)
0x10000770 <main+320>: 7c 0a 03 78 mr r10,r0
0x10000774 <main+324>: 81 6a 00 08 lwz r11,8(r10)
0x10000778 <main+328>: 7d 29 03 a6 mtctr r9
0x1000077c <main+332>: 7c 0a 03 78 mr r10,r0
0x10000780 <main+336>: 80 4a 00 04 lwz r2,4(r10)
0x10000784 <main+340>: 4e 80 04 21 bctrl
0x10000788 <main+344>: 80 41 00 14 lwz r2,20(r1)
above is the point of entering func().
below is the core dump snapshot:
(gdb)
0x10000550 90 puts("Hello");
0x1000054c <foo+20>: 80 62 00 58 lwz r3,88(r2)
=> 0x10000550 <foo+24>: 48 00 03 41 bl 0x10000890 <puts>
0x10000554 <foo+28>: 80 41 00 14 lwz r2,20(r1)
(gdb)
0x10000890 in puts ()
=> 0x10000890 <puts+0>: 81 82 00 5c lwz r12,92(r2)
(gdb)
0x10000894 in puts ()
=> 0x10000894 <puts+4>: 90 41 00 14 stw r2,20(r1)
(gdb) info reg r12
r12 0x0 0
(gdb) ni
0x10000898 in puts ()
=> 0x10000898 <puts+8>: 80 0c 00 00 lwz r0,0(r12)
(gdb)
0x1000089c in puts ()
=> 0x1000089c <puts+12>: 80 4c 00 04 lwz r2,4(r12)
(gdb)
0x100008a0 in puts ()
=> 0x100008a0 <puts+16>: 7c 09 03 a6 mtctr r0
(gdb) info reg r0
r0 0x0 0
(gdb) ni
0x100008a4 in puts ()
=> 0x100008a4 <puts+20>: 4e 80 04 20 bctr
(gdb)
Program received signal SIGILL, Illegal instruction.
0x00000000 in ?? ()
=> 0x00000000: 00 00 00 00 .long 0x0
in puts(), it jump to wrong address 0x0, at the begining of func(), compiler has modified R2(lwz r2,4(r10)), it leads to the problem, but it is compiler generated code, I can't change it. I don't know how to handle this issue. Please help me! Thank you.

You need to understand "glue code" and "glink code".
As a practical method, write just a simple program where main calls puts. Then look at the assembly produced as well as single step (i.e. stepi) through the code.
The "function pointer" puts() does not point to the executable when it is in another module. Instead it points to a toc entry that has three entries: the address of the function, the toc of the module, and another value that I can't recall. The glue code / glink code (IIRC) takes a pointer in r11 to this entry and it will then correctly call the destination function.
Upon return, the next instruction after the bl will be an instruction to restore the toc.
The compiler, ld, as well as the loader all participate in the magic to make this happen.
Happy hunting...

Related

Writing out a binary structure to file, unexpected results?

I'm trying to write out a packed data structure to a binary file, however as you can see from od -x below the results are not expected in their ordering. I'm using gcc on a 64-bit Intel system. Does anyone know why the ordering is wrong? It doesn't look like an endianness issue.
#include <stdio.h>
#include <stdlib.h>
struct B {
char a;
int b;
char c;
short d;
} __attribute__ ((packed));
int main(int argc, char *argv[])
{
FILE *fp;
fp = fopen("temp.bin", "w");
struct B b = {'a', 0xA5A5, 'b', 0xFF};
if (fwrite(&b, sizeof(b), 1, fp) != 1)
printf("Error fwrite\n");
exit(0);
}
ASCII 61 is 'a', so the b.a member. ASCII 62 is 'b', so the b.c member. It's odd how 0xA5A5 is spread out over the sequence.
$ od -x temp.bin
0000000 a561 00a5 6200 00ff
0000010
od -x groups the input into 2-byte units and swaps their endianness. It's a confusing output format. Use -t x1 to leave the bytes alone.
$ od -t x1 temp.bin
0000000 61 a5 a5 00 00 62 ff 00
0000010
Or, easier to remember, use hd (hex dump) instead of od (octal dump). hd's default format doesn't need adjusting, plus it shows both a hex and ASCII dump.
$ hd temp.bin
00000000 61 a5 a5 00 00 62 ff 00 |a....b..|
00000008
od -x writes out two little-endian bytes. Per the od man page:
-x same as -t x2, select hexadecimal 2-byte units
So
0000000 a561 00a5 6200 00ff
is, on disk:
0000000 61a5 a500 0062 ff00

sha512: c program using the openSSL library

I have a program in C, which calculates sha256 hash of input file. It is using the openSSL library. Here is the core of the program:
#include <openssl/sha.h>
SHA256_CTX ctx;
unsigned char buffer[512];
SHA256_Init(&ctx);
SHA256_Update(&ctx, buffer, len);
SHA256_Final(buffer, &ctx);
fwrite(&buffer,32,1,stdout);
I need to change it to calculate sha512 hash instead.
Can I just (naively) change all the names of the functions from SHA256 to SHA512, and then in the last step fwrite 64 bytes, instead of the 32 bytes ? Is that all, or do I have to make more changes ?
Yes, this will work. The man page for the SHA family of functions lists the following:
int SHA256_Init(SHA256_CTX *c);
int SHA256_Update(SHA256_CTX *c, const void *data, size_t len);
int SHA256_Final(unsigned char *md, SHA256_CTX *c);
unsigned char *SHA256(const unsigned char *d, size_t n,
unsigned char *md);
...
int SHA512_Init(SHA512_CTX *c);
int SHA512_Update(SHA512_CTX *c, const void *data, size_t len);
int SHA512_Final(unsigned char *md, SHA512_CTX *c);
unsigned char *SHA512(const unsigned char *d, size_t n,
unsigned char *md);
...
SHA1_Init() initializes a SHA_CTX structure.
SHA1_Update() can be called repeatedly with chunks of the message
to be hashed (len bytes at data).
SHA1_Final() places the message digest in md, which must have space
for SHA_DIGEST_LENGTH == 20 bytes of output, and erases the
SHA_CTX.
The SHA224, SHA256, SHA384 and SHA512 families of functions operate
in the same way as for the SHA1 functions. Note that SHA224 and
SHA256 use a SHA256_CTX object instead of SHA_CTX. SHA384 and
SHA512 use SHA512_CTX. The buffer md must have space for the
output from the SHA variant being used (defined by
SHA224_DIGEST_LENGTH, SHA256_DIGEST_LENGTH, SHA384_DIGEST_LENGTH
and SHA512_DIGEST_LENGTH). Also note that, as for the SHA1()
function above, the SHA224(), SHA256(), SHA384() and SHA512()
functions are not thread safe if md is NULL.
To confirm, let's look at some code segments. First with SHA256:
SHA256_CTX ctx;
unsigned char buffer[512];
char *str = "this is a test";
int len = strlen(str);
strcpy(buffer,str);
SHA256_Init(&ctx);
SHA256_Update(&ctx, buffer, len);
SHA256_Final(buffer, &ctx);
fwrite(&buffer,32,1,stdout);
When run as:
./test1 | od -t x1
Outputs:
0000000 2e 99 75 85 48 97 2a 8e 88 22 ad 47 fa 10 17 ff
0000020 72 f0 6f 3f f6 a0 16 85 1f 45 c3 98 73 2b c5 0c
0000040
Which matches the output of:
echo -n "this is a test" | openssl sha256
Which is:
(stdin)= 2e99758548972a8e8822ad47fa1017ff72f06f3ff6a016851f45c398732bc50c
Now the same code with the changes you suggested:
SHA512_CTX ctx;
unsigned char buffer[512];
char *str = "this is a test";
int len = strlen(str);
strcpy(buffer,str);
SHA512_Init(&ctx);
SHA512_Update(&ctx, buffer, len);
SHA512_Final(buffer, &ctx);
fwrite(&buffer,64,1,stdout);
The output when passed through "od" gives us:
0000000 7d 0a 84 68 ed 22 04 00 c0 b8 e6 f3 35 ba a7 e0
0000020 70 ce 88 0a 37 e2 ac 59 95 b9 a9 7b 80 90 26 de
0000040 62 6d a6 36 ac 73 65 24 9b b9 74 c7 19 ed f5 43
0000060 b5 2e d2 86 64 6f 43 7d c7 f8 10 cc 20 68 37 5c
0000100
Which matches the output of:
echo -n "this is a test" | openssl sha512
Which is:
(stdin)= 7d0a8468ed220400c0b8e6f335baa7e070ce880a37e2ac5995b9a97b809026de626da636ac7365249bb974c719edf543b52ed286646f437dc7f810cc2068375c
I am using macOS and openssl. This works for me:
#include <openssl/sha.h>
#include <stdio.h>
#include <string.h>
int main() {
unsigned char data[] = "some text";
unsigned char hash[SHA512_DIGEST_LENGTH];
SHA512(data, strlen((char *)data), hash);
for (int i = 0; i < SHA512_DIGEST_LENGTH; i++)
printf("%02x", hash[i]);
putchar('\n');
}
I compile using,
~$ gcc -o sha512 sha512.c \
-I /usr/local/opt/openssl/include \
-L /usr/local/opt/openssl/lib \
-lcrypto
~S ./sha512
e2732baedca3eac1407828637de1dbca702c3fc9ece16cf536ddb8d6139cd85dfe7464b8235
b29826f608ccf4ac643e29b19c637858a3d8710a59111df42ddb5

How to load IDT?

I'm working on writing own little os. I already done booting, entering protected mode, some text printing and serial communication with qemu.
I've been trying to add interrupts for over two days. I was looking everywhere including other systems sources. No I've got code below and qemu (I don't know why) shuts down. I including geust errors from qemu.
This is my main file kernel.cpp
__asm__ (".pushsection .text.start\r\n" \
"jmp main\r\n" \
".popsection\r\n");
#include <stdbool.h>
#include "utils/debug.h"
#include "drivers/display.h"
#include "drivers/serial.h"
#include "drivers/keyboard.h"
#include "drivers/pic.h"
#include "interrupt/interrupt.h"
void initDrivers(){
serial::init();
pic::init();
interrupt::init();
interrupt::enable();
terminal_initialize();
}
int main() {
initDrivers();
terminal_setcolor(VGA_COLOR_WHITE);
terminal_writestring("Hello!");
debug::println("after println");
bool alive = true;
while(alive) {
}
return 0;
}
PIC driver pic.cpp and pic.h
#include "pic.h"
#include <stddef.h>
#include <stdint.h>
#include "IO.h"
#include "../utils/debug.h"
/*
* IMR - Interrupt Mask Register
* IRR - Interrupt Request Register
* ISR - In-Service Register or Interrupt Service Routine
*
*/
namespace pic {
static const uint16_t PIC1_PORT = 0x20; // I/O offset address for master PIC
static const uint16_t PIC2_PORT = 0xA0; // I/O offset address for slave PIC
static const uint16_t PIC1_SPORT = 0x21; // I/O offset address for master PIC data
static const uint16_t PIC2_SPORT = 0xA1; // I/O offset address for slave PIC data
static const uint8_t OCW3_READ_IRR = 0x0A; // OCW3 irq ready next CMD read
static const uint8_t OCW3_READ_ISR = 0x0B; // OCW3 irq service next CMD read
void init(){
outb(PIC1_PORT, 0x11); // Sending ICW1 to first PIC (starting initialization)
io_wait(); // Waiting for PIC to process
outb(PIC2_PORT, 0x11); // Sending ICW1 to second PIC
io_wait(); // Waiting for PIC to process
outb(PIC1_SPORT, 0x20); // Sending ICW2 to first PIC (Mapping IRQs)
io_wait(); // Waiting for PIC to process
outb(PIC2_SPORT, 0x28); // Sending ICW2 to second PIC
io_wait(); // Waiting for PIC to process
outb(PIC1_SPORT, 4); // Sending ICW3 to first PIC, "there is a second one - slave" (0000 0100)
io_wait(); // Waiting for PIC to process
outb(PIC2_SPORT, 2); // Sending ICW3 to second PIC "You are slave" (0000 0010)
io_wait(); // Waiting for PIC to process
outb(PIC1_SPORT, 1); // Sending ICW4 to first PIC putting it into 80x86 mode (0000 0001)
io_wait(); // Waiting for PIC to process
outb(PIC2_SPORT, 1); // Sending ICW4 to second PIC putting it into 80x86 mode (0000 0001)
io_wait(); // Waiting for PIC to process
debug::println("PIC initialized!");
}
uint16_t __getIrqReg(uint8_t ocw3) {
outb(PIC1_SPORT, ocw3);
outb(PIC2_SPORT, ocw3);
return (inb(PIC2_SPORT) << 8) | inb(PIC1_SPORT);
}
uint16_t getIRR() {
return __getIrqReg(OCW3_READ_IRR);
}
uint16_t getISR() {
return __getIrqReg(OCW3_READ_ISR);
}
void ack(uint8_t irq) {
/* write EOI */
if (irq >= 16) return; // It's not a valid irq
if (irq >= 8) outb(PIC2_SPORT, 0x20); // It's a PIC2
// PIC1 EOI must be called anyway
outb(PIC1_SPORT, 0x20);
}
void mask(uint8_t irq, bool enable) {
uint16_t port;
if (irq < 8) port = PIC1_SPORT;
else if (irq < 16) {
port = PIC2_SPORT;
irq -= 8;
} else return;
uint8_t value = inb(port);
wait(150);
if (enable) value = value & ~(1 << irq);
else value = value | (1 << irq);
outb(port, value);
wait(150);
}
}
header:
#ifndef PIC_H
#define PIC_H
#include <stdint.h>
namespace pic {
void init();
uint16_t getIRR();
uint16_t getISR();
void ack(uint8_t irq);
void mask(uint8_t irq, bool enabled);
}
#endif
All setup for IDT idt.cpp
#include <stdint.h>
extern "C" {
// CPU Exceptions https://wiki.osdev.org/Exceptions
extern void isr_0(void); // Divide By Zero
extern void isr_1(void); // Debug
extern void isr_2(void); // Non-Maskable Interrupt
extern void isr_3(void); // Breakpoint
extern void isr_4(void); // Overflow
extern void isr_5(void); // Bound Range Exceeded
extern void isr_6(void); // Invalid Opcode
extern void isr_7(void); // Device Not Avaible
extern void isr_8(void); // Double Fault
extern void isr_9(void); // Coprocessor Segment Overrun (Old - out of use)
extern void isr_10(void); // Invalid TTS
extern void isr_11(void); // Segment Not Present
extern void isr_12(void); // Stack Segment Fault
extern void isr_13(void); // General Protection Fault
extern void isr_14(void); // Page Fault
extern void isr_16(void); // x87 Floating-Point Exception
extern void isr_17(void); // Aligment Check
extern void isr_18(void); // Machine Check
extern void isr_19(void); // SIMD Floating Point Exception
extern void isr_20(void); // Virtualization Exception
extern void isr_30(void); // Security Exception
// IRQs https://wiki.osdev.org/Interrupts#General_IBM-PC_Compatible_Interrupt_Information
extern void isr_32(void); // Programmable Interrupt Timer
extern void isr_33(void); // Keyboard
extern void isr_34(void); // Used Internally (Never Raised)
extern void isr_35(void); // COM2
extern void isr_36(void); // COM1
extern void isr_37(void); // LPT2
extern void isr_38(void); // Floppy Disk
extern void isr_39(void); // LPT1
extern void isr_40(void); // CMOS RTC
extern void isr_41(void); // Peripherals
extern void isr_42(void); // Peripherals
extern void isr_43(void); // Peripherals
extern void isr_44(void); // PS/2 Mouse
extern void isr_45(void); // FPU / Coprocessor / Inter-processor
extern void isr_46(void); // Primary ATA Hard Disk
extern void isr_47(void); // Seconadry ATA Hard Disk
extern void syscallHandler(void);
extern void isr_49(void);
extern void isr_50(void);
}
/* IDT - Interrupt Descriptor Table https://wiki.osdev.org/IDT
* offset - tells where Interrupt Service Routine Is Located
* selector - something like properties
* flags:
* | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 |
* +---+---+---+---+---+---+---+---+
* | P | DPL | S | GateType |
* +---+---+---+---+---+---+---+---+
* P - Present (0 if unused)
* DPL - Sort of permission to call`
* Storage Segment - We want it 0
* GateType - Trap, Interrupt, Task
*/
struct idt_entry {
uint16_t offset_low;
uint16_t selector;
uint8_t unused;
uint8_t flags;
uint16_t offset_high;
};
#define IDT_ENTRY(offset, selector, flags) { \
(uint16_t)((uintptr_t)(offset) & 0xFFFF), \
(selector), \
0, \
(flags), \
(uint16_t)(((uintptr_t)(offset) >> 16) & 0xFFFF) \
}
#define IDT_INTERRUPT_GATE 0xE
#define IDT_TRAP_GATE 0xF
#define IDT_RING0 (0 << 5)
#define IDT_RING3 (3 << 5)
#define IDT_PRESENT (1 << 7)
extern "C" {
idt_entry idt[] = {
IDT_ENTRY(isr_0, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_1, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_2, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_3, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_4, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_5, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_6, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_7, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_8, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_9, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_10, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_11, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_12, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_13, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_14, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(0, 0, 0),
IDT_ENTRY(isr_16, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_17, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_18, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_19, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_20, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(0, 0, 0),
IDT_ENTRY(0, 0, 0),
IDT_ENTRY(0, 0, 0),
IDT_ENTRY(0, 0, 0),
IDT_ENTRY(0, 0, 0),
IDT_ENTRY(0, 0, 0),
IDT_ENTRY(0, 0, 0),
IDT_ENTRY(0, 0, 0),
IDT_ENTRY(0, 0, 0),
IDT_ENTRY(0, 0, 0),
IDT_ENTRY(0, 0, 0),
IDT_ENTRY(isr_32, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_33, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_34, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_35, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_36, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_37, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_38, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_39, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_40, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_41, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_42, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_43, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_44, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_45, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_46, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
IDT_ENTRY(isr_47, 0x8, IDT_INTERRUPT_GATE | IDT_RING0 | IDT_PRESENT),
//IDT_ENTRY(syscallHandler, 0x8, IDT_TRAP_GATE | IDT_RING3 | IDT_PRESENT),
IDT_ENTRY(isr_49, 0x8, IDT_INTERRUPT_GATE | IDT_RING3 | IDT_PRESENT),
IDT_ENTRY(isr_50, 0x8, IDT_INTERRUPT_GATE | IDT_RING3 | IDT_PRESENT),
};
uint16_t idt_size = sizeof(idt) - 1;
}
File that glues that together interrupt.cpp and interrupt.h (In that file there is a function that causes crashes, but i will explain it after all files.)
#include "interrupt.h"
#include "signals.h"
#include <stdint.h>
#include "../utils/debug.h"
#include "../drivers/IO.h"
#define PIC1_COMMAND 0x20
#define PIC1_DATA 0x21
#define PIC2_COMMAND 0xA0
#define PIC2_DATA 0xA1
#define PIC_EOI 0x20
#define EX_DEVIDE_BY_ZERO 0
#define EX_DEBUG 1
#define EX_NON_MASKABLE_INTERRUPT 2
#define EX_BREAKPOINT 3
#define EX_OVERFLOW 4
#define EX_BOUND_RANGE_EXCEEDED 5
#define EX_INVALID_OPCODE 6
#define EX_DEVICE_NOT_AVAILABLE 7
#define EX_DOUBLE_FAULT 8
#define EX_COPROCESSOR_SEGMENT_OVERRUN 9
#define EX_INVAILD_TSS 10
#define EX_SEGMENT_NOT_PRESENT 11
#define EX_STACK_SEGMENT_FAULT 12
#define EX_GENERAL_PROTECTION_FAULT 13
#define EX_PAGE_FAULT 14
#define EX_X87_FLOATING_POINT_EXCEPTION 16
#define EX_ALIGNMENT_CHECK 17
#define EX_MACHINE_CHECK 18
#define EX_SIMD_FLOATING_POINT_EXCEPTION 19
#define EX_VIRTUALIZATION_EXCEPTION 20
const char *exception_messages[] = {
"Division By Zero",
"Debug",
"Non Maskable Interrupt",
"Breakpoint",
"Into Detected Overflow",
"Out of Bounds",
"Invalid Opcode",
"No Coprocessor",
"Double Fault",
"Coprocessor Segment Overrun",
"Bad TSS",
"Segment Not Present",
"Stack Fault",
"General Protection Fault",
"Page Fault",
"Unknown Interrupt",
"Coprocessor Fault",
"Alignment Check",
"Machine Check",
"Reserved",
"Reserved",
"Reserved",
"Reserved",
"Reserved",
"Reserved",
"Reserved",
"Reserved",
"Reserved",
"Reserved",
"Reserved",
"Reserved",
"Reserved"
};
void (*interrupt::irqHandlers[16])(int) = {0};
void interrupt::init(){
__asm__ (
"push $idt \r\n" \
"pushw idt_size \r\n" \
"lidt (%esp) \r\n"
);
}
void interrupt::disable() {
asm volatile ("cli");
}
void interrupt::enable() {
asm volatile ("sti");
}
extern "C" {
volatile unsigned long signalPending = 0;
}
static bool handleUserspaceException(const InterruptContext* context) {
siginfo_t siginfo = {};
switch (context->interrupt) {
case EX_DEVIDE_BY_ZERO:
siginfo.si_signo = SIGFPE;
siginfo.si_code = FPE_INTDIV;
siginfo.si_addr = (void*) context->eip;
break;
case EX_DEBUG:
case EX_BREAKPOINT:
siginfo.si_signo = SIGTRAP;
siginfo.si_code = TRAP_BRKPT;
siginfo.si_addr = (void*) context->eip;
break;
case EX_OVERFLOW:
case EX_BOUND_RANGE_EXCEEDED:
case EX_STACK_SEGMENT_FAULT:
case EX_GENERAL_PROTECTION_FAULT:
siginfo.si_signo = SIGSEGV;
siginfo.si_code = SI_KERNEL;
siginfo.si_addr = (void*) context->eip;
break;
case EX_INVALID_OPCODE:
siginfo.si_signo = SIGILL;
siginfo.si_code = ILL_ILLOPC;
siginfo.si_addr = (void*) context->eip;
break;
case EX_PAGE_FAULT:
siginfo.si_signo = SIGSEGV;
siginfo.si_code = SEGV_MAPERR;
asm ("mov %%cr2, %0" : "=r"(siginfo.si_addr));
break;
case EX_X87_FLOATING_POINT_EXCEPTION:
case EX_SIMD_FLOATING_POINT_EXCEPTION:
siginfo.si_signo = SIGFPE;
siginfo.si_code = FPE_FLTINV;
siginfo.si_addr = (void*) context->eip;
break;
default:
return false;
}
//Process::current->raiseSignal(siginfo);
return true;
}
extern "C" InterruptContext* handleInterrupt(InterruptContext* context) {
InterruptContext* newContext = context;
if (context->interrupt <= 31 && context->cs != 0x8) {
if (!handleUserspaceException(context)) goto handleKernelException;
} else if (context->interrupt <= 31) { // CPU Exception
handleKernelException:
debug::print("Exception ");
debug::print(context->interrupt);
debug::print(" (");
debug::print(exception_messages[context->interrupt]);
debug::println(") occurred!");
debug::print("eax: 0x");
debug::print(context->eax);
debug::print(", ebx: 0x");
debug::print(context->ebx);
debug::print(", ecx: 0x");
debug::print(context->ecx);
debug::print(", edx: 0x");
debug::println(context->edx);
debug::print("edi: 0x");
debug::print(context->edi);
debug::print(", esi: 0x");
debug::print(context->esi);
debug::print(", ebp: 0x");
debug::print(context->ebp);
debug::print(", error: 0x");
debug::println(context->error);
debug::print("eip: 0x");
debug::print(context->eip);
debug::print(", cs: 0x");
debug::print(context->cs);
debug::print(", eflags: 0x");
debug::println(context->eflags);
if (context->cs != 0x8) {
debug::print("ss: 0x");
debug::print(context->ss);
debug::print(", esp: 0x");
debug::println(context->esp);
}
// Halt the cpu
while (true) asm volatile ("cli; hlt");
} else if (context->interrupt <= 47) { // IRQ
int irq = context->interrupt - 32;
if (irq == 0) {
//newContext = Process::schedule(context);
}
if (interrupt::irqHandlers[irq]) {
interrupt::irqHandlers[irq](irq);
}
// Send End of Interrupt
if (irq >= 8) {
outb(PIC2_COMMAND, PIC_EOI);
}
outb(PIC1_COMMAND, PIC_EOI);
} else if (context->interrupt == 0x32) {
//newContext = Signal::sigreturn(context);
debug::println("Keyboard!!");
} else {
debug::print("Unknown interrupt ");
debug::print(context->interrupt);
debug::println("!");
}
return newContext;
}
header:
#ifndef INTERRUPS_H
#define INTERRUPS_H
#include <stdint.h>
extern "C" volatile unsigned long signalPending;
namespace Signal {
static inline bool isPending() { return signalPending; }
}
struct InterruptContext {
uint32_t eax;
uint32_t ebx;
uint32_t ecx;
uint32_t edx;
uint32_t esi;
uint32_t edi;
uint32_t ebp;
uint32_t interrupt;
uint32_t error;
// These are pushed by the cpu when an interrupt happens.
uint32_t eip;
uint32_t cs;
uint32_t eflags;
// These are only valid if the interrupt came from Ring 3
uint32_t esp;
uint32_t ss;
};
union sigval {
int sival_int;
void* sival_ptr;
};
typedef __UINT64_TYPE__ __uid_t;
typedef struct {
int si_signo;
int si_code;
int si_pid;
__uid_t si_uid;
void* si_addr;
int si_status;
union sigval si_value;
} siginfo_t;
namespace interrupt {
extern void (*irqHandlers[])(int);
void init();
void disable();
void enable();
}
#endif
And finally Assembly code intr.s
.section .text
.macro isr no
.global isr_\no
isr_\no:
push $0 # no error code
push $\no
jmp commonHandler
.endm
.macro isr_error_code no
.global isr_\no
isr_\no:
# an error code was already pushed
push $\no
jmp commonHandler
.endm
commonHandler:
cld
# Push registers
push %ebp
push %edi
push %esi
push %edx
push %ecx
push %ebx
push %eax
# Switch to kernel data segment
mov $0x10, %ax
mov %ax, %ds
mov %ax, %es
mov %esp, %eax
and $(~0xFF), %esp # Align the stack
sub $12, %esp
push %eax
call handleInterrupt
mov %eax, %esp
# Check whether signals are pending.
mov signalPending, %ebx
test %ebx, %ebx
jz 1f
# Don't handle signals when returning to kernelspace.
mov 40(%esp), %ebx # cs
cmp $0x8, %ebx
je 1f
and $(~0xFF), %esp # Align the stack
sub $12, %esp
push %eax
# call handleSignal
mov %eax, %esp
# Switch back to user data segment
1: mov $0x23, %ax
mov %ax, %ds
mov %ax, %es
pop %eax
pop %ebx
pop %ecx
pop %edx
pop %esi
pop %edi
pop %ebp
# Remove error code and interrupt number from stack
add $8, %esp
iret
# CPU Exceptions
isr 0 # Devide-by-zero Error
isr 1 # Debug
isr 2 # Non-maskable Interrupt
isr 3 # Breakpoint
isr 4 # Overflow
isr 5 # Bound Range Exceeded
isr 6 # Invalid Opcode
isr 7 # Device Not Available
isr_error_code 8 # Double Fault
isr 9 # Coprocessor Segment Overrun
isr_error_code 10 # Invalid TSS
isr_error_code 11 # Segment Not Present
isr_error_code 12 # Stack-Segement Fault
isr_error_code 13 # General Protection Fault
isr_error_code 14 # Page Fault
#isr 15 # Reserved
isr 16 # x87 Floating-Point Exception
isr_error_code 17 # Alignment Check
isr 18 # Machine Check
isr 19 # SIMD Floating-Point Exception
isr 20 # Virtualization Exception
# IRQs
isr 32
isr 33
isr 34
isr 35
isr 36
isr 37
isr 38
isr 39
isr 40
isr 41
isr 42
isr 43
isr 44
isr 45
isr 46
isr 47
#isr 48 # Syscall
isr 49 # Schedule
isr 50 # sigreturn
.global beginSigreturn
beginSigreturn:
# This section is mapped in all user address spaces. When a userspace
# program returns from a signal handler it will return to this address and
# and then perform a sigreturn.
int $0x32
.global endSigreturn
endSigreturn:
Now compile.sh that I'm using to macro compilation (I know about makefiles I will trying to involve them)
if [ $# -eq 0 ]; then
nasm -g -f elf32 -F dwarf -o boot.o bootloader/bootloader.asm
ld -melf_i386 -Ttext=0x7c00 -nostdlib --nmagic -o boot.elf boot.o
objcopy -O binary boot.elf boot.bin
g++ -std=c++14 -g -c -m32 -ffreestanding kernel.cpp
g++ -std=c++14 -g -c -m32 -ffreestanding drivers/display.cpp
g++ -std=c++14 -g -c -m32 -ffreestanding drivers/serial.cpp
g++ -std=c++14 -g -c -m32 -ffreestanding drivers/keyboard.cpp
g++ -std=c++14 -g -c -m32 -ffreestanding drivers/pic.cpp
g++ -std=c++14 -g -c -m32 -ffreestanding interrupt/interrupt.cpp
g++ -std=c++14 -g -c -m32 -ffreestanding interrupt/idt.cpp
g++ -std=c++14 -g -c -m32 -ffreestanding utils/debug.cpp
g++ -m32 -c interrupt/intr.s -o intr.o
ld -r -m elf_i386 kernel.o display.o serial.o keyboard.o pic.o interrupt.o idt.o intr.o debug.o -o main.o
ld -melf_i386 -Tlinker.ld -nostdlib --nmagic -o main.elf main.o
objcopy -O binary main.elf main.bin
dd if=/dev/zero of=disk.img bs=512 count=2880
dd if=boot.bin of=disk.img bs=512 conv=notrunc
dd if=main.bin of=disk.img bs=512 seek=1 conv=notrunc
rm -rf boot.o
rm -rf kernel.o
rm -rf serial.o
rm -rf main.o
rm -rf keyboard.o
rm -rf debug.o
rm -rf pic.o
rm -rf display.o
rm -rf boot.bin
rm -rf kernel.bin
else
if [ $1 = "clean" ]; then
rm -rf boot.o
echo successfully removed boot.o!
rm -rf kernel.o
echo successfully removed kernel.o!
rm -rf IO.o
echo successfully removed IO.o!
rm -rf main.o
echo successfully removed main.o!
rm -rf boot.bin
echo successfully removed boot.bin!
rm -rf kernel.bin
echo successfully removed kernel.bin!
rm -rf boot.elf
echo successfully removed boot.elf!
rm -rf kernel.elf
echo successfully removed kernel.elf!
rm -rf disk.img
echo successfully removed disk.img!
echo Cleanup done!
fi
fi
And the last ones run.sh and rund.sh that I'm using to macro qemu etc.
qemu-system-i386 -fda disk.img -d guest_errors & \
echo "Reading kernel log (Ctrl+C for exit)"
tail -f virtual.log
qemu-system-i386 -m 1024 -fda disk.img -S -s &
gdb main.elf \
-ex 'target remote localhost:1234' \
-ex 'layout src' \
-ex 'layout reg' \
-ex 'break main' \
-ex 'continue'
I'm including also qemu guest errors output
EAX=0000be44 EBX=0000be44 ECX=00090000 EDX=000003f8
ESI=00000000 EDI=00000000 EBP=bee00000 ESP=0008ffba
EIP=ffc80000 EFL=00000016 [----AP-] CPL=0 II=0 A20=1 SMM=0 HLT=0
ES =0010 00000000 ffffffff 00cf9300 DPL=0 DS [-WA]
CS =0008 00000000 ffffffff 00cf9a00 DPL=0 CS32 [-R-]
SS =0010 00000000 ffffffff 00cf9300 DPL=0 DS [-WA]
DS =0010 00000000 ffffffff 00cf9300 DPL=0 DS [-WA]
FS =0010 00000000 ffffffff 00cf9300 DPL=0 DS [-WA]
GS =0010 00000000 ffffffff 00cf9300 DPL=0 DS [-WA]
LDT=0000 00000000 0000ffff 00008200 DPL=0 LDT
TR =0000 00000000 0000ffff 00008b00 DPL=0 TSS32-busy
GDT= 00007d35 00000017
IDT= 0000bee0 00000000
CR0=00000011 CR2=00000000 CR3=00000000 CR4=00000000
DR0=00000000 DR1=00000000 DR2=00000000 DR3=00000000
DR6=ffff0ff0 DR7=00000400
CCS=0000268b CCD=0000be44 CCO=ADDL
EFER=0000000000000000
FCW=037f FSW=0000 [ST=0] FTW=00 MXCSR=00001f80
FPR0=0000000000000000 0000 FPR1=0000000000000000 0000
FPR2=0000000000000000 0000 FPR3=0000000000000000 0000
FPR4=0000000000000000 0000 FPR5=0000000000000000 0000
FPR6=0000000000000000 0000 FPR7=0000000000000000 0000
XMM00=00000000000000000000000000000000 XMM01=00000000000000000000000000000000
XMM02=00000000000000000000000000000000 XMM03=00000000000000000000000000000000
XMM04=00000000000000000000000000000000 XMM05=00000000000000000000000000000000
XMM06=00000000000000000000000000000000 XMM07=00000000000000000000000000000000
So like I said. Code fragment that causes shutdowns is in interrupt.cpp
void interrupt::init(){
__asm__ (
"push $idt \r\n" \
"pushw idt_size \r\n" \
"lidt (%esp) \r\n"
);
}
If I comment this one, os working, but there is no interrupts.
I have no ideas how to fix it.
Any ideas?
The problem is that you don't restore the stack pointer in interrupt::init, so the return from that function returns into the weeds.
The simplest, but poor, fix for this is to simply add
add $6, %esp
after lidt.
A much better solution is to change it to not modify esp within the inline assembly.
void interrupt::init()
{
struct __attribute__((packed)) { uint16_t size; uint32_t idt; }
descr = { idt_size, idt };
__asm__ __volatile__ ("lidt %0" : : "m"(descr));
}
Although I appreciate your including all your code, I didn't carefully read all of it; I simply answered the first problem I noticed, so you may still have problems.

Atomically increment two integers with CAS

Apparently, it is possible to atomically increment two integers with compare-and-swap instructions. This talk claims that such an algorithm exists but it does not detail what it looks like.
How can this be done?
(Note, that the obvious solution of incrementing the integers one after the other is not atomic. Also, stuffing multiple integers into one machine word does not count because it would restrict the possible range.)
Make me think of a sequence lock. Not very accurate (putting this from memory) but something along the lines of:
let x,y and s be 64 bit integers.
To increment:
atomic s++ (I mean atomic increment using 64 bit CAS op)
memory barrier
atomic x++
atomic y++
atomic s++
memory barrier
To read:
do {
S1 = load s
X = load x
Y = load y
memory barrier
S2 = load s
} while (S1 != S2)
Also see https://en.wikipedia.org/wiki/Seqlock
If sse2 is available, you can use paddq to add 2 64 bit integers to two other 64 bit integers in one instruction.
#include "emmintrin.h"
//initialize your values somewhere:
//const __m128i ones = _mm_set1_epi64x(1);
//volatile register __m128i vars =
// _mm_set_epi64x(24,7);
static inline __m128i inc_both(__m128i vars, __m128i ones){
return _mm_add_epi64(vars,ones);
}
This should compile to
paddq %xmm0, %xmm1
Since it is static inline, it may use other xmm registers though. If there is significant register pressure the ones operands may become ones(℅rip)
Note: this can be used for adding values other than 1 and there are similar operations for most other math, bitwise and compare instructions, should you need them.
So you can use the lock prefix and make it into an inline asm macro
#define inc64x2(vars) asm volatile( \
"paddq %0, %1\n":"+x"(vars):"x"(ones) \
);
The arm neon equivalent is something like: vaddq_s64(...), but there is a great article about arm/x86 equivalents here.
I've got a solution I've tested. Contained herein is a soup to nuts proof of concept program.
The algorithm is a "use CAS thread id gate" as the 3rd integer. I watched the video talk twice, and I believe this qualifies. It may not be the algorithm that the presenter was thinking of, but it does work.
The X and Y values can be anywhere in memory and the program places them far enough away from each other that they are on different cache lines. It doesn't really matter.
A quick description of the algorithm:
Each thread has a unique id number or tid (non-zero), taken from one's favorite source: pthead_t, getpid, gettid, make one up by whatever means you want. In the program, it just assigns them sequentially starting from 1.
Each thread will call the increment function with this number.
The increment function will spin on a global gate variable using CAS with an old value of 0 and a new value of tid.
When the CAS succeeds, the thread now "owns" things. In other words, if the gate is zero, it's up for grabs. A non-zero value is the tid of the owner and the gate is locked.
Now, the owner is free to increment the X and Y values with simple x += 1 and y += 1.
After that, the increment function releases by doing a store of 0 into the gate.
Here is the diagnostic/proof-of-concept program with everything. The algorithm itself has no restrictions, but I coded it for my machine.
Some caveats:
It assumes gcc/clang
It assumes a 64 bit x86_64 arch.
This was coded using nothing but inline asm and needs no [nor uses any] compiler atomic support for clarity, simplicity, and transparency.
This was built under linux, but should work on any "reasonable" x86 machine/OS (e.g. BSD, OSX should be fine, cygwin probably, and mingw maybe)
Other arches are fine if they support CAS, I just didn't code for them (e.g. arm might work if you code the CAS with ldex/stex pairs)
There are enough abstract primitives that this would/should be easy.
No attempt at Windows compatibility [if you want it, do your own port but send me no tears--or comments :-)].
The makefile and program have been defaulted to best values
Some x86 CPUs may need to use different defaults (e.g. need fence instructions). See the makefile.
Anyway, here it is:
// caslock -- prove cas lock algorithm
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <time.h>
#include <pthread.h>
#define systls __thread
// repeat the madness only once
#ifdef __clang__
#define inline_common inline
#else
#define inline_common static inline
#endif
#define inline_always inline_common __attribute__((__always_inline__))
#define inline_never __attribute__((__noinline__))
// WARNING: inline CAS fails for gcc but works for clang!
#if _USE_CASINLINE_
#define inline_cas inline_always
#else
#define inline_cas inline_never
#endif
typedef unsigned int u32;
typedef unsigned long long u64;
#ifndef LOOPMAX
#define LOOPMAX 1000000
#endif
#ifndef TIDMAX
#define TIDMAX 20
#endif
#if _USE_VPTR_
typedef volatile u32 *xptr32_p;
typedef volatile u64 *xptr64_p;
#else
typedef u32 *xptr32_p;
typedef u64 *xptr64_p;
#endif
#if _USE_TID64_
typedef u64 tid_t;
#define tidload(_xptr) loadu64(_xptr)
#define tidcas(_xptr,_oval,_nval) casu64(_xptr,_oval,_nval)
#define tidstore(_xptr,_nval) storeu64(_xptr,_nval)
#else
typedef u32 tid_t;
#define tidload(_xptr) loadu32(_xptr)
#define tidcas(_xptr,_oval,_nval) casu32(_xptr,_oval,_nval)
#define tidstore(_xptr,_nval) storeu32(_xptr,_nval)
#endif
tid_t tidgate; // gate control
tid_t readycnt; // number of threads ready
tid_t donecnt; // number of threads complete
// ensure that the variables are nowhere near each other
u64 ary[100];
#define kickoff ary[32] // sync to fire threads
#define xval ary[31] // the X value
#define yval ary[87] // the Y value
int inctype; // increment algorithm to use
tid_t tidmax; // maximum number of tasks
u64 loopmax; // loop maximum for each task
// task control
struct tsk {
tid_t tsk_tid; // task id
u32 tsk_casmiss; // cas miss count
};
typedef struct tsk tsk_t;
tsk_t *tsklist; // task list
systls tsk_t *tskcur; // current task block
// show progress
#define PGR(_pgr) \
do { \
fputs(_pgr,stdout); \
fflush(stdout); \
} while (0)
// NOTE: some x86 arches need fence instructions
// 0 -- no fence instructions
// 1 -- use mfence
// 2 -- use lfence/sfence
#if _USE_BARRIER_ == 0
#define BARRIER_RELEASE ""
#define BARRIER_ACQUIRE ""
#define BARRIER_ALL ""
#elif _USE_BARRIER_ == 1
#define BARRIER_ACQUIRE "\tmfence\n"
#define BARRIER_RELEASE "\tmfence\n"
#define BARRIER_ALL "\tmfence\n"
#elif _USE_BARRIER_ == 2
#define BARRIER_ACQUIRE "\tlfence\n"
#define BARRIER_RELEASE "\tsfence\n"
#define BARRIER_ALL "\tmfence\n"
#else
#error caslock: unknown barrier type
#endif
// barrier_acquire -- acquire barrier
inline_always void
barrier_acquire(void)
{
__asm__ __volatile__ (
BARRIER_ACQUIRE
:
:
: "memory");
}
// barrier_release -- release barrier
inline_always void
barrier_release(void)
{
__asm__ __volatile__ (
BARRIER_RELEASE
:
:
: "memory");
}
// barrier -- barrier
inline_always void
barrier(void)
{
__asm__ __volatile__ (
BARRIER_ALL
:
:
: "memory");
}
// casu32 -- compare and exchange four bytes
// RETURNS: 1=ok, 0=fail
inline_cas int
casu32(xptr32_p xptr,u32 oldval,u32 newval)
{
char ok;
__asm__ __volatile__ (
" lock\n"
" cmpxchg %[newval],%[xptr]\n"
" sete %[ok]\n"
: [ok] "=r" (ok),
[xptr] "=m" (*xptr)
: "a" (oldval),
[newval] "r" (newval)
: "memory");
return ok;
}
// casu64 -- compare and exchange eight bytes
// RETURNS: 1=ok, 0=fail
inline_cas int
casu64(xptr64_p xptr,u64 oldval,u64 newval)
{
char ok;
__asm__ __volatile__ (
" lock\n"
" cmpxchg %[newval],%[xptr]\n"
" sete %[ok]\n"
: [ok] "=r" (ok),
[xptr] "=m" (*xptr)
: "a" (oldval),
[newval] "r" (newval)
: "memory");
return ok;
}
// loadu32 -- load value with barrier
// RETURNS: loaded value
inline_always u32
loadu32(const xptr32_p xptr)
{
u32 val;
barrier_acquire();
val = *xptr;
return val;
}
// loadu64 -- load value with barrier
// RETURNS: loaded value
inline_always u64
loadu64(const xptr64_p xptr)
{
u64 val;
barrier_acquire();
val = *xptr;
return val;
}
// storeu32 -- store value with barrier
inline_always void
storeu32(xptr32_p xptr,u32 val)
{
*xptr = val;
barrier_release();
}
// storeu64 -- store value with barrier
inline_always void
storeu64(xptr64_p xptr,u64 val)
{
*xptr = val;
barrier_release();
}
// qsleep -- do a quick sleep
inline_always void
qsleep(int bigflg)
{
struct timespec ts;
if (bigflg) {
ts.tv_sec = 1;
ts.tv_nsec = 0;
}
else {
ts.tv_sec = 0;
ts.tv_nsec = 1000;
}
nanosleep(&ts,NULL);
}
// incby_tidgate -- increment by using thread id gate
void
incby_tidgate(tid_t tid)
// tid -- unique id for accessing entity (e.g. thread id)
{
tid_t *gptr;
tid_t oval;
gptr = &tidgate;
// acquire the gate
while (1) {
oval = 0;
// test mode -- just do a nop instead of CAS to prove diagnostic
#if _USE_CASOFF_
*gptr = oval;
break;
#else
if (tidcas(gptr,oval,tid))
break;
#endif
++tskcur->tsk_casmiss;
}
#if _USE_INCBARRIER_
barrier_acquire();
#endif
// increment the values
xval += 1;
yval += 1;
#if _USE_INCBARRIER_
barrier_release();
#endif
// release the gate
// NOTE: CAS will always provide a barrier
#if _USE_CASPOST_ && (_USE_CASOFF_ == 0)
oval = tidcas(gptr,tid,0);
#else
tidstore(gptr,0);
#endif
}
// tskcld -- child task
void *
tskcld(void *arg)
{
tid_t tid;
tid_t oval;
u64 loopcur;
tskcur = arg;
tid = tskcur->tsk_tid;
// tell master thread that we're fully ready
while (1) {
oval = tidload(&readycnt);
if (tidcas(&readycnt,oval,oval + 1))
break;
}
// wait until we're given the starting gun
while (1) {
if (loadu64(&kickoff))
break;
qsleep(0);
}
// do the increments
for (loopcur = loopmax; loopcur > 0; --loopcur)
incby_tidgate(tid);
barrier();
// tell master thread that we're fully complete
while (1) {
oval = tidload(&donecnt);
if (tidcas(&donecnt,oval,oval + 1))
break;
}
return (void *) 0;
}
// tskstart -- start a child task
void
tskstart(tid_t tid)
{
pthread_attr_t attr;
pthread_t thr;
int err;
tsk_t *tsk;
tsk = tsklist + tid;
tsk->tsk_tid = tid;
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr,1);
err = pthread_create(&thr,&attr,tskcld,tsk);
pthread_attr_destroy(&attr);
if (err)
printf("tskstart: error -- err=%d\n",err);
}
// tskall -- run a single test
void
tskall(void)
{
tid_t tidcur;
tsk_t *tsk;
u64 incmax;
u64 val;
int err;
xval = 0;
yval = 0;
kickoff = 0;
readycnt = 0;
donecnt = 0;
tidgate = 0;
// prealloc the task blocks
tsklist = calloc(tidmax + 1,sizeof(tsk_t));
// start all tasks
PGR(" St");
for (tidcur = 1; tidcur <= tidmax; ++tidcur)
tskstart(tidcur);
// wait for all tasks to be fully ready
PGR(" Sw");
while (1) {
if (tidload(&readycnt) == tidmax)
break;
qsleep(1);
}
// the starting gun -- all tasks are waiting for this
PGR(" Ko");
storeu64(&kickoff,1);
// wait for all tasks to be fully done
PGR(" Wd");
while (1) {
if (tidload(&donecnt) == tidmax)
break;
qsleep(1);
}
PGR(" Done\n");
// check the final count
incmax = loopmax * tidmax;
// show per-task statistics
for (tidcur = 1; tidcur <= tidmax; ++tidcur) {
tsk = tsklist + tidcur;
printf("tskall: tsk=%llu tsk_casmiss=%d (%.3f%%)\n",
(u64) tidcur,tsk->tsk_casmiss,(double) tsk->tsk_casmiss / loopmax);
}
err = 0;
// check for failure
val = loadu64(&xval);
if (val != incmax) {
printf("tskall: xval fault -- xval=%lld incmax=%lld\n",val,incmax);
err = 1;
}
// check for failure
val = loadu64(&yval);
if (val != incmax) {
printf("tskall: yval fault -- yval=%lld incmax=%lld\n",val,incmax);
err = 1;
}
if (! err)
printf("tskall: SUCCESS\n");
free(tsklist);
}
// main -- master control
int
main(void)
{
loopmax = LOOPMAX;
tidmax = TIDMAX;
inctype = 0;
tskall();
return 0;
}
Here is the Makefile. Sorry for the extra boilerplate:
# caslock/Makefile -- make file for caslock
#
# options:
# LOOPMAX -- maximum loops / thread
#
# TIDMAX -- maximum number of threads
#
# BARRIER -- generate fence/barrier instructions
# 0 -- none
# 1 -- use mfence everywhere
# 2 -- use lfence for acquire, sfence for release
#
# CASOFF -- disable CAS to prove diagnostic works
# 0 -- normal mode
# 1 -- inhibit CAS during X/Y increment
#
# CASINLINE -- inline the CAS functions
# 0 -- do _not_ inline
# 1 -- inline them (WARNING: this fails for gcc but works for clang!)
#
# CASPOST -- increment gate release mode
# 0 -- use fenced store
# 1 -- use CAS store (NOTE: not really required)
#
# INCBARRIER -- use extra barriers around increments
# 0 -- rely on CAS for barrier
# 1 -- add extra safety barriers immediately before increment of X/Y
#
# TID64 -- use 64 bit thread "id"s
# 0 -- use 32 bit
# 1 -- use 64 bit
#
# VPTR -- use volatile pointers in function definitions
# 0 -- use ordinary pointers
# 1 -- use volatile pointers (NOTE: not really required)
ifndef _CASLOCK_MK_
_CASLOCK_MK_ = 1
OLIST += caslock.o
ifndef LOOPMAX
LOOPMAX = 1000000
endif
ifndef TIDMAX
TIDMAX = 20
endif
ifndef BARRIER
BARRIER = 0
endif
ifndef CASINLINE
CASINLINE = 0
endif
ifndef CASOFF
CASOFF = 0
endif
ifndef CASPOST
CASPOST = 0
endif
ifndef INCBARRIER
INCBARRIER = 0
endif
ifndef TID64
TID64 = 0
endif
ifndef VPTR
VPTR = 0
endif
CFLAGS += -DLOOPMAX=$(LOOPMAX)
CFLAGS += -DTIDMAX=$(TIDMAX)
CFLAGS += -D_USE_BARRIER_=$(BARRIER)
CFLAGS += -D_USE_CASINLINE_=$(CASINLINE)
CFLAGS += -D_USE_CASOFF_=$(CASOFF)
CFLAGS += -D_USE_CASPOST_=$(CASPOST)
CFLAGS += -D_USE_INCBARRIER_=$(INCBARRIER)
CFLAGS += -D_USE_TID64_=$(TID64)
CFLAGS += -D_USE_VPTR_=$(VPTR)
STDLIB += -lpthread
ALL += caslock
CLEAN += caslock
OVRPUB := 1
ifndef OVRTOP
OVRTOP := $(shell pwd)
OVRTOP := $(dir $(OVRTOP))
endif
endif
# ovrlib/rules.mk -- rules control
#
# options:
# GDB -- enable debug symbols
# 0 -- normal
# 1 -- use -O0 and define _USE_GDB_=1
#
# CLANG -- use clang instead of gcc
# 0 -- use gcc
# 1 -- use clang
#
# BNC -- enable benchmarks
# 0 -- normal mode
# 1 -- enable benchmarks for function enter/exit pairs
ifdef OVRPUB
ifndef SDIR
SDIR := $(shell pwd)
STAIL := $(notdir $(SDIR))
endif
ifndef GENTOP
GENTOP := $(dir $(SDIR))
endif
ifndef GENDIR
GENDIR := $(GENTOP)/$(STAIL)
endif
ifndef ODIR
ODIR := $(GENDIR)
endif
PROTOLST := true
PROTOGEN := #true
endif
ifndef SDIR
$(error rules: SDIR not defined)
endif
ifndef ODIR
$(error rules: ODIR not defined)
endif
ifndef GENDIR
$(error rules: GENDIR not defined)
endif
ifndef GENTOP
$(error rules: GENTOP not defined)
endif
ifndef _RULES_MK_
_RULES_MK_ = 1
CLEAN += *.proto
CLEAN += *.a
CLEAN += *.o
CLEAN += *.i
CLEAN += *.dis
CLEAN += *.TMP
QPROTO := $(shell $(PROTOLST) -i -l -O$(GENTOP) $(SDIR)/*.c $(CPROTO))
HDEP += $(QPROTO)
###VPATH += $(GENDIR)
###VPATH += $(SDIR)
ifdef INCLUDE_MK
-include $(INCLUDE_MK)
endif
ifdef GSYM
CFLAGS += -gdwarf-2
endif
ifdef GDB
CFLAGS += -gdwarf-2
DFLAGS += -D_USE_GDB_
else
CFLAGS += -O2
endif
ifndef ZPRT
DFLAGS += -D_USE_ZPRT_=0
endif
ifdef BNC
DFLAGS += -D_USE_BNC_=1
endif
ifdef CLANG
CC := clang
endif
DFLAGS += -I$(GENTOP)
DFLAGS += -I$(OVRTOP)
CFLAGS += -Wall -Werror
CFLAGS += -Wno-unknown-pragmas
CFLAGS += -Wempty-body
CFLAGS += -fno-diagnostics-color
# NOTE: we now need this to prevent inlining (enabled at -O2)
ifndef CLANG
CFLAGS += -fno-inline-small-functions
endif
# NOTE: we now need this to prevent inlining (enabled at -O3)
CFLAGS += -fno-inline-functions
CFLAGS += $(DFLAGS)
endif
all: $(PREP) proto $(ALL)
%.o: %.c $(HDEP)
$(CC) $(CFLAGS) -c -o $*.o $<
%.i: %.c
cpp $(DFLAGS) -P $*.c > $*.i
%.s: %.c
$(CC) $(CFLAGS) -S -o $*.s $<
# build a library (type (2) build)
$(LIBNAME):: $(OLIST)
ar rv $# $(OLIST)
.PHONY: proto
proto::
$(PROTOGEN) -i -v -O$(GENTOP) $(SDIR)/*.c $(CPROTO)
.PHONY: clean
clean::
rm -f $(CLEAN)
.PHONY: help
help::
egrep '^#' Makefile
caslock:: $(OLIST) $(LIBLIST) $(STDLIB)
$(CC) $(CFLAGS) -o caslock $(OLIST) $(LIBLIST) $(STDLIB)
NOTE: I may have blown some of the asm constraints because when doing the CAS function as an inline, compiling with gcc produces incorrect results. However, clang works fine with inline. So, the default is that the CAS function is not inline. For consistency, I didn't use a different default for gcc/clang, even though I could.
Here's the disassembly of the relevant function with inline as built by gcc (this fails):
00000000004009c0 <incby_tidgate>:
4009c0: 31 c0 xor %eax,%eax
4009c2: f0 0f b1 3d 3a 1a 20 lock cmpxchg %edi,0x201a3a(%rip) # 602404 <tidgate>
4009c9: 00
4009ca: 0f 94 c2 sete %dl
4009cd: 84 d2 test %dl,%dl
4009cf: 75 23 jne 4009f4 <L01>
4009d1: 0f 1f 80 00 00 00 00 nopl 0x0(%rax)
4009d8:L00 64 48 8b 14 25 f8 ff mov %fs:0xfffffffffffffff8,%rdx
4009df: ff ff
4009e1: 83 42 04 01 addl $0x1,0x4(%rdx)
4009e5: f0 0f b1 3d 17 1a 20 lock cmpxchg %edi,0x201a17(%rip) # 602404 <tidgate>
4009ec: 00
4009ed: 0f 94 c2 sete %dl
4009f0: 84 d2 test %dl,%dl
4009f2: 74 e4 je 4009d8 <L00>
4009f4:L01 48 83 05 dc 17 20 00 addq $0x1,0x2017dc(%rip) # 6021d8 <ary+0xf8>
4009fb: 01
4009fc: 48 83 05 94 19 20 00 addq $0x1,0x201994(%rip) # 602398 <ary+0x2b8>
400a03: 01
400a04: c7 05 f6 19 20 00 00 movl $0x0,0x2019f6(%rip) # 602404 <tidgate>
400a0b: 00 00 00
400a0e: c3 retq
Here's the disassembly of the relevant function with inline as built by clang (this succeeds):
0000000000400990 <incby_tidgate>:
400990: 31 c0 xor %eax,%eax
400992: f0 0f b1 3d 3a 1a 20 lock cmpxchg %edi,0x201a3a(%rip) # 6023d4 <tidgate>
400999: 00
40099a: 0f 94 c0 sete %al
40099d: eb 1a jmp 4009b9 <L01>
40099f: 90 nop
4009a0:L00 64 48 8b 04 25 f8 ff mov %fs:0xfffffffffffffff8,%rax
4009a7: ff ff
4009a9: ff 40 04 incl 0x4(%rax)
4009ac: 31 c0 xor %eax,%eax
4009ae: f0 0f b1 3d 1e 1a 20 lock cmpxchg %edi,0x201a1e(%rip) # 6023d4 <tidgate>
4009b5: 00
4009b6: 0f 94 c0 sete %al
4009b9:L01 84 c0 test %al,%al
4009bb: 74 e3 je 4009a0 <L00>
4009bd: 48 ff 05 e4 17 20 00 incq 0x2017e4(%rip) # 6021a8 <ary+0xf8>
4009c4: 48 ff 05 9d 19 20 00 incq 0x20199d(%rip) # 602368 <ary+0x2b8>
4009cb: c7 05 ff 19 20 00 00 movl $0x0,0x2019ff(%rip) # 6023d4 <tidgate>
4009d2: 00 00 00
4009d5: c3 retq
4009d6: 66 2e 0f 1f 84 00 00 nopw %cs:0x0(%rax,%rax,1)
4009dd: 00 00 00

Segfault occurs due to one line of code in C file and entire program does not run

I've created a C program to write to a serial port (/dev/ttyS0) on an embedded ARM system. The kernel running on the embedded ARM system is Linux version 3.0.4, built with the same cross-compiler as the one listed below.
My cross-compiler is arm-linux-gcc (Buildroot 2011.08) 4.3.6, running on an Ubuntu x86_64 host (3.0.0-14-generic #23-Ubuntu SMP). I have used the stty utility to set up the serial port from the command line.
Mysteriously, it seems that the program will refuse to run on the embedded ARM system if a single line of code is present. If the line is removed, the program will run.
Here is a full code listing replicating the problem:
EDIT: I now close the file on error, as suggested in the comments below.
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdint.h>
#include <string.h>
#include <errno.h>
#include <termios.h>
int test();
void run_experiment();
int main()
{
run_experiment();
return 0;
}
void run_experiment()
{
printf("Starting program\n");
test();
}
int test()
{
int fd;
int ret;
fd = open("/dev/ttyS0", O_RDWR | O_NOCTTY);
printf("fd = %u\n", fd);
if (fd < 0)
{
close(fd);
return 0;
}
fcntl(fd, F_SETFL, 0);
printf("Now writing to serial port\n");
//TODO:
// segfault occurs due to line of code here
// removing this line causes the program to run properly
ret = write( fd, "test\r\n", sizeof("test\r\n") );
if (ret < 0)
{
close(fd);
return 0;
}
close(fd);
return 1;
}
The output of this program on the ARM system is the following:
Segmentation fault
However, if I remove the line listed above and recompile the program, the problem goes away, and the output is the following:
Starting program
fd = 3
Now writing to serial port
What could be going wrong here, and how do I debug the problem? Would this be an issue with the code, with the cross-compiler compiler, or with a version of the OS?
I have also tried various combinations of O_WRONLY and O_RDWR without O_NOCTTY when opening the file, but the problem still persists.
As suggested by #wildplasser in the comments below, I have replaced the test function with the following code, heavily based on the code at another site (http://www.warpspeed.com.au/cgi-bin/inf2html.cmd?..\html\book\Toolkt40\XPG4REF.INF+112).
However, the program still doesn't run, and I receive the mysterious Segmentation Fault again.
Here is the code:
int test()
{
int fh;
FILE *fp;
char *cp;
if (-1 == (fh = open("/dev/ttyS0", O_RDWR)))
{
perror("Unable to open");
return EXIT_FAILURE;
}
if (NULL == (fp = fdopen(fh, "w")))
{
perror("fdopen failed");
close(fh);
return EXIT_FAILURE;
}
for (cp = "hello world\r\n"; *cp; cp++)
fputc( *cp, fp);
fclose(fp);
return 0;
}
This is very mysterious, since using other programs that I have written, I can use the write() function in a similar fashion to write to sysfs files, without any problem.
HOWEVER, if the program is exactly in the same structure, then I cannot write to /dev/null.
BUT I can successfully write to a sysfs file using exactly the same program!
If the segfault occurred at a particular line in the function, then I would assume that the function call would be causing the segfault. However, the full program does not run!
UPDATE: To provide more information, here is the cross-compiler information used to build on ARM system:
$ arm-linux-gcc --v
Using built-in specs.
Target: arm-unknown-linux-uclibcgnueabi
Configured with: /media/RESEARCH/SAS2-version2/device-system/buildroot/buildroot-2011.08/output/toolchain/gcc-4.3.6/configure --prefix=/media/RESEARCH/SAS2-version2/device-system/buildroot/buildroot-2011.08/output/host/usr --build=x86_64-unknown-linux-gnu --host=x86_64-unknown-linux-gnu --target=arm-unknown-linux-uclibcgnueabi --enable-languages=c,c++ --with-sysroot=/media/RESEARCH/SAS2-version2/device-system/buildroot/buildroot-2011.08/output/host/usr/arm-unknown-linux-uclibcgnueabi/sysroot --with-build-time-tools=/media/RESEARCH/SAS2-version2/device-system/buildroot/buildroot-2011.08/output/host/usr/arm-unknown-linux-uclibcgnueabi/bin --disable-__cxa_atexit --enable-target-optspace --disable-libgomp --with-gnu-ld --disable-libssp --disable-multilib --enable-tls --enable-shared --with-gmp=/media/RESEARCH/SAS2-version2/device-system/buildroot/buildroot-2011.08/output/host/usr --with-mpfr=/media/RESEARCH/SAS2-version2/device-system/buildroot/buildroot-2011.08/output/host/usr --disable-nls --enable-threads --disable-decimal-float --with-float=soft --with-abi=aapcs-linux --with-arch=armv5te --with-tune=arm926ej-s --disable-largefile --with-pkgversion='Buildroot 2011.08' --with-bugurl=http://bugs.buildroot.net/
Thread model: posix
gcc version 4.3.6 (Buildroot 2011.08)
Here is the makefile that I am using to compile my code:
CC=arm-linux-gcc
CFLAGS=-Wall
datacollector: datacollector.o
clean:
rm -f datacollector datacollector.o
UPDATE: Using the debugging suggestions given in the comments and answers below, I found that the segfault was caused by including the \r escape sequence in the string. For some strange reason, the compiler doesn't like the \r escape sequence, and will cause a segfault without running the code.
If the \r escape sequence is removed, then the code runs as expected.
Thus, the offending line of code should be the following:
ret = write( fd, "test\n", sizeof("test\n") );
So for the record, a full test program that actually runs is the following (could someone comment?):
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdint.h>
#include <string.h>
#include <errno.h>
#include <termios.h>
int test();
void run_experiment();
int main()
{
run_experiment();
return 0;
}
void run_experiment()
{
printf("Starting program\n");
fflush(stdout);
test();
}
int test()
{
int fd;
int ret;
char *msg = "test\n";
// NOTE: This does not work and will cause a segfault!
// even if the fflush is called after each printf,
// the program will still refuse to run
//char *msg = "test\r\n";
fd = open("/dev/ttyS0", O_RDWR | O_NOCTTY);
printf("fd = %u\n", fd);
fflush(stdout);
if (fd < 0)
{
close(fd);
return 0;
}
fcntl(fd, F_SETFL, 0);
printf("Now writing to serial port\n");
fflush(stdout);
ret = write( fd, msg, strlen(msg) );
if (ret < 0)
{
close(fd);
return 0;
}
close(fd);
return 1;
}
EDIT: As an aside to all of this, is it better to use:
ret = write( fd, msg, sizeof(msg) );
or is it better to use:
ret = write( fd, msg, strlen(msg) );
Which is better? Is it better to use sizeof() or strlen()? It appears that some of the data in the string is truncated and not written to the serial port using the sizeof() function.
As I understand from Pavel's comment below, it is better to use strlen() if msg is declared as char*.
Moreover, it appears that gcc is not creating a proper binary when the escape sequence \r is being used to write to a tty.
Referring to the last test program given in my post above, the following line of code causes a segfault without the program running:
char *msg = "test\r\n";
As suggested by Igor in the comments, I have run the gdb debugger on the binary with the offending line of code. I had to compile the program with the -g switch.
The gdb debugger is being run natively on the ARM system, and all binaries are being built for the ARM architecture on the host using the same Makefile. All binaries are being built using the arm-linux-gcc cross-compiler.
The output of gdb (running natively on the ARM system) is as follows:
GNU gdb 6.8
Copyright (C) 2008 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "arm-unknown-linux-uclibcgnueabi"...
"/programs/datacollector": not in executable format: File format not recognized
(gdb) run
Starting program:
No executable file specified.
Use the "file" or "exec-file" command.
(gdb) file datacollector
"/programs/datacollector": not in executable format: File format not recognized
(gdb)
However, if I change the single line of code to the following, the binary compiles and runs properly. Note that the \r escape sequence is missing:
char *msg = "test\n";
Here is the output of gdb after changing the single line of code:
GNU gdb 6.8
Copyright (C) 2008 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "arm-unknown-linux-uclibcgnueabi"...
(gdb) run
Starting program: /programs/datacollector
Starting program
fd = 4
Now writing to serial port
test
Program exited normally.
(gdb)
UPDATE:
As suggested by Zack in an answer below, I have now ran a test program on the embedded
Linux system. Although Zack gives a detailed script to run on the embedded system, I was
unable to run the script due to the lack of development tools (compiler and headers) installed in the root file system.
In lieu of installing these tools, I simply compiled the nice test program that Zack provided in the script and
used the strace utility. The strace utility was run on the embedded system.
At last, I think that I understand what is happening.
The bad binary was transferred to the embedded system over FTP, using an SPI-to-Ethernet bridge (KSZ8851SNL).
There is a driver for the KSZ8851SNL in the Linux kernel.
It appears that either the Linux kernel driver, the proftpd server software running on the embedded system, or the actual hardware itself (KSZ8851SNL)
was somehow corrupting the binary. The binary runs well on the embedded system.
Here is the output of strace on the testz binary transferred to the embedded Linux system over the Ethernet serial link:
Bad binary tests:
# strace ./testz /dev/null
execve("./testz", ["./testz", "/dev/null"], [/* 17 vars */]) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|0x4000000, -1, 0) = 0x40089000
--- SIGSEGV (Segmentation fault) # 0 (0) ---
+++ killed by SIGSEGV +++
Segmentation fault
# strace ./testz /dev/ttyS0
execve("./testz", ["./testz", "/dev/ttyS0"], [/* 17 vars */]) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|0x4000000, -1, 0) = 0x400ca000
--- SIGSEGV (Segmentation fault) # 0 (0) ---
+++ killed by SIGSEGV +++
Segmentation fault
#
Here is the output of strace on the testz binary transferred on SD card to the embedded Linux system:
Good binary tests:
# strace ./testz /dev/null
execve("./testz", ["./testz", "/dev/null"], [/* 17 vars */]) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|0x4000000, -1, 0) = 0x40058000
open("/lib/libc.so.0", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0755, st_size=298016, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|0x4000000, -1, 0) = 0x400b8000
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0\240\230\0\0004\0\0\0"..., 4096) = 4096
mmap2(NULL, 348160, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40147000
mmap2(0x40147000, 290576, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED, 3, 0) = 0x40147000
mmap2(0x40196000, 4832, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0x47) = 0x40196000
mmap2(0x40198000, 14160, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x40198000
close(3) = 0
munmap(0x400b8000, 4096) = 0
stat("/lib/ld-uClibc.so.0", {st_mode=S_IFREG|0755, st_size=25296, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|0x4000000, -1, 0) = 0x400c4000
set_tls(0x400c4470, 0x400c4470, 0x4007b088, 0x400c4b18, 0x40) = 0
mprotect(0x40196000, 4096, PROT_READ) = 0
mprotect(0x4007a000, 4096, PROT_READ) = 0
ioctl(0, SNDCTL_TMR_TIMEBASE or TCGETS, {B115200 opost isig icanon echo ...}) = 0
ioctl(1, SNDCTL_TMR_TIMEBASE or TCGETS, {B115200 opost isig icanon echo ...}) = 0
open("/dev/null", O_RDWR|O_NOCTTY|O_NONBLOCK) = 3
write(3, "1\n", 2) = 2
write(3, "12\n", 3) = 3
write(3, "123\n", 4) = 4
write(3, "1234\n", 5) = 5
write(3, "12345\n", 6) = 6
write(3, "1\r\n", 3) = 3
write(3, "12\r\n", 4) = 4
write(3, "123\r\n", 5) = 5
write(3, "1234\r\n", 6) = 6
close(3) = 0
exit_group(0) = ?
# strace ./testz /dev/ttyS0
execve("./testz", ["./testz", "/dev/ttyS0"], [/* 17 vars */]) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|0x4000000, -1, 0) = 0x400ed000
open("/lib/libc.so.0", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0755, st_size=298016, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|0x4000000, -1, 0) = 0x40176000
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0\240\230\0\0004\0\0\0"..., 4096) = 4096
mmap2(NULL, 348160, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x40238000
mmap2(0x40238000, 290576, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED, 3, 0) = 0x40238000
mmap2(0x40287000, 4832, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED, 3, 0x47) = 0x40287000
mmap2(0x40289000, 14160, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x40289000
close(3) = 0
munmap(0x40176000, 4096) = 0
stat("/lib/ld-uClibc.so.0", {st_mode=S_IFREG|0755, st_size=25296, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|0x4000000, -1, 0) = 0x400d1000
set_tls(0x400d1470, 0x400d1470, 0x40084088, 0x400d1b18, 0x40) = 0
mprotect(0x40287000, 4096, PROT_READ) = 0
mprotect(0x40083000, 4096, PROT_READ) = 0
ioctl(0, SNDCTL_TMR_TIMEBASE or TCGETS, {B115200 opost isig icanon echo ...}) = 0
ioctl(1, SNDCTL_TMR_TIMEBASE or TCGETS, {B115200 opost isig icanon echo ...}) = 0
open("/dev/ttyS0", O_RDWR|O_NOCTTY|O_NONBLOCK) = 3
write(3, "1\n", 21
) = 2
write(3, "12\n", 312
) = 3
write(3, "123\n", 4123
) = 4
write(3, "1234\n", 51234
) = 5
write(3, "12345\n", 612345
) = 6
write(3, "1\r\n", 31
) = 3
write(3, "12\r\n", 412
) = 4
write(3, "123\r\n", 5123
) = 5
write(3, "1234\r\n", 61234
) = 6
close(3) = 0
exit_group(0) = ?
EDIT: Read on for gory details, but the quick answer is, your FTP client is corrupting your program. This is an intentional feature of FTP, which can be turned off by typing binary at the FTP prompt before get whatever or put whatever. If you're using a graphical FTP client it should have a checkbox somewhere with the same effect. Or switch to scp, which does not have this inconvenient feature.
First off, there is no difference in the generated assembly code
between (one of the) working object files and the broken object file.
$ objdump -dr dc-good.o > dc-good.s
$ objdump -dr dc-bad.o > dc-bad.s
$ diff -u dc-good.s dc-bad.s
--- dc-good.s 2012-01-21 08:20:05.318518596 -0800
+++ dc-bad.s 2012-01-21 08:20:10.954566852 -0800
## -1,5 +1,5 ##
-dc-good.o: file format elf32-littlearm
+dc-bad.o: file format elf32-littlearm
Disassembly of section .text:
In fact, there are only two bytes that differ between the good and
bad object files. (You misunderstood what I was asking for with
"test\r\n" versus "testX\n": I wanted the two strings to be the
same length, so that everything would have the same offset in the
object files. Fortunately, your compiler padded the shorter string to
the same length as the longer string, so everything has the same
offset anyway.)
$ hd dc-good.o > dc-good.x
$ hd dc-bad.o > dc-bad.x
$ diff -u1 dc-good.x dc-bad.x
--- dc-good.x 2012-01-21 08:17:28.713174977 -0800
+++ dc-bad.x 2012-01-21 08:17:39.129264489 -0800
## -154,3 +154,3 ##
00000990 53 74 61 72 74 69 6e 67 20 70 72 6f 67 72 61 6d |Starting program|
-000009a0 00 00 00 00 74 65 73 74 58 0a 00 00 2f 64 65 76 |....testX.../dev|
+000009a0 00 00 00 00 74 65 73 74 58 0d 0a 00 2f 64 65 76 |....testX.../dev|
000009b0 2f 74 74 79 53 30 00 00 66 64 20 3d 20 25 75 0a |/ttyS0..fd = %u.|
## -223,3 +223,3 ##
00000de0 61 72 69 65 73 2f 64 61 74 61 63 6f 6c 6c 65 63 |aries/datacollec|
-00000df0 74 6f 72 2d 62 61 64 2d 62 69 6e 61 72 79 2d 32 |tor-bad-binary-2|
+00000df0 74 6f 72 2d 62 61 64 2d 62 69 6e 61 72 79 2d 31 |tor-bad-binary-1|
00000e00 00 46 49 4c 45 00 5f 5f 73 74 61 74 65 00 5f 5f |.FILE.__state.__|
The first difference is the difference that should be there: 74 65 73
74 58 0a 00 00 is the correct encoding of "test\n" (with one byte
of padding), 74 65 73 74 58 0d 0a 00 is the correct encoding of
"test\r\n". The other difference appears to be debugging
information: the name of the directory in which you compiled the
programs. This is harmless.
The object files are as they should be, so at this point we can rule
out a bug in the compiler or the assembler. Now let's look at the
executables.
$ hd dc-good > dc-good.xe
$ hd dc-bad > dc-bad.xe
$ diff -u1 dc-good.xe dc-bad.xe
--- dc-good.xe 2012-01-21 08:31:33.456437417 -0800
+++ dc-bad.xe 2012-01-21 08:31:38.388480238 -0800
## -120,3 +120,3 ##
00000770 f0 af 1b e9 53 74 61 72 74 69 6e 67 20 70 72 6f |....Starting pro|
-00000780 67 72 61 6d 00 00 00 00 74 65 73 74 58 0a 00 00 |gram....testX...|
+00000780 67 72 61 6d 00 00 00 00 74 65 73 74 58 0d 0a 00 |gram....testX...|
00000790 2f 64 65 76 2f 74 74 79 53 30 00 00 66 64 20 3d |/dev/ttyS0..fd =|
## -373,3 +373,3 ##
00001750 63 6f 6c 6c 65 63 74 6f 72 2d 62 61 64 2d 62 69 |collector-bad-bi|
-00001760 6e 61 72 79 2d 32 00 46 49 4c 45 00 5f 5f 73 74 |nary-2.FILE.__st|
+00001760 6e 61 72 79 2d 31 00 46 49 4c 45 00 5f 5f 73 74 |nary-1.FILE.__st|
00001770 61 74 65 00 5f 5f 67 63 73 00 73 74 64 6f 75 74 |ate.__gcs.stdout|
Same two differences, different offsets within the executable. This
is also as it should be. We can rule out a bug in the linker as well
(if it was screwing up the address of the string, it would have to be
screwing it up the same way in both executables and they both ought to
crash).
At this point I think we are looking at a bug in your C library or
kernel. To pin it down further, I would like you to try this test
script. Run it as sh testz.sh on the ARM board, and send us the
complete output.
#! /bin/sh
set -e
cat >testz.c <<\EOF
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#define W(f, s) write(f, s, sizeof s - 1)
int
main(int ac, char **av)
{
int f;
if (ac != 2) return 2;
f = open(av[1], O_RDWR|O_NOCTTY|O_NONBLOCK);
if (f == -1) return 1;
W(f, "1\n");
W(f, "12\n");
W(f, "123\n");
W(f, "1234\n");
W(f, "12345\n");
W(f, "1\r\n");
W(f, "12\r\n");
W(f, "123\r\n");
W(f, "1234\r\n");
close(f);
return 0;
}
EOF
arm-linux-gcc -Wall -g testz.c -o testz
set +e
strace ./testz /dev/null
echo ----
strace ./testz /dev/ttyS0
echo ----
exit 0
I've looked at the damaged binary you provided and now I know what's wrong.
$ ls -l testz*
-rwxr-x--- 1 zack zack 7528 Dec 31 1979 testz-bad
-rwxr-x--- 1 zack zack 7532 Jan 21 16:35 testz-good
Ignore the odd datestamp; see how the -bad version is four bytes smaller than the -good version? There were exactly four \r characters in the source code. Let's have a look at the differences in the hex dumps. I've pulled out the interesting bit of the diff and shuffled it around a little to make it easier to see what's going on.
00000620 00 00 00 00 31 32 33 34 0a 00 00 00 31 32 33 34 |....1234....1234|
-00000630 35 0a 00 00 31 0d 0a 00 31 32 0d 0a 00 00 00 00 |5...1...12......|
+00000630 35 0a 00 00 31 0a 00 31 32 0a 00 00 00 00 31 32 |5...1..12.....12|
-00000640 31 32 33 0d 0a 00 00 00 31 32 33 34 0d 0a 00 00 |123.....1234....|
+00000640 33 0a 00 00 00 31 32 33 34 0a 00 00 00 00 00 00 |3....1234.......|
-00000650 00 00 00 00 68 84 00 00 1c 84 00 00 00 00 00 00 |....h...........|
+00000650 68 84 00 00 1c 84 00 00 00 00 00 00 01 00 00 00 |h...............|
The file transfer is replacing 0d 0a (that is, \r\n) sequences with 0a (just \n). This causes everything after this point in the file to be displaced four bytes from where it's supposed to be. The code is before this point, and so are all the ELF headers that the kernel looks at, which is why you don't get
execve("./testz-bad", ["./testz-bad", "/dev/null"], [/* 36 vars */]) = -1 ENOEXEC (Exec format error)
from the test script; instead, you get a segfault inside the dynamic loader, because the DYNAMIC segment (which tells the dynamic loader what to do) is after the displacement starts.
$ readelf -d testz-bad 2> /dev/null
Dynamic section at offset 0x660 contains 13 entries:
Tag Type Name/Value
0x00000035 (<unknown>: 35) 0xc
0x0000832c (<unknown>: 832c) 0xd
0x00008604 (<unknown>: 8604) 0x19
0x00010654 (<unknown>: 10654) 0x1b
0x00000004 (HASH) 0x1a
0x00010658 (<unknown>: 10658) 0x1c
0x00000004 (HASH) 0x4
0x00008108 (<unknown>: 8108) 0x5
0x0000825c (<unknown>: 825c) 0x6
0x0000815c (<unknown>: 815c) 0xa
0x00000098 (<unknown>: 98) 0xb
0x00000010 (SYMBOLIC) 0x15
0x00000000 (NULL) 0x3
Contrast:
$ readelf -d testz-good
Dynamic section at offset 0x660 contains 18 entries:
Tag Type Name/Value
0x00000001 (NEEDED) Shared library: [libc.so.0]
0x0000000c (INIT) 0x832c
0x0000000d (FINI) 0x8604
0x00000019 (INIT_ARRAY) 0x10654
0x0000001b (INIT_ARRAYSZ) 4 (bytes)
0x0000001a (FINI_ARRAY) 0x10658
0x0000001c (FINI_ARRAYSZ) 4 (bytes)
0x00000004 (HASH) 0x8108
0x00000005 (STRTAB) 0x825c
0x00000006 (SYMTAB) 0x815c
0x0000000a (STRSZ) 152 (bytes)
0x0000000b (SYMENT) 16 (bytes)
0x00000015 (DEBUG) 0x0
0x00000003 (PLTGOT) 0x10718
0x00000002 (PLTRELSZ) 56 (bytes)
0x00000014 (PLTREL) REL
0x00000017 (JMPREL) 0x82f4
0x00000000 (NULL) 0x0
The debugging information is also after the displacement, which is why gdb didn't like the program.
So why this very particular corruption? It's not a bug in anything; it's an intentional feature of your FTP client, which defaults to transferring files in "text mode", which means (among other things) that it converts DOS-style line endings (\r\n) to Unix-style (\n). Because that would be what you wanted if this were 1991 and you were transferring text files off your IBM PC to your institutional file server. It is basically never what is wanted nowadays, even if you are moving text files around. Fortunately, you can turn it off: just type binary at the FTP prompt before the file transfer commands. *Un*fortunately, as far as I know there is no way to make that stick; you have to do that every time. I recommend switching to scp, which always transfers files verbatim and is also easier to operate from build automation.
First things first - the fact that you only see the seg fault is NOT indicative that the program failed to run at all. What happens is that the output from the printf calls is line buffered, and when the program seg faults, it's never written out.
If you add
fflush(stdout);
after every printf, you'll see your output prior to the segfault.
Now, in your original program, what's the point of the fcntl(fd, F_SETFL, 0); call? What are you trying to achieve with it? Are you trying to turn off non-blocking mode? What if you don't make that call?
As to your second test, I see that you are using perror, but again the lack of error messages doesn't tell you that the program isn't running - it just tells you that you didn't get any error messages, and you still aren't flushing stdout, so you'll never see the printf from run_experiment.
I also see that in your second test you're doing an fdopen with read mode, then trying to write to that FILE pointer. While that certainly shouldn't crash, it also certainly shouldn't work.
Now, outside of your program, are you sure the serial port works OK? Try doing 'cat > /dev/ttyS0' and see what happens, just to be sure it's not something wonky with the hardware.

Resources