Can't calculate end address of the section at linker script - c

I need to make firmware image with 8K size. Firmware image should have a version word at the end. Area between data section and version section must be filled with 0xFF.
This linker script works fine if test and data sections not too big.
MEMORY
{
PM_RAM (rwx): ORIGIN = 0x00000000, LENGTH = 20K
DM_RAM (rwx) : ORIGIN = 0x0001E000, LENGTH = 6K
PM_ROM (rx) : ORIGIN = 0x00080000, LENGTH = 256K
PM_OTP (rx): ORIGIN = 0x000C0000, LENGTH = 8K - VERSION_SIZE
PM_OTP_VER (rx): ORIGIN = 0x000C0000 + 8K - VERSION_SIZE, LENGTH = VERSION_SIZE
}
SECTIONS {
.text :
{
*(.text*)
. = ALIGN(4);
} >PM_OTP
_etext = .;
.data : {
_sdata = .;
_ldata = LOADADDR(.data);
*(.data .data.*)
*(.sdata .sdata.*)
. = ALIGN(4);
} >DM_RAM AT>PM_OTP
. = ALIGN(4);
_edata = .;
.fill : {
BYTE(0xFF)
FILL(0xFF)
. = LENGTH(PM_OTP) - SIZEOF(.text) - SIZEOF(.data) - 1;
} > PM_OTP = 0xFF
.version : {
. = ALIGN(4);
__version_start__ = .;
KEEP(*(.version .version*))
__version_end__ = .;
} > PM_OTP_VER
.bss : {
_sbss = .;
*(.bss .bss.*)
. = ALIGN(4);
*(.sbss .sbss.*)
. = ALIGN(4);
} >DM_RAM
. = ALIGN(4);
_ebss = .;
_end = .;
.stack ORIGIN(DM_RAM) + LENGTH(DM_RAM) - STACK_SIZE : {
PROVIDE(__STACK_START__ = .);
. += STACK_SIZE;
PROVIDE(__C_STACK_TOP__ = .);
} >DM_RAM
}
But if text + data close to 8K I get the error:
cannot move location counter backwards (from 00000000000c2095 to 00000000000c1ffb)
If change this string:
. = LENGTH(PM_OTP) - SIZEOF(.text) - SIZEOF(.data) - 1;
by hardcoded value like:
. = 0x3f;
Build completes successfully and all OK.
What is wrong with LENGTH(PM_OTP) - SIZEOF(.text) - SIZEOF(.data) calculations?
In listing file it corresponds to 0x3f.
UPDATE
I have linker script
STACK_SIZE = 1024;
TEST_MEMORY_SIZE = 4;
VERSION_SIZE = 4;
MEMORY
{
PM_RAM (rwx): ORIGIN = 0xFD000000, LENGTH = 120K
DM_RAM (rwx) : ORIGIN = 0xFD01E000, LENGTH = 6K
PM_ROM (rx) : ORIGIN = 0xFD080000, LENGTH = 256K
PM_OTP (rx): ORIGIN = 0xFD0C0000, LENGTH = 8K - VERSION_SIZE
PM_OTP_VER (rx): ORIGIN = 0xFD0C0000 + 8K - VERSION_SIZE, LENGTH = VERSION_SIZE
}
SECTIONS {
.init ORIGIN(PM_OTP):
{
. = 0x000;
*(.init.startup)
. = ALIGN(4);
*(.init.traphandler)
. = ALIGN(4);
} >PM_OTP
.text :
{
*(.text*)
. = ALIGN(4);
*(.text.emulate*)
. = ALIGN(4);
*(.rodata .rodata.*)
. = ALIGN(4);
} >PM_OTP
_etext = .;
.data : {
_sdata = .;
_ldata = LOADADDR(.data);
*(.data .data.*)
*(.sdata .sdata.*)
. = ALIGN(4);
} >DM_RAM AT>PM_OTP
. = ALIGN(4);
_edata = .;
.fill :
{
BYTE(0xFF)
FILL(0xFF)
} > PM_OTP = 0xFF
.version :
{
. = ALIGN(4);
__version_start__ = .;
KEEP(*(.version .version*))
__version_end__ = .;
} > PM_OTP_VER
.bss : {
_sbss = .;
*(.bss .bss.*)
. = ALIGN(4);
*(.sbss .sbss.*)
. = ALIGN(4);
} >DM_RAM
. = ALIGN(4);
_ebss = .;
_end = .;
.stack ORIGIN(DM_RAM) + LENGTH(DM_RAM) - STACK_SIZE : {
PROVIDE(__STACK_START__ = .);
. += STACK_SIZE;
PROVIDE(__C_STACK_TOP__ = .);
} >DM_RAM
_test_memory = ORIGIN(DM_RAM) + LENGTH(DM_RAM);
_loader_descr = _test_memory + TEST_MEMORY_SIZE;
}
Build terminates successfully and I get:
text data bss dec hex
7700 321 1056 9077 2375
Binary look:
00001f10 94 05 0c fd 98 05 0c fd 9c 05 0c fd a0 05 0c fd |................|
00001f20 a4 05 0c fd a8 05 0c fd 00 00 00 00 00 00 00 00 |................|
00001f30 00 00 00 00 ac 05 0c fd 00 00 00 00 00 00 00 00 |................|
00001f40 00 00 00 00 00 00 00 00 00 00 00 00 01 01 00 00 |................|
00001f50 ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
00001f60 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
*
00001ff0 00 00 00 00 00 00 00 00 00 00 00 00 60 b8 ee 61 |............`..a|
00002000
There are zeros here from 00001f60 to the end. But I need 0xFF.
I added line:
. = LENGTH(PM_OTP) - SIZEOF(.init) - SIZEOF(.text) - SIZEOF(.data);`
In setion .fill after FILL(0xFF) and got error:
cannot move location counter backwards (from 00000000fd0c2025 to 00000000fd0c1ffc)
Very strange. But ok. I gradually reduce my code volume. And at a certain code decrease, the build begins to be successful.
text data bss dec hex
7656 536 1056 9248 2420
Binary look:
00001ef0 a0 05 0c fd a4 05 0c fd a8 05 0c fd 00 00 00 00 |................|
00001f00 00 00 00 00 00 00 00 00 ac 05 0c fd 00 00 00 00 |................|
00001f10 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
00001f20 01 01 00 00 ff ff ff ff ff ff ff ff ff ff ff ff |................|
00001f30 ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
*
00001ff0 ff ff ff ff ff ff ff ff ff ff ff ff 50 bb ee 61 |............P..a|
00002000
There is a lot of 0xFF in the end. And offset is correctly calculated, because 0xFF goes up to version. But other side 0xFF is empty area that could be filled with useful code.

0x00000000000c2095 is apparently beyond PM_OTP (rx): ORIGIN = 0x000C0000, LENGTH = 8K - VERSION_SIZE. Your firmware is too large.

Related

Confusing error when writing structure to file in C

i'm trying to write a structure to a dat file, everything works fine but the data in the dat file turns into random characters (it's the same in other formats like txt)
#include <stdio.h>
#include <string.h>
struct Product
{
char ProId[20];
char ProName[30];
float Price;
int Quantity;
int CatId;
};
int main(){
FILE *f;
f = fopen ("Product.dat", "w+");
if (f == NULL)
{
fprintf(stderr, "\nError opened file\n");
}
Product p1;
strcpy(p1.ProId, "1");
strcpy(p1.ProName, "Candy");
p1.Price = 4.5;
p1.Quantity = 5;
p1.CatId = 1;
fwrite(&p1, sizeof(p1), 1, f);
fclose(f);
return(0);
}
The data in Product.dat:
1 u ÿÿÿÿCandy Ù$# #
I tried searching for this error but to no avail. Please help me, what is wrong?
Thank you.
I compile and ran your code on my WSL2, and met the same problem, here is the hexdump:
00000000 31 00 00 00 00 00 00 00 e7 6a 5b 66 fd 7f 00 00 |1........j[f....|
00000010 e6 6a 5b 66 43 61 6e 64 79 00 d4 1a 18 56 00 00 |.j[fCandy....V..|
00000020 e8 32 96 dd 70 7f 00 00 80 12 d4 1a 18 56 00 00 |.2..p........V..|
00000030 00 00 00 00 00 00 90 40 05 00 00 00 01 00 00 00 |.......#........|
The string "Candy" is indeed copied to the struct.
I assume the reason it is polluted by some trash data is because this struct is allocated on stack.
So I made a clean up after the declaration of this struct, and here is the effect:
00000000 31 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |1...............|
00000010 00 00 00 00 43 61 6e 64 79 00 00 00 00 00 00 00 |....Candy.......|
00000020 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
00000030 00 00 00 00 00 00 90 40 05 00 00 00 01 00 00 00 |.......#........|
Here is the code I modified, note the call to memset():
#include <stdio.h>
#include <string.h>
struct Product
{
char ProId[20];
char ProName[30];
float Price;
int Quantity;
int CatId;
};
int main(){
FILE *f;
f = fopen ("Product.dat", "w+");
if (f == NULL)
{
fprintf(stderr, "\nError opened file\n");
}
struct Product p1;
memset(&p1, 0, sizeof(p1));
strcpy(p1.ProId, "1");
strcpy(p1.ProName, "Candy");
p1.Price = 4.5;
p1.Quantity = 5;
p1.CatId = 1;
fwrite(&p1, sizeof(p1), 1, f);
fclose(f);
return(0);
}

Vector of tables, flatbuffers C

I'm experimenting with flatbuffers, in C, just getting a feel for it. I'm having a weird issue. Well, a couple more than likely.
I think I'm building the flatbuffers and adding all the correct components in the correct manner. However, when I try to read a vector of tables, _vec_len(T) is returning size 0, even though when I hexdump the buffer I can clearly see the said tables being added to/removed from the buffer.
I've used more methods of getting the vector of tables to be readable than I can count. Most have ended with type mismatch compile errors. None of the ones that have actually compiled have resulted in _vec_len(T) returning anything other than 0.
I've also tried using _vec_at(0), but this causes an assertion to fail stating that the given index is out of range.
At this point, I'm either: a) missing something really simple, b) building the flatbuffer completely wrong, or c) making assumptions about how flatbuffers are built that are just plain incorrect.
I have attached my schema and source code, as well as debugging output from the program.
(I realized after posting that the code blocks dont include line numbers.)
The line that seems to be failing is size_t hops_vec_len = TransitHop_vec_len(hopsTable); in the C code, near the bottom. Either that, or the entire block starting at // test_root routing near the top is wrong.
By my understanding, using _start(B) and _end(B), everything that gets added to said component of the schema should be added to the buffer and placed into the v-tables for access automatically.
So, I'm not sure where to go. Can anyone point to what I'm doing wrong/misunderstanding? Any help is greatly appreciated.
flatbuffers_test.fbs
table Ping {
sent:uint;
}
table Pong {
sent:uint;
}
union Type {
Ping,
Pong,
}
enum ServerTypes:ubyte {
mach = 0,
cnc,
mill,
press,
drill,
controller,
unknown,
}
table TransitHop {
serverID: uint64;
serverType: ServerTypes;
}
table TransitHeader {
from: TransitHop;
to: TransitHop;
hops: [TransitHop];
}
table TransitSequence {
transitID: uint64;
begin: uint8;
current: uint8;
total: uint8;
}
table TestRoot {
type: Type;
sequence: TransitSequence;
routing: TransitHeader;
}
root_type TestRoot;
flatbuffers_test.c
#include <stdio.h>
#include "flatcc/support/hexdump.h"
#include "flatbuffers_test_builder.h"
// This allows us to verify result in optimized builds.
#define test_assert(x) do { if (!(x)) { assert(0); return -1; }} while(0)
int main(void) {
flatcc_builder_t builder, *B;
B = &builder;
void *buff = NULL;
size_t size;
FILE *fh = NULL;
flatcc_builder_init(B);
printf("TestRoot - Routing\n\n");
TestRoot_start_as_root(B);
{
// test_root type
{
Ping_ref_t testType = Ping_create(B, 0); // the 0 here would be the time the ping is sent
TestRoot_type_add(B, Type_as_Ping(testType));
}
// test_root sequence
{
TransitSequence_ref_t sequence = TransitSequence_create(B, 1, 1, 1, 1);
TestRoot_sequence_add(B, sequence);
}
// test_root routing
{
TransitHeader_start(B);
{
TransitHop_ref_t hopFrom = 0;
TransitHop_ref_t hopTo = 0;
TransitHeader_hops_start(B);
{
hopFrom = TransitHop_create(B, 1, ServerTypes_mach);
TransitHop_create(B, 2, ServerTypes_controller);
TransitHop_create(B, 3, ServerTypes_controller);
hopTo = TransitHop_create(B, 4, ServerTypes_cnc);
}
TransitHop_ref_t hops = TransitHeader_hops_end(B);
TransitHeader_from_add(B, hopFrom);
TransitHeader_to_add(B, hopTo);
TransitHeader_hops_add(B, hops);
}
TransitHeader_ref_t routing = TransitHeader_end(B);
TestRoot_routing_add(B, routing);
}
}
TestRoot_end_as_root(B);
buff = flatcc_builder_finalize_aligned_buffer(&builder, &size);
printf("flatbuffer size: %ld\n", size);
TestRoot_table_t test_root = TestRoot_as_root(buff);
test_assert(test_root != 0);
int type_present = TestRoot_type_is_present(test_root);
int sequence_present = TestRoot_sequence_is_present(test_root);
int routing_present = TestRoot_routing_is_present(test_root);
printf(" type present: %d\n", type_present);
if (type_present == 1) {
Type_union_type_t type = TestRoot_type_type(test_root);
printf("\ttype: %s\n", Type_type_name(type));
}
printf("sequence present: %d\n", sequence_present);
if (sequence_present == 1) {
TransitSequence_table_t sequence = TestRoot_sequence(test_root);
printf("\ttransitID: %ld\n", TransitSequence_transitID(sequence));
printf("\t begin: %d\n", TransitSequence_begin(sequence));
printf("\t current: %d\n", TransitSequence_current(sequence));
printf("\t total: %d\n", TransitSequence_total(sequence));
}
printf(" routing present: %d\n", routing_present);
if (routing_present == 1) {
TransitHeader_table_t routing = TestRoot_routing(test_root);
TransitHop_table_t from = TransitHeader_from(routing);
TransitHop_table_t to = TransitHeader_to(routing);
TransitHop_vec_t hopsTable = TransitHeader_hops(routing);
test_assert(hopsTable != 0);
ServerTypes_enum_t serverTypeFrom = TransitHop_serverType(from);
ServerTypes_enum_t serverTypeTo = TransitHop_serverType(to);
printf("\tfrom: %ld\t%s\n", TransitHop_serverID(from), ServerTypes_name(serverTypeFrom));
printf("\t to: %ld\t%s\n", TransitHop_serverID(to), ServerTypes_name(serverTypeTo));
int hops_present = TransitHeader_hops_is_present(routing);
if (hops_present == 1) {
// // this causes [Assertion `flatbuffers_vec_len(vec) > (i) && "index out of range"' failed.] ?????
// TransitHop_table_t hopsTableZero = TransitHop_vec_at(hopsTable, 0);
//
// test_assert(hopsTableZero == 0);
size_t hops_vec_len = TransitHop_vec_len(hopsTable);
printf("\thops: %lu\n", hops_vec_len);
for (size_t i = 0; i < hops_vec_len; i++) {
printf("\t\t%lu\n", i);
}
}
}
fh = fopen("/tmp/fbhd-test1", "w");
if (fh == NULL) {
printf("Cannot open /tmp/fbhd-test1 for writing\n\n");
return -1;
} else {
hexdump(NULL, buff, size, fh);
}
fclose(fh);
flatcc_builder_aligned_free(buff);
return 0;
}
debug1.txt
Program output with the vector of tables being added to the flatbuffer:
TestRoot - Routing
flatbuffer size: 188
type present: 1
type: Ping
sequence present: 1
transitID: 1
begin: 1
current: 1
total: 1
routing present: 1
from: 1 mach
to: 4 cnc
hops: 0
00000000 0c 00 00 00 54 45 53 54 00 00 00 00 5c ff ff ff |....TEST....\...|
00000010 01 00 00 00 70 00 00 00 5c 00 00 00 04 00 00 00 |....p...\.......|
00000020 7a ff ff ff 0c 00 00 00 3c 00 00 00 08 00 00 00 |z.......<.......|
00000030 00 00 00 00 96 ff ff ff 04 00 00 00 00 00 00 00 |................|
00000040 01 00 00 00 a6 ff ff ff 03 00 00 00 00 00 00 00 |................|
00000050 05 00 00 00 b6 ff ff ff 02 00 00 00 00 00 00 00 |................|
00000060 05 00 00 00 cc ff ff ff 01 00 00 00 00 00 00 00 |................|
00000070 00 00 00 00 e8 ff ff ff 01 00 00 00 00 00 00 00 |................|
00000080 01 01 01 00 fc ff ff ff 04 00 04 00 0c 00 0f 00 |................|
00000090 04 00 0c 00 0d 00 0e 00 06 00 0c 00 04 00 08 00 |................|
000000a0 0d 00 04 00 0c 00 0a 00 10 00 08 00 0c 00 04 00 |................|
000000b0 0c 00 14 00 04 00 08 00 0c 00 10 00 |............|
Program output without the vector of tables being added to the flatbuffer:
TestRoot - Routing
flatbuffer size: 74
type present: 1
type: Ping
sequence present: 1
transitID: 1
begin: 1
current: 1
total: 1
routing present: 0
00000000 0c 00 00 00 54 45 53 54 00 00 00 00 cc ff ff ff |....TEST........|
00000010 01 00 00 00 18 00 00 00 04 00 00 00 e8 ff ff ff |................|
00000020 01 00 00 00 00 00 00 00 01 01 01 00 fc ff ff ff |................|
00000030 04 00 04 00 0c 00 0f 00 04 00 0c 00 0d 00 0e 00 |................|
00000040 0a 00 10 00 04 00 08 00 0c 00 |..........|
Program output when trying to use _vec_at(0):
TestRoot - Routing
flatbuffer size: 188
type present: 1
type: Ping
sequence present: 1
transitID: 1
begin: 1
current: 1
total: 1
routing present: 1
from: 1 mach
to: 4 cnc
flatbuffers_test: /home/vadtec/schema/generated/flatbuffers_test_reader.h:192: TransitHop_vec_at: Assertion `flatbuffers_vec_len(vec) > (i) && "index out of range"' failed.
I received a response from the flatcc author.
See https://groups.google.com/g/flatbuffers/c/mjkyhbA6vS8 for the complete response.

GNU linker, different adressing for same memory region

So i have a simple linker script for my stm32f7 mcu
MEMORY{
ROM_AXIM (rx) : ORIGIN = 0x08000000, LENGTH = 1M
ROM_ITCM (rx) : ORIGIN = 0x00200000, LENGTH = 1M
RAM_ITCM (rwx): ORIGIN = 0x00000000, LENGTH = 16K
RAM_DTCM (rwx): ORIGIN = 0x20000000, LENGTH = 64K
SRAM (rwx): ORIGIN = 0x20010000, LENGTH = 240K
SRAM2 (rwx): ORIGIN = 0x2004C000, LENGTH = 16K
}
_estack = LENGTH(RAM_DTCM) + ORIGIN(RAM_DTCM);
SECTIONS{
.isr_vector : {
KEEP(*(.isr_vector))
} /* Placed at 0x0 */
.text : {
. = ALIGN(4);
*(.text)
} >ROM_ITCM
.data : {
. = ALIGN(4);
_sdata = .;
*(.data)
. = ALIGN(4);
_sdata = .;
} >SRAM2 AT>ROM_AXIM
.bss : {
. = ALIGN(4);
_sbss = .;
*(.bss)
. = ALIGN(4);
_ebss = .;
} >SRAM2
}
The idea is to place text section to ROM_ITCM because instruction fetching is accelerated with ART accelerator. But the problem is that ROM_AXIM and ROM_ITCM is the same flash storage. How to tell linker that is physically same storage but accessed on separate buses. So it links like it is two separate buses, but the text section should actually follow .isr_vector immediately in memory and offset is taken into account
For example, here is my bin file that will go to flash:
00000000 00 00 01 20 01 00 20 00 00 00 00 00 00 00 00 00 |... .. .........|
00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
*
00200000 00 20 70 47 |. pG|
00200004
As you can see a lot flash is wasted and it ill try to write this bin beyond flash boundary as well.
VS:
00000000 00 00 01 20 09 00 00 08 00 20 70 47
0000000c
this hexdump, is what i am looking for but, as you can see Reset_Handler has address of AXIM bus. What i want to do is by using the linker script provided above to get an output like this:
00000000 00 00 01 20 09 00 20 00 00 20 70 47
0000000c
The difference here is that it will use 0x00200008 to look for my reset handler.
What i have tried so far:
.text : {
. = ALIGN(4);
*(.text)
} >ROM_ITCM AT>ROM_AXIM
This one would work, but the problem is that it will give this output
00000000 00 00 01 20 01 00 20 00 00 20 70 47 |... .. .. pG|
0000000c
which will load instruction at 0x00200000 and by doing that it will load first entry of vector table (stack pointer) as an instruction
I managed to solve a problem by consulting a gnu linker page. What i did was to specify runtime offset of a section like this.
.text (0x00200000 + SIZEOF(.isr_vector)): {
. = ALIGN(4);
*(.text)
} AT>ROM_AXIM
What it does is:
.text (0x00200000 + SIZEOF(.isr_vector)) specify run-time address with an offset of vector table size. my pointers are now resolved correctly
AT>ROM_AXIM places the code right after vector table which produced the offset in first place. and above line fixes it.
-

deciphering "invalid mem access map_ptr" on BPF program upload

I'm writing a custom application that reads C code, calls LLVM to generate BPF byte code from that C code, then relocates any bpf map symbols and uploads it to the kernel. I can successfully upload and run programs that don't use BPF maps, but as soon as I relocate a program to use a BPF map I get the following error:
invalid mem access 'map_ptr'
Details follow:
The following input is given to LLVM:
// Placeholder values for user-requested maps
void *a;
#include <linux/ptrace.h>
#include <uapi/linux/bpf.h>
#include "bpf_helpers.h"
int kprobe__blk_start_request(struct pt_regs *ctx) {
long rq = PT_REGS_PARM1(ctx);
u64 val = bpf_ktime_get_ns();
bpf_map_update_elem(a, &rq, &val, BPF_ANY);
return 0;
}
which generates the following bytecode
Found function: kprobe__blk_start_request
size: 120 bytes
addr: 0
relocate: a # 32
/tmp/bpf7990/bpf.o: file format ELF64-BPF
Disassembly of section .text:
kprobe__blk_start_request:
0: 79 11 70 00 00 00 00 00 r1 = *(u64 *)(r1 + 112)
1: 7b 1a f8 ff 00 00 00 00 *(u64 *)(r10 - 8) = r1
2: 85 00 00 00 05 00 00 00 call 5
3: 7b 0a f0 ff 00 00 00 00 *(u64 *)(r10 - 16) = r0
4: 18 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 r1 = 0ll
6: 79 11 00 00 00 00 00 00 r1 = *(u64 *)(r1 + 0)
7: bf a2 00 00 00 00 00 00 r2 = r10
8: 07 02 00 00 f8 ff ff ff r2 += -8
9: bf a3 00 00 00 00 00 00 r3 = r10
10: 07 03 00 00 f0 ff ff ff r3 += -16
11: b7 04 00 00 00 00 00 00 r4 = 0
12: 85 00 00 00 02 00 00 00 call 2
13: b7 00 00 00 00 00 00 00 r0 = 0
14: 95 00 00 00 00 00 00 00 exit
and the "a" symbol gets relocated in instruction #4 as follows:
Relocating instruction 4
from code:0x18 dst_reg:1 src_reg:0 off:0x0 imm:0x0
to code:0x18 dst_reg:1 src_reg:1 off:0x0 imm:0x4
kprobe__blk_start_request:
0: 79 11 70 00 00 00 00 00
1: 7b 1a f8 ff 00 00 00 00
2: 85 00 00 00 05 00 00 00
3: 7b 0a f0 ff 00 00 00 00
4: 18 11 00 00 04 00 00 00
5: 00 00 00 00 00 00 00 00
6: 79 11 00 00 00 00 00 00
7: bf a2 00 00 00 00 00 00
8: 07 02 00 00 f8 ff ff ff
9: bf a3 00 00 00 00 00 00
10: 07 03 00 00 f0 ff ff ff
11: b7 04 00 00 00 00 00 00
12: 85 00 00 00 02 00 00 00
13: b7 00 00 00 00 00 00 00
14: 95 00 00 00 00 00 00 00
The "04" in instruction #4 is the FD to the map for symbol "a".
After calling BPF_PROG_LOAD I get the following in the error log from the kernel.
Failed to load kprobe__blk_start_request BPF code
0: R1=ctx(id=0,off=0,imm=0) R10=fp0
0: (79) r1 = *(u64 *)(r1 +112)
1: R1=inv(id=0) R10=fp0
1: (7b) *(u64 *)(r10 -8) = r1
2: R1=inv(id=0) R10=fp0
2: (85) call bpf_ktime_get_ns#5
3: R0=inv(id=0) R10=fp0
3: (7b) *(u64 *)(r10 -16) = r0
4: R0=inv(id=0) R10=fp0
4: (18) r1 = 0xffff88042be0b000
6: R0=inv(id=0) R1=map_ptr(id=0,off=0,ks=8,vs=8) R10=fp0
6: (79) r1 = *(u64 *)(r1 +0)
R1 invalid mem access 'map_ptr'
Errno: 13 (Permission denied)
I'm having trouble deciphering this error log. What is the kernel trying to tell me?
Error explanation
invalid mem access 'map_ptr' means you're trying to read from an invalid memory location, in particular one pointed to by a map pointer.
Indeed, referring to your bytecode:
4: 18 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 r1 = 0ll
6: 79 11 00 00 00 00 00 00 r1 = *(u64 *)(r1 + 0)
you first load an immediate value inside r1, then read the memory location pointed to by r1. However, the kernel verifier identifies BPF_LD_IMM instructions as map pointer loading. So, it tags r1 with the map_ptr type. The verifier then rejects instruction #6 because it tries to read the memory location pointed to by the map pointer (you're only supposed to pass it to map helpers, not use it otherwise in BPF programs).
Basically, instruction #6 is both invalid and unneeded. Without it, your program should pass the verifier.
Likely fix
I don't have your relocation code so I can't reproduce, but I'd guess this invalid bytecode as to do with how you declare the map. If you take a look at BPF samples in the Linux kernel, you'll see that maps are usually declared as global structure to which the first map helper argument points:
// Placeholder values for user-requested maps
struct bpf_map_def a = {
.type = BPF_MAP_TYPE_ARRAY,
.key_size = sizeof(u32),
.value_size = sizeof(int),
.max_entries = 1024,
};
#include <linux/ptrace.h>
#include <uapi/linux/bpf.h>
#include "bpf_helpers.h"
int kprobe__blk_start_request(struct pt_regs *ctx) {
long rq = PT_REGS_PARM1(ctx);
u64 val = bpf_ktime_get_ns();
bpf_map_update_elem(&a, &rq, &val, BPF_ANY);
return 0;
}

Opening and Reading from a "large" gzip compressed file in C

I have been trying to open and read a gzip compressed file using gzip-based file IO functions in C. The compressed file that I have with me is quite large of size 12 GB. The uncompressed file was ~260 GB and hence I am not prepared to uncompress the file using gunzip and go ahead from there.
I am specifically using the below code to read and write into the buffers available to us-
#define windowBits 15
#define ENABLE_ZLIB_GZIP 32
#define CHUNK 0x4000
#define CALL_ZLIB(x) { \
int status; \
status = x; \
if (status < 0) \
{ \
fprintf(stderr, "%s:%d: %s returned a bad status of %d.\n", __FILE__, __LINE__, #x, status); \
exit(EXIT_FAILURE);\
} \
} \
int main ()
{
const char * file_name = "test.gz";
FILE * file;
z_stream strm = {0};
unsigned char in[CHUNK];
unsigned char out[CHUNK];
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.next_in = in;
strm.avail_in = 0;
CALL_ZLIB (inflateInit2 (& strm, windowBits | ENABLE_ZLIB_GZIP));
/* Open the file. */
file = fopen (file_name, "rb");
while (1) {
int bytes_read;
bytes_read = fread (in, sizeof (char), sizeof (in), file);
strm.avail_in = bytes_read;
do {
unsigned have;
strm.avail_out = CHUNK;
strm.next_out = out;
CALL_ZLIB (inflate (& strm, Z_NO_FLUSH));
have = CHUNK - strm.avail_out;
fwrite (out, sizeof (unsigned char), have, stdout);
}
while (strm.avail_out == 0);
if (feof (file)) {
inflateEnd (& strm);
break;
}
}
return 0;
}
The code accurately reads and writes from the zlib file based on a buffer that you specify initially. The buffer size is being fixed to a certain value (in the above case to 0x4000).
The problem now is that I cannot increase the size of this buffer beyond a certain value ( I can use 3276008 as buffer size, but not 32760008 ). To read a 12 GB compressed value, would need me to use a very big buffer. As specified in my edits, this looks like some kind of a DATA_ERROR not a BUFFERerror... so it is not a buffer error after all!
Is there any way how I can be able to record the whole 12 GB compressed file using the zlib functions above ?
EDIT #1
The error code returned by the function inflate is encapsulated by the CALL_ZLIB function which I am sorry to have not included. So I get the below error code when I run with the buffer size of 0x4000. I have added the CALL_ZLIB function to the code for your reference also.
Error msg :
parser.c:96: inflate(&strm, Z_NO_FLUSH) returned a bad status of -3. This obviously looks like a **DATA_ERROR.
EDIT #2
I have tried adding a negative value of windowBits to InflateInit2() but that did not solve any of my problems. The inflate() function initially reads my file correctly -- displaying all of my data the way I want it to..
0x55b0 [0x40]: event: 3
.
. ... raw event: size 64 bytes
. 0000: 03 00 00 00 00 00 40 00 18 03 00 00 18 03 00 00 ......#.........
. 0010: 4d 6f 64 65 6d 4d 61 6e 61 67 65 72 00 00 00 00 ModemManager....
. 0020: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
. 0030: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
0 0 0x55b0 [0x40]: PERF_RECORD_COMM: ModemManager:792/792
0x55f0 [0x40]: event: 7
.
. ... raw event: size 64 bytes
. 0000: 07 00 00 00 00 00 40 00 19 03 00 00 01 00 00 00 ......#.........
. 0010: 19 03 00 00 01 00 00 00 00 00 00 00 00 00 00 00 ................
. 0020: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
. 0030: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
0 0 0x55f0 [0x40]: PERF_RECORD_FORK(793:793):(1:1)
0x5630 [0x40]: event: 3
.
But after some time, the displayed output becomes garbled and I cannot read from it anymore..
0x4d68 [0x38]: ........... 001 0..
0 0 00 00 00 0 00 000 00 ze 64s
. 0000: 07 00 00 00 00 00 40 00 19 03 00 00 01 00 00 00 .. 00 0 event: size 64 bytes
. 0000: 03 00 00 00 si sisizsiz4s
. 0000: 07 00 00 00 00 00 40 00 19 0....
. 0030: 00 00 00 00 00 00 00 00 00 00 00 00 ..#.#. 0010: 19 03 00 00 [0x38]: ........... 001 0..
0 0 00 00 00 0 00 000 00 ze 64s
. 0000: 07 00 00 00 00 00 40 00 100 00 00 00 00 ..............0 0 0x4d28 [0x40]: PERF_RECORD_FORK(135:135):(2:62)
0x4d68 [0x38]: ........... 001 0..
0 0 00 00 00 0 00 000 00 00 00 00: PERORD_FORK(135:135):(2:2)
This finally terminates with the error message I described in the Edit #1
I have resolved the problem.
The basic problem was that I was not initializing the strm.next_in member of z_stream in my code inside the loop. Hence after doing 1 iteration, the buffer got corrupted and I was getting the above errors.
I modified my code to --
strm.next_in = in;
strm.avail_in = 0;
CALL_ZLIB(inflateInit2 (&strm, windowBits | ENABLE_ZLIB_GZIP));
file = fopen(filename, "rb");
while(1)
{
int bytes_read;
strm.next_in = in; // added this line
bytes_read = fread(in, sizeof(char), sizeof(in), file);
strm.avail_in = bytes_read;
do
{
unsigned have;
strm.avail_out = CHUNK;
strm.next_out = out;

Resources