I am messing with mmap in c and I have encountered a very strange error. When I run the following block of code (which is sample code from this website)
/* The file descriptor. */
int fd;
/* Information about the file. */
struct stat s;
int status;
size_t size;
/* The file name to open. */
const char * file_name = "myfile.txt";
/* The memory-mapped thing itself. */
const void * mapped;
int i;
/* Open the file for reading. */
fd = open ("myfile.txt", O_RDONLY);
check (fd < 0, "open %s failed: %s", file_name, strerror (errno));
/* Get the size of the file. */
status = fstat (fd, & s);
check (status < 0, "stat %s failed: %s", file_name, strerror (errno));
size = s.st_size;
/* Memory-map the file. */
mapped = mmap (0, size, PROT_READ, MAP_SHARED, fd, 0);
check (mapped == MAP_FAILED, "mmap %s failed: %s",
file_name, strerror (errno));
I am greeted with an Invalid Argument error.
My research has led me to conclude that this is an offset issue, but I am totally lost as to what I can do to fix it. Any advice would be greatly appreciated.
Thank you
I decided to try running this code on another machine and it worked fine, it appears to be a problem machine-side rather than a problem in the code. At least now I know the code isn't broken :)
Related
I'm new in the area of memory mapped and I was wondering if there is any way I can read a text file using memory mapped to a string. I don't really know how to start to write the code.
The general idea with memory mapped I/O is that you tell the operating system (OS) what file you want, and it (after doing some amount of set-up work) tells you where that file now is in memory.
Once that contract is performed, you should be able to copy things to and from that memory in any way you wish (such as with memcpy), and it will magically handle the I/O for you.
Detail depends on which OS you're using since the ISO C standard doesn't specify this behaviour - it's therefore OS-specific.
For example, Windows uses a file mapping paradigm shown here, while Linux uses mmap to allow you to subject a file you've already opened to memory mapping (via its file descriptor).
By way of example, this Linux program, a little voluminous due mainly to its error checking and progress reports, memory maps the file.txt file and outputs its content:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <sys/mman.h>
#include <fcntl.h>
#include <unistd.h>
// Helper function to minimise error code in main.
static int clean(
int retVal, // value to return.
char *err, // error/NULL, allows optional %s for strerror(errno).
int fd, // fd/-1 to close.
void *filMem, // memory/NULL to unmap.
off_t sz, // size if unmapping.
void *locMem // memory/NULL to free.
) {
if (err) printf (err, strerror(errno));
if (locMem) free(locMem);
if (filMem) munmap(filMem, sz);
if (fd >= 0) close(fd);
return retVal;
}
int main(void) {
int fd = open("file.txt", O_RDONLY);
if (fd < 0) return clean(-1, "Can't open: %s\n", -1, NULL, 0, NULL);
printf("File opened okay, fd = %d.\n", fd);
off_t sz = lseek(fd, 0, SEEK_END);
if (sz == (off_t) -1) return clean(-1, "Can't seek: %s\n", fd, NULL, 0, NULL);
printf("File size is %ld.\n", sz);
void *fileArea = mmap(NULL, sz, PROT_READ, MAP_SHARED, fd, 0);
if (! fileArea) return clean(-1, "Can't map: %s\n", fd, NULL, 0, NULL);
printf("File mapped to address %p.\n", fileArea);
char *localArea = calloc(1, sz + 1);
if (! localArea) return clean(-1, "Can't allocate\n", fd, fileArea, sz, NULL);
memcpy(localArea, fileArea, sz);
printf("Data copied to %p, result is [\n%s]\n", localArea, localArea);
return clean(0, NULL, fd, fileArea, sz, localArea);
}
Running that on my local system, the results can be seen from the following transcript:
pax$ cat file.txt
...This is the input file content.
pax$ ./testprog
File opened okay, fd = 3.
File size is 35.
File mapped to address 0x7f868a93b000.
Data copied to 0x1756420, result is [
...This is the input file content.
]
I am trying to "mmap" a binary file in order to encrypt it using AES then write the encrypted data to another file(outFile) using the following code. I tried to modify the flags for both functions mmap() and open() but I always get segmentation fault when I run the executable.
int main (void)
{
FILE *outFile; //The output file (encrypted)
/* A 256 bit key */
unsigned char *key = (unsigned char *)"01234567890123456789012345678901";
/* A 128 bit IV */
unsigned char *iv = (unsigned char *)"01234567890123456";
int fd;
struct stat sb;
void * memblock;
fd = open("result0.jpg",O_RDONLY);
outFile=fopen("result0enc.jpg","wb");
fstat(fd, &sb);
printf("Size: %lu\n", sb.st_size);
unsigned char decryptedtext[sb.st_size];
int decryptedtext_len, ciphertext_len;
/* Initialise the library */
ERR_load_crypto_strings();
OpenSSL_add_all_algorithms();
OPENSSL_config(NULL);
memblock = mmap(NULL, sb.st_size,PROT_READ, MAP_SHARED, fd, 0);
if (memblock == MAP_FAILED) {
close(fd);
perror("Error mmapping the file");
exit(EXIT_FAILURE);
}
ciphertext_len = encrypt((unsigned char *)memblock, sb.st_size,key,iv,ciphertext);
fwrite( ciphertext,1, sb.st_size,outFile);
if (munmap(memblock, sb.st_size) == -1) {
perror("Error un-mmapping the file");
/* Decide here whether to close(fd) and exit() or not. Depends... */
}
close(fd);
fclose(outFile);
EVP_cleanup();
ERR_free_strings();
return 0;
}
As yano mentioned in the comments, your error is here:
memcpy(outFile, ciphertext, sb.st_size);
You're trying to memcpy to a FILE * which is completely wrong. That doesn't do at all what you expect. You're overwriting the private internals of the FILE structure to which outFile points.
You should instead operate on a buffer and use fwrite to write to the file.
I suggest you get familiar with basic file I/O operations using f... functions before digging into mmap and encryption.
I have made many researches about MD5 of an xls file but my effort seems be in vain
I tried to used lirary and recommendation in this link "https://stackoverflow.com/questions/27858288/calculate-md5-for-a-file-in-c-language"
but , still give wrong result ,
can you help me ??
Well I used to answer the link you gave but then the question was closed.
The idea is as follows. First read the file into a buffer. You can do this using following function:
unsigned char * readFile(const char *path)
{
FILE * pFile;
long lSize;
unsigned char * buffer;
size_t result;
pFile = fopen (path , "rb" );
if (pFile==NULL) {fputs ("File error",stderr); exit (1);}
// obtain file size:
fseek (pFile , 0 , SEEK_END);
lSize = ftell (pFile);
rewind (pFile);
// allocate memory to contain the whole file:
buffer = malloc (sizeof(char)*lSize);
if (buffer == NULL) {fputs ("Memory error",stderr); exit (2);}
// copy the file into the buffer:
result = fread (buffer,1,lSize,pFile);
if (result != lSize) {fputs ("Reading error",stderr); exit (3);}
// terminate
fclose (pFile);
return buffer;
}
Read the file
unsigned char * data = readFile("c:\\file.xls");
Then you must apply MD5 on this buffer of data. You can use code similar
to the one in that question (though I am not sure which library/implementation
of md5 author of that question used). e.g.,
char hash[64] = {0};
md5_byte_t digest[16] = {0};
md5_init(&state);
md5_append(&state, (const md5_byte_t *)data, filesize);
md5_finish(&state,digest);
int i=0;
for(i; i<16; i++)
{
snprintf(hash + i*2,sizeof(hash),"%02x",digest[i]);
}
Now hash should store the hash of the file, encoded in hexadecimal string. ps. Indeed that sample is incorrectly using strlen with binary file. That is why I suggested the readFile method above; that function also contains code to get file size - you can use that code to get file size and then pass the file size to md5_append method.
ps. also don't forget to free data when you are done with it.
MD5 of xls file is very same of MD5 of any other kind of file since it operates on bytes. See by example openssl implementation openssl/crypto/md5/md5.c and md5test.c ( code is in git://git.openssl.org/openssl.git ).
The problem is that your example uses strlen to determine the file size. But .xls format is binary, so strlen will not work properly.
Adapt the function to return the total data read from the file, and it should work.
Edit. Try something like this code:
void *addr;
struct stat s;
int ret, fd;
ret = stat(filename, &s);
if (ret) {
fprintf(stderr, "Error while stat()ing file: %m\n");
return -1;
}
fd = open(filename, O_RDONLY);;
if (fd < 0) {
fprintf(stderr, "Error while opening file: %m\n");
return -1;
}
addr = mmap(NULL, s.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
if (addr == MAP_FAILED) {
fprintf(stderr, "Error while mapping file: %m\n");
close(fd);
return -1;
}
md5_init(&state);
md5_append(&state,addr, s.st_size);
md5_finish(&state,digest);
I was testing a code from APUE, in chapter 14(Advanced I/O) of memory map file, the fstat() always return the fdin's st_size as zero, and I tried stat() instead, and also get the same result. I list the code below(I have removed the apue.h dependencies):
#include <fcntl.h>
#include <sys/mman.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <sys/stat.h>
#include <unistd.h>
#define COPYINCR (1024*1024*1024) /* 1GB */
int main(int argc, char *argv[]) {
if (argc != 3) {
printf("usage: %s <fromfile> <tofile>", argv[0]);
exit(1);
}
int fdin, fdout;
if ((fdin = open(argv[1], O_RDONLY)) < 0) {
printf("can not open %s for reading", argv[1]);
exit(1);
}
if ((fdout = open(argv[2] /* typo fix */, O_RDONLY | O_CREAT | O_TRUNC)) < 0) {
printf("can not open %s for writing", argv[2]);
exit(1);
}
struct stat sbuf;
if (fstat(fdin, &sbuf) < 0) { /* need size fo input file */
printf("fstat error");
exit(1);
}
// always zero, and cause truncate error (parameter error)
printf("input_file size: %lld\n", (long long)sbuf.st_size);
if (ftruncate(fdout, sbuf.st_size) < 0) { /* set output file size */
printf("ftruncate error");
exit(1);
}
void *src, *dst;
off_t fsz = 0;
size_t copysz;
while (fsz < sbuf.st_size) {
if (sbuf.st_size - fsz > COPYINCR)
copysz = COPYINCR;
else
copysz = sbuf.st_size - fsz;
if (MAP_FAILED == (src = mmap(0, copysz, PROT_READ,
MAP_SHARED, fdin, fsz))) {
printf("mmap error for input\n");
exit(1);
}
if (MAP_FAILED == (dst = mmap(0, copysz,
PROT_READ | PROT_WRITE,
MAP_SHARED, fdout, fsz))) {
printf("mmap error for output\n");
exit(1);
}
memcpy(dst, src, copysz);
munmap(src, copysz);
munmap(dst, copysz);
fsz += copysz;
}
return 0;
}
And then I have tried the Python os.stat, it also get the zero result, why this happened? I have tried these and got the same result on Mac OS (Darwin kernel 13.4) and Ubuntu (kernel 3.13).
UPDATE:
Oh, there was a typo error, I should refer to fdout to argv[2], and the O_TRUNC flag certainly make the fdin to zero. Should I close or delete this question?
The reason why Python's os.stat() also return (stat.st_size == 0) is that I passed the same test file (argv[1]) to test, and the file has been previously truncated to zero (I haven't check its size using ls -lh before passing to os.stat()), and certainly os.stat() return zero.
Do not ask SO questions before you go to bed or in a rush.
Ok, the real problem is double open the same input file, and this does not cause any build or runtime error until the ftruncate().
The first open get a read-only fdin, the second open create a new file (fdout and truncated) to copy from fdin via memory map, and the second open truncated the first file (argv[1]), and cleaned all its content. But the fdin still working with fstat (and certainly), this make me hard to find the reason.
The second part is I always use the same file for testing (generated via dd) and have not checking the size, so the os.stat(/path/to/file) and stat(/path/to/file) also return st_size == 0, this makes me believe that this must be some os-level-prolicy defined the behaviour, and I rushed to Mac OS (using the same typo code), and got the same result (they really consistent on POSIX level, event the bug!), and at last, I came to SO for help.
I'd like to give a try at copying the contents of a file over to another one by using memory mapped I/O in Linux via mmap(). The intention is to check by myself if that's better than using fread() and fwrite() and how would it deal with big files (like couple of GiBs for example, since the file is read whole I want to know if I need to have such amount of memory for it).
This is the code I'm working with right now:
// Open original file descriptor:
int orig_fd = open(argv[1], O_RDONLY);
// Check if it was really opened:
if (orig_fd == -1) {
fprintf(stderr, "ERROR: File %s couldn't be opened:\n", argv[1]);
fprintf(stderr, "%d - %s\n", errno, strerror(errno));
exit(EX_NOINPUT);
}
// Idem for the destination file:
int dest_fd = open(argv[2], O_WRONLY | O_CREAT | O_TRUNC, 0644);
// Check if it was really opened:
if (dest_fd == -1) {
fprintf(stderr, "ERROR: File %s couldn't be opened:\n", argv[2]);
fprintf(stderr, "%d - %s\n", errno, strerror(errno));
// Close original file descriptor too:
close(orig_fd);
exit(EX_CANTCREAT);
}
// Acquire file size:
struct stat info = {0};
if (fstat(orig_fd, &info)) {
fprintf(stderr, "ERROR: Couldn't get info on %s:\n", argv[1]);
fprintf(stderr, "%d - %s\n", errno, strerror(errno));
// Close file descriptors:
close(orig_fd);
close(dest_fd);
exit(EX_IOERR);
}
// Set destination file size:
if (ftruncate(dest_fd, info.st_size)) {
fprintf(stderr, "ERROR: Unable to set %s file size:\n", argv[2]);
fprintf(stderr, "%d - %s\n", errno, strerror(errno));
// Close file descriptors:
close(orig_fd);
close(dest_fd);
exit(EX_IOERR);
}
// Map original file and close its descriptor:
char *orig = mmap(NULL, info.st_size, PROT_READ, MAP_PRIVATE, orig_fd, 0);
if (orig == MAP_FAILED) {
fprintf(stderr, "ERROR: Mapping of %s failed:\n", argv[1]);
fprintf(stderr, "%d - %s\n", errno, strerror(errno));
// Close file descriptors:
close(orig_fd);
close(dest_fd);
exit(EX_IOERR);
}
close(orig_fd);
// Map destination file and close its descriptor:
char *dest = mmap(NULL, info.st_size, PROT_WRITE, MAP_SHARED, dest_fd, 0);
if (dest == MAP_FAILED) {
fprintf(stderr, "ERROR: Mapping of %s failed:\n", argv[2]);
fprintf(stderr, "%d - %s\n", errno, strerror(errno));
// Close file descriptors and unmap first file:
munmap(orig, info.st_size);
close(dest_fd);
exit(EX_IOERR);
}
close(dest_fd);
// Copy file contents:
int i = info.st_size;
char *read_ptr = orig, *write_ptr = dest;
while (--i) {
*write_ptr++ = *read_ptr++;
}
// Unmap files:
munmap(orig, info.st_size);
munmap(dest, info.st_size);
I think it may be a way of doing it but I keep getting an error trying to map the destination file, concretely code 13 (permission denied).
I don't have a clue on why is it failing, I can write to that file since the file gets created and all and the file I'm trying to copy is just a couple of KiBs in size.
Can anybody spot the problem? How come I had permission to map the original file but not the destination one?
NOTE: If anyone is to use the loop to copy bytes posted in the question instead of memcpy for example, the loop condition should be i-- instead to copy all contents. Thanks to jxh for spotting that.
From the mmap() man page:
EACCES A file descriptor refers to a non-regular file. Or MAP_PRIVATE
was requested, but fd is not open for reading. Or MAP_SHARED
was requested and PROT_WRITE is set, but fd is not open in
read/write (O_RDWR) mode. Or PROT_WRITE is set, but the file is
append-only.
You are opening your destination file with O_WRONLY. Use O_RDWR instead.
Also, you should use memcpy to copy the memory rather than using your own loop:
memcpy(dest, orig, info.st_size);
Your loop has an off by 1 bug.
This works for me. Note that I had to open the destination O_RDWR. I suspect the kernel attempts to map whole pages from the file into memory (reading it) because you're updating it a byte or word at a time, and that might not change the whole page.
A couple of other points:
You don't need to close and unmap stuff on error if you're just going to exit.
Use memcpy and don't write your own byte-copying loop. Memcpy will be a lot better optimised in general. (Though it's not always the absolute best.)
You might want to read the source code to FreeBSD's "cp" utility. Take a look here and search for the use of mmap. http://svnweb.freebsd.org/base/stable/9/bin/cp/utils.c?revision=225736&view=markup
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/mman.h>
#include <string.h>
#include <sys/stat.h>
int main(int argc, char *argv[])
{
int s, d;
struct stat st;
void *sp, *dp;
s = open(argv[1], O_RDONLY);
if (s == -1) {
perror("open source");
exit(1);
}
d = open(argv[2], O_RDWR | O_CREAT | O_TRUNC, 0644);
if (d == -1) {
perror("open destintation");
exit(1);
}
if (fstat(s, &st)) {
perror("stat source");
exit(1);
}
if (ftruncate(d, st.st_size)) {
perror("truncate destination");
exit(1);
}
sp = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, s, 0);
if (sp == MAP_FAILED) {
perror("map source");
exit(1);
}
dp = mmap(NULL, st.st_size, PROT_WRITE | PROT_READ, MAP_SHARED, d, 0);
if (dp == MAP_FAILED) {
perror("map destintation");
exit(1);
}
memcpy(dp, sp, st.st_size);
return 0;
}
Original File: O_RDONLY open, MAP_PRIVATE mmap
destination file: O_WRONLY open, MAP_SHARED mmap
You need to open with O_RDWR flag for using MAP_SHARED.
Don't you actually need to do MAP_FILE | MAP_SHARED ?