memory mapped files - c

I wrote a code for writing the content to the mapped buffer which mapped by using the mmap() system call.
After I did some the changes in the mapped buffer,then I called the msync().It should update to the file on disk.
But,It doesn't made any changes to the file on disk.
#include<stdio.h>
#include<stdlib.h>
#include<string.h>
#include<unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include<sys/mman.h>
#include<fcntl.h>
#define FILEMODE S_IRWXU | S_IRGRP | S_IROTH
#define MAX 150
main(int argc,char *argv[])
{
int fd,ret,len;
long int len_file;
struct stat st;
char *addr;
char buf[MAX];
if(argc > 1)
{
if((fd = open(argv[1],O_RDWR | O_APPEND | O_CREAT ,FILEMODE)) < 0)
perror("Error in file opening");
if((ret=fstat(fd,&st)) < 0)
perror("Error in fstat");
len_file = st.st_size;
/*len_file having the total length of the file(fd).*/
if((addr=mmap(NULL,len_file,PROT_READ|PROT_WRITE,MAP_PRIVATE,fd,0)) == MAP_FAILED)
perror("Error in mmap");
len = len_file;
while((fgets(buf,MAX,stdin)) != NULL)
{
strcat(addr+len,buf);
printf( "Val:%s\n",addr ) ; //Checking purpose
len = len + (strlen(buf));
}
if((msync(addr,len,MS_SYNC)) < 0)
perror("Error in msync");
if( munmap(addr,len) == -1)
printf("Error:\n");
printf("addr %p\n",addr);
}
else
{
printf("Usage a.out <filename>\n");
}
}

If you want your changes to be reflected in the on-disk file, you must map the file as MAP_SHARED, not MAP_PRIVATE.
Additionally, you cannot extend the file simply by writing beyond the end of the mapping. You must use ftruncate() to extend the file to the new size, then change the mapping to include the new portion of the file. The portable way to change the mapping is to unmap the mapping then recreate it with the new size; on Linux you can instead use mremap().
Your len and len_file variables should be of type size_t, and you should use memcpy() rather than strcat(), since you know exactly the length of the string, exactly where you want to copy it, and you don't want to copy the null-terminator.
The following modification of your code works on Linux (using mremap()) :
#define _GNU_SOURCE
#include<stdio.h>
#include<stdlib.h>
#include<string.h>
#include<unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include<sys/mman.h>
#include<fcntl.h>
#define FILEMODE S_IRWXU | S_IRGRP | S_IROTH
#define MAX 150
int main(int argc,char *argv[])
{
int fd, ret;
size_t len_file, len;
struct stat st;
char *addr;
char buf[MAX];
if (argc < 2)
{
printf("Usage a.out <filename>\n");
return EXIT_FAILURE;
}
if ((fd = open(argv[1],O_RDWR | O_CREAT, FILEMODE)) < 0)
{
perror("Error in file opening");
return EXIT_FAILURE;
}
if ((ret = fstat(fd,&st)) < 0)
{
perror("Error in fstat");
return EXIT_FAILURE;
}
len_file = st.st_size;
/*len_file having the total length of the file(fd).*/
if ((addr = mmap(NULL,len_file,PROT_READ|PROT_WRITE,MAP_SHARED,fd,0)) == MAP_FAILED)
{
perror("Error in mmap");
return EXIT_FAILURE;
}
while ((fgets(buf,MAX,stdin)) != NULL)
{
len = len_file;
len_file += strlen(buf);
if (ftruncate(fd, len_file) != 0)
{
perror("Error extending file");
return EXIT_FAILURE;
}
if ((addr = mremap(addr, len, len_file, MREMAP_MAYMOVE)) == MAP_FAILED)
{
perror("Error extending mapping");
return EXIT_FAILURE;
}
memcpy(addr+len, buf, len_file - len);
printf( "Val:%s\n",addr ) ; //Checking purpose
}
if((msync(addr,len,MS_SYNC)) < 0)
perror("Error in msync");
if (munmap(addr,len) == -1)
perror("Error in munmap");
if (close(fd))
perror("Error in close");
return 0;
}

Note that you've provided a mapping for the file that is exactly the size of the file. If you create the file in your call to open(2), it will have a length of 0, and I wouldn't be surprised if the kernel doesn't bother setting up any kind of memory mapping from a 0 length mapping. (Maybe it does? I've never tried...)
I would suggest using ftruncate(2) to extend the length of your file before performing the mapping. (Note that extending files using ftruncate(2) isn't very portable; not all platforms provide extending functionality and not all filesystem drivers support the extending functionality. See your system's manpage for details.)
You must use the MAP_SHARED mapping for your file modifications to be saved to disk.
Your use of perror(3) isn't quite correct; perror(3) will not terminate your program, so it will continue executing with incorrect assumptions:
if((ret=fstat(fd,&st)) < 0)
perror("Error in fstat");
Should read:
if((ret=fstat(fd,&st)) < 0) {
perror("Error in fstat");
exit(1);
}
(Or exit(EXIT_FAILURE) if you want to be more portable -- I find that a little harder on the eyes but I live in Linux-land.)
strcat(3) expects to find an ASCII NUL character (byte value 0x00, C representation '\0') -- the usual C end-of-string marker -- at the end of the dest string. Your file will not contain an ASCII NUL if you create it in this program -- its length is zero, after all -- and I don't know the consequences of trying to read a zero-byte file via mmap(2). If the file already exists and has data in it, it probably doesn't have an ASCII NUL encoded in the file. strcat(3) is almost certainly the wrong tool to write into your file. (No one wants ASCII NULs in their files anyway.) Try memcpy(3) instead.

Related

Implementing the cp command using read/write system calls [duplicate]

This question already has an answer here:
Using open(), read() and write() system calls to copy a file
(1 answer)
Closed last year.
I am trying to implement the cp command only using read/write system calls.
Here is my code:
/**
* cp file1 file 2
*/
#include <stdio.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
int main(int argc, char *argv[])
{
int errsv;
char contents[1024];
int fd_read, fd_write;
fd_read = open(argv[1], O_RDONLY);
if (fd_read == -1)
{
errsv = errno;
printf("Error occured: %d\n", errsv);
}
read(fd_read, contents, sizeof(contents));
fd_write = open(argv[2], O_CREAT | O_WRONLY | O_TRUNC, 0744);
if (fd_write == -1)
{
errsv = errno;
printf("Error occured: %d\n", errsv);
}
write(fd_write, contents, sizeof(contents));
close(fd_read);
close(fd_write);
return 0;
}
I tested the code using the commands:
cc test.c
./a.out file1 file2
Here is my file1:
dummy text
dummy text
After running the code, although file2 contains the text from file1, it also has some gibberish characters. [not keeping this here.]
Why is this so?
You need to call read() and write() in a loop to copy the entire file. read() returns 0 when you reach EOF, or a negative result if there's an error, then you can end the loop.
read() returns the number of bytes that were read, which may be less than the size of the buffer. You need to use that number when calling write(), otherwise you'll write extra characters to the output file. These will be unitialized characters on the first iteration, and on other iterations they'll be left over characters from previous iterations.
int main(int argc, char *argv[])
{
char contents[1024];
int fd_read, fd_write;
fd_read = open(argv[1], O_RDONLY);
if (fd_read == -1)
{
perror("open input file");
exit(1);
}
fd_write = open(argv[2], O_CREAT | O_WRONLY | O_TRUNC, 0744);
if (fd_write == -1)
{
perror("open output file");
exit(1)
}
int n_read;
while ((n_read = read(fd_read, contents, sizeof(contents))) > 0) {
write(fd_write, contents, n_read);
}
close(fd_read);
close(fd_write);
return 0;
}
write(fd_write, contents, strlen(contents));
Strlen returns the filled entries number but sizeof returns the buffer size which is 1024

write() to sysfs entry /sys/bus/pci/devices/.../driver/remove_id fails

Seeing the write() function failing on /sys/bus/pci/devices/.../driver/remove_id file returning -1 with errno equal to 19 (ENODEV).
But, same works fine via command line. I have checked the file permission which seems to be fine (--w-------) for user to perform write on this file.
int fp = 0;
int buffer_length = 0;
int bytes_written = 0;
fp = open(cmd_buf, O_WRONLY); // where cmd_buf will hold this string
// "/sys/bus/pci/devices/.../driver/remove_id"
if (fp == -1)
{
return -1;
}
// where inbuf will be a char * pointing to pci vendor device id like
// this, "XXXX YYYY"
bytes_written = write(fp, in_buf, sizeof(in_buf));
printf(" bytes_written : %d \n ", bytes_written);
Seeing bytes_written equal to -1 and errno shows 19.
Please let me know if you find something wrong with the code snippet?
You do not provide enough information to pinpoint the problem.
However, here is an example program, example.c, that shows that it is your implementation that has the bug:
#define _POSIX_C_SOURCE 200809L
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include <string.h>
#include <stdio.h>
/* Write string 'data' to existing file or device at 'path'.
Returns 0 if success, errno error code otherwise.
*/
int write_file(const char *path, const char *data)
{
const char *const ends = (data) ? data + strlen(data) : data;
ssize_t n;
int fd;
/* NULL or empty path is invalid. */
if (!path || !*path)
return errno = EINVAL;
fd = open(path, O_WRONLY | O_CREAT | O_TRUNC | O_NOCTTY | O_CLOEXEC, 0666);
if (fd == -1)
return errno; /* errno already set by open(). */
/* Write the contents of data. */
while (data < ends) {
n = write(fd, data, (size_t)(ends - data));
if (n > 0) {
/* Wrote n bytes. */
data += n;
} else
if (n != -1) {
/* C Library bug: Should never occur. */
close(fd);
return errno = EIO;
} else {
/* Error in errno. */
const int saved_errno = errno;
close(fd);
return errno = saved_errno;
}
}
if (close(fd) == -1) {
/* It is possible for close() to report a delayed I/O error. */
return errno;
}
/* Success. */
return 0;
}
static void usage(const char *argv0)
{
fprintf(stderr, "\n");
fprintf(stderr, "Usage: %s [ -h | --help ]\n", argv0);
fprintf(stderr, " %s FILE CONTENTS\n", argv0);
fprintf(stderr, "\n");
fprintf(stderr, "This does the same thing as 'echo -n \"CONTENTS\" > FILE'.\n");
fprintf(stderr, "\n");
}
int main(int argc, char *argv[])
{
if (argc < 2) {
usage((argv && argv[0] && argv[0][0]) ? argv[0] : "(this)");
return EXIT_SUCCESS;
} else
if (argc > 3) {
usage((argv && argv[0] && argv[0][0]) ? argv[0] : "(this)");
return EXIT_FAILURE;
} else
if (!strcmp(argv[1], "-h") || !strcmp(argv[1], "--help")) {
usage((argv && argv[0] && argv[0][0]) ? argv[0] : "(this)");
return EXIT_SUCCESS;
}
if (write_file(argv[1], argv[2])) {
fprintf(stderr, "%s: %s.\n", argv[1], strerror(errno));
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
Compile it using e.g. gcc -Wall -Wextra -O2 example.c -o example, and run using e.g. ./example /sys/bus/pci/devices/.../driver/remove_id "vendor_id device_id".
Run it without arguments, or with -h or --help as the only argument, and it will print usage information to standard error.
The program essentially does what echo -n "vendor_id device_id" > /sys/bus/pci/devices/.../drivers/remove_id does.
If it is successful, it will not output anything, just return success (exit status 0). If there is any kind of an error, it will report it to standard error.
If you know the target path is always a device or a pseudo-file (like those in /sys or /proc), use fd = open(path, O_WRONLY | O_NOCTTY | O_CLOEXEC); instead. O_CLOEXEC means that if the process forks at any point, this particular file descriptor is not copied to the child process. O_NOCTTY means that if the path is a tty device, and the current process does not have a controlling terminal, the kernel is NOT to make the opened device the controlling terminal.
echo -n uses O_CREAT | O_TRUNC, so that if the target path exists and is a normal file, it is truncated, but if it does not exist, it is created. It does not affect opening existing character devices and pseudofiles. Whenever O_CREAT is used, there must be a third parameter, which affects the access mode of the file created. This mode is usually 0666, allowing read and write access as moderated by the current umask. One can obtain the current umask using mode_t mask = umask(0); umask(mask);. Access mode bits set in umask are always zero in the final access mode, and access mode bits clear in umask are taken from the third parameter of the open() command when the file is created.
Two possible problems:
Using sizeof(in_buf) in the write() system call writes the string "vendorId deviceId" and may be more garbage data behind it if in_buf[] is bigger than 10 chars.
Perhaps in_buf is not a table but a pointer and so, sizeof(in_buf) will return 4 or 8 (the size of the pointer respectively for 32 or 64 bits systems) but not the length of the string it points to.
So, in both cases (in_buf defined as a table or a pointer), strlen(in_buf) instead of sizeof(in_buf) is the most secured solution for the length of the data to write provided that the string is terminated by '\0'.

Unwanted characters when copying file using scatter/gather I/O (readv/writev)

I'm trying to build a program to copy existing content from an existing file to the new file using readv() and writev().
Here is my code:
#include <stdio.h>
#include <sys/types.h>
#include <fcntl.h>
#include <sys/uio.h>
#include <unistd.h>
#include <string.h>
int main(int argc, char *argv[])
{
int fs, fd;
ssize_t bytes_read, bytes_written;
char buf[3][50];
int iovcnt;
struct iovec iov[3];
int i;
fs = open(argv[1], O_RDONLY);
if (fs == -1) {
perror("open");
return -1;
}
fd = open(argv[2], O_RDWR | O_CREAT | O_TRUNC, S_IRWXU);
if (fd == -1) {
perror("open");
return 1;
}
for(i = 0; i < 3; i++) {
iov[i].iov_base = buf[i];
iov[i].iov_len = sizeof(buf[i]);
}
iovcnt = sizeof(iov) / sizeof(struct iovec);
if ((bytes_read=readv(fs, iov, iovcnt)) != -1)
if ((bytes_written=writev(fd, iov, iovcnt)) == -1)
perror("error writev");
printf("read: %ld bytes, write: %ld bytes\n", bytes_read, bytes_written);
if (close (fs)) {
perror("close fs");
return 1;
}
if (close (fd)) {
perror("close fd");
return 1;
}
return 0;
}
Problem: Let's say I ran the program with argv[1] corresponding to the file called file1.txt and copied it to argv[2], let's say it's called as hello.txt.
This is the content of file1.txt:
Ini adalah line pertamaS
Ini adalah line kedua
Ini adalah line ketiga
When I ran the program, the new created file specified in argv[2] were filled by unwanted characters such as \00.
Output after running the program:
Ini adalah line pertamaS
Ini adalah line kedua
Ini adalah line ketiga
\00\00\FF\B5\F0\00\00\00\00\00\C2\00\00\00\00\00\00\00W\D4\CF\FF\00\00V\D4\CF\FF\00\00\8D\C4|\8C\F8U\00\00\C8o\A6U\E5\00\00#\C4|\8C\F8U\00\00\00\00\00\00\00\00\00\00 \C1|\8C\F8U\00\00`\D5\CF\FF
I suspect the main cause of the problem is unfitted size of buf array. I've already look up internet for the solutions and there are nothing to be found. Can anyone give me some enlightment to fix this problem? I tried to make the buf or iov_len to be variable-length but I couldn't find the right way to do it. Thanks everyone!
readv() works with byte counts driven by each .iov_len and no special treatment for any content (like a line-feed). The readv() in the original posting is passed an array of (3) struct iovec, each with .iov_len set to 50. After a successful readv(), the content of the local buf[3][50] would be:
buf[0] : first 50 bytes from the input file
buf[1] : next 20 bytes from the input file, then 30 bytes of uninitialized/leftover stack data
buf[2] : another 50 bytes of uninitialized/leftover stack data
The writev() reuses the same struct iovec array with all (3) .iov_len unchanged from 50, and writes 150 bytes as expected. The content of the output file has the first 70 bytes copied from the input file and 80 bytes of leftover stack data. If the local buf was cleared before calling readv(), the output file would contain trailing NULLs.

Why stat and fstat return the st_size == 0?

I was testing a code from APUE, in chapter 14(Advanced I/O) of memory map file, the fstat() always return the fdin's st_size as zero, and I tried stat() instead, and also get the same result. I list the code below(I have removed the apue.h dependencies):
#include <fcntl.h>
#include <sys/mman.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <sys/stat.h>
#include <unistd.h>
#define COPYINCR (1024*1024*1024) /* 1GB */
int main(int argc, char *argv[]) {
if (argc != 3) {
printf("usage: %s <fromfile> <tofile>", argv[0]);
exit(1);
}
int fdin, fdout;
if ((fdin = open(argv[1], O_RDONLY)) < 0) {
printf("can not open %s for reading", argv[1]);
exit(1);
}
if ((fdout = open(argv[2] /* typo fix */, O_RDONLY | O_CREAT | O_TRUNC)) < 0) {
printf("can not open %s for writing", argv[2]);
exit(1);
}
struct stat sbuf;
if (fstat(fdin, &sbuf) < 0) { /* need size fo input file */
printf("fstat error");
exit(1);
}
// always zero, and cause truncate error (parameter error)
printf("input_file size: %lld\n", (long long)sbuf.st_size);
if (ftruncate(fdout, sbuf.st_size) < 0) { /* set output file size */
printf("ftruncate error");
exit(1);
}
void *src, *dst;
off_t fsz = 0;
size_t copysz;
while (fsz < sbuf.st_size) {
if (sbuf.st_size - fsz > COPYINCR)
copysz = COPYINCR;
else
copysz = sbuf.st_size - fsz;
if (MAP_FAILED == (src = mmap(0, copysz, PROT_READ,
MAP_SHARED, fdin, fsz))) {
printf("mmap error for input\n");
exit(1);
}
if (MAP_FAILED == (dst = mmap(0, copysz,
PROT_READ | PROT_WRITE,
MAP_SHARED, fdout, fsz))) {
printf("mmap error for output\n");
exit(1);
}
memcpy(dst, src, copysz);
munmap(src, copysz);
munmap(dst, copysz);
fsz += copysz;
}
return 0;
}
And then I have tried the Python os.stat, it also get the zero result, why this happened? I have tried these and got the same result on Mac OS (Darwin kernel 13.4) and Ubuntu (kernel 3.13).
UPDATE:
Oh, there was a typo error, I should refer to fdout to argv[2], and the O_TRUNC flag certainly make the fdin to zero. Should I close or delete this question?
The reason why Python's os.stat() also return (stat.st_size == 0) is that I passed the same test file (argv[1]) to test, and the file has been previously truncated to zero (I haven't check its size using ls -lh before passing to os.stat()), and certainly os.stat() return zero.
Do not ask SO questions before you go to bed or in a rush.
Ok, the real problem is double open the same input file, and this does not cause any build or runtime error until the ftruncate().
The first open get a read-only fdin, the second open create a new file (fdout and truncated) to copy from fdin via memory map, and the second open truncated the first file (argv[1]), and cleaned all its content. But the fdin still working with fstat (and certainly), this make me hard to find the reason.
The second part is I always use the same file for testing (generated via dd) and have not checking the size, so the os.stat(/path/to/file) and stat(/path/to/file) also return st_size == 0, this makes me believe that this must be some os-level-prolicy defined the behaviour, and I rushed to Mac OS (using the same typo code), and got the same result (they really consistent on POSIX level, event the bug!), and at last, I came to SO for help.

How to read mp3 file tags in C (unix)?

I would like to read mp3 tags from mp3 file :D and save it to txt file. But my code doesnt work :( I mean I have some problems with setting the proper position in my mp3 file, take a look: (why doesnt it want to work?). I have to do it by myself, with no extra libs.
#include <stdio.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <sys/types.h>
int getFileSize(const char *filename)
{
struct stat st;
if (stat(filename, &st) == 0)
return st.st_size;
return -1;
}
int main(int argc, char **argv)
{
char *infile = "in.mp3", *outfile = "out.txt";
int infd, bytes_read = 0, buffsize = 255;
char buffer[255];
infd = open(infile, O_RDONLY);
if (infd == -1)
return -1;
int outfd = open(outfile, O_WRONLY | O_TRUNC, S_IRUSR | S_IWUSR);
if (outfd == -1)
return -1;
if(lseek(infd, -128, SEEK_END) < 0)
return -1;
for(;;)
{
bytes_read = read(infd, buffer, buffsize);
if (bytes_read > 0)
{
write(outfd, buffer, bytes_read);
}
else
{
if (bytes_read == 0)
{
if (close(infd) < 0)
return -1;
break;
}
else if (bytes_read == -1)
{
break;
return -1;
}
}
}
return 0;
}
One approach to solve this problem:
You'll need to scan through the file depending on the Version of ID3 you're using (question has no specific version specified as noted by Steven), find either the whole tag or the tag header and decode from there.
For ID3v2 the header sequence is 10 Bytes and as follows (from the ID3v2 spec):
ID3v2/file identifier "ID3"
ID3v2 version $04 00
ID3v2 flags %abcd0000
ID3v2 size 4 * %0xxxxxxx
My suggestion is, take a look at the spec of ID3v2 here. Check Chapter 3.1 since part of the work is doing the background research.
For ID3v1 check that overview spec here. Decoding that information is quite easy and works exactly as noted in the comments to your question. Looking at your code this is probably what you want to do (jumping to 128 bytes at the end of the file and starting to read from there).
Make sure you have a properly tagged file and are sure about the tag version you're using before throwing your decoder at it.

Resources