Loading big file in c - c

This is my program code that can load inputs 1GB, 2GB, but I would need to load files larger than 2GB. I work at Windows.
#include <stdio.h>
#include <stdlib.h>
int main() {
FILE *stream;
size_t size;
int fileSize = 0;
char *input;
// Open file, find the size of it
stream = fopen("3GB.bin", "rb");
fseek(stream, 0L, SEEK_END);
fileSize = ftell(stream);
fseek(stream, 0L, SEEK_SET);
input = (char *)malloc(fileSize + 1);
// Stream file into memory
size = fread(input, 1, fileSize, stream);
input[size] = 0;
fclose(stream);
printf("size file is: %d\n", fileSize);
free(input);
return 0;
}
The problem is that it does not load files larger than 2GB at all...
Don't know where the problem is?

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
int main()
{
FILE *stream;
size_t size;
uint32_t fileSize = 0;
char *input;
int result1, result2;
// Open file, find the size of it
stream = fopen("4GB.bin", "rb");
if (result1 = _fseeki64(stream, 0L, SEEK_END) != 0)
{
printf("_fseeki64 result1 error\n");
}
fileSize = _ftelli64(stream);
if (result2 = _fseeki64(stream, 0L, SEEK_SET)!= 0)
{
printf("_fseeki64 result2 error\n");
}
input = (char *) malloc(fileSize + 1);
if (input == NULL) {
printf("Memory not allocated.\n");
exit(0);
}
//Stream file into memory
size = fread(input, 1, fileSize, stream);
input[size] = 0;
fclose(stream);
printf("size file is: %u\n", fileSize);
free(input);
return 0;
}

Related

Trying to read and write image as byte array in C

The following code is supposed to load and save an image file (and not only) into a copy file:
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
int main()
{
FILE* file = fopen("pexels.jpg", "r");
if (!file) {
perror("File opening failed");
return EXIT_FAILURE;
}
fseek(file, 0, SEEK_END);
long file_size = ftell(file);
fseek(file, 0, SEEK_SET);
void* data = malloc(file_size);
memset(data, 0, file_size);
fread(data, 1, file_size, file);
fclose(file);
FILE *copy = fopen("copy.jpg", "w");
if (!copy) {
perror("File opening failed");
free(data);
return EXIT_FAILURE;
}
fwrite(data, 1, file_size, copy);
free(data);
fclose(copy);
}
the file gets loaded and saved as an image using only array of bytes but the result gets corrupted.
I wonder what could be wrong here.
On windows you need to call fopen() with the 'b' flag to read and write binary files. You don't need memset() (and if you did, prefer calloc() instead). You will probably see similar performance writing 4k or 8k at a time and eliminate the error case of running out of memory if your input file is huge. In either case I recommend you check the return value from fread() and fread() and wrap the read/write operation in a loop.
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
int main()
{
FILE* file = fopen("pexels.jpg", "br");
if (!file) {
perror("File opening failed");
return EXIT_FAILURE;
}
fseek(file, 0, SEEK_END);
long file_size = ftell(file);
fseek(file, 0, SEEK_SET);
void* data = malloc(file_size);
fread(data, 1, file_size, file);
fclose(file);
FILE *copy = fopen("copy.jpg", "bw");
if (!copy) {
perror("File opening failed");
free(data);
return EXIT_FAILURE;
}
fwrite(data, 1, file_size, copy);
free(data);
fclose(copy);
}

memcpy returns junk data when copying from character buffer

I'm attempting to read 4 bytes from the start of a character buffer, but I'm having an issue. memcpy is returning junk.
buffer contains the contents of the file. Using breakpoints I see that the file starts with 41 53 45 46 or ASEF in ASCII. This is the file signature for an Adobe Swatch File.
But when I copy those 4 bytes from a character buffer, to a 4 byte array signature, I get random data.
Any idea what I'm doing wrong here?
#include <stdio.h>
#include <stdlib.h>
#include <memory.h>
int main(int argc, char **argv)
{
errno_t err = NULL;
FILE *fptr = NULL;
long fileSize = 0;
unsigned char* buffer;
int i, bytesRead;
char signature[4] = { 0 };
err = fopen_s(&fptr, argv[1], "rb");
if (err || fptr == NULL) {
fprintf(stderr, "Could not open file: %s\n", argv[1]);
return 1;
}
// Get filesize
fseek(fptr, 0, SEEK_END);
fileSize = ftell(fptr);
rewind(fptr);
// Allocate memory to store file contents
buffer = malloc(fileSize);
if (buffer == NULL) {
fprintf(stderr, "Could not allocate %i bytes of memory\n", fileSize);
return 1;
}
// Read file contents into buffer
bytesRead = fread(buffer, 1, fileSize, fptr);
if (bytesRead == 0) {
fprintf(stderr, "Failed to read bytes from file: %s\n", argv[1]);
return 1;
}
// Read and check signature
memcpy(signature, &buffer, 4);
fclose(fptr);
return 0;
}
This line is wrong:
memcpy(signature, &buffer, 4);
You want to copy the data in your buffer, not the value of the buffer pointer itself. That means you want:
memcpy(signature, buffer, 4);

Does fread work,when you want the elements in blocks?

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <time.h>
#pragma warning(disable:4996)
size_t file_size(FILE *fd){
if (fd == NULL) {
printf("File not found");
return -1;
}
fseek(fd, 0, SEEK_END);
return ftell(fd);
}
int main() {
FILE *in;
FILE *out;
char buffer[2] = { 0 };
int n = 0;
in = fopen("test.txt", "rb");
if (in == NULL) {
printf("cona\n"); return -1;
}
out = fopen("out.txt", "wb");
if (out == NULL) {
printf("cona1\n"); return -1;
}
size_t size = file_size(in);
for(n = 0; n < size; n += 2){
if (fread(&buffer, sizeof(char), 2, in) !=2) {
printf("cona2 \n");
} //keeps given erros in here
fwrite(&buffer, sizeof(char), 2, out);
memset(buffer, 0, sizeof(buffer));
}
printf(" \n in: %zu \n", size);
printf(" \n out: %zu \n", file_size(out));
fclose(out);
fclose(in);
system("PAUSE");
return(0);
}
My main question in here is if fread fuction works in a cycle, and if i can ask just n numbers of element in the file ervytime, and not the all file at once, i try this but it keeps me given error reading the file, it doesn't read anything.
There is a problem after your function finds the file size with
fseek(fd, 0, SEEK_END);
return ftell(fd);
because you do not rewind to the start of the file before trying to read from it.
rewind(fd);
or
fseek(fd, 0, SEEK_SET);

Better Way To Read From An exec'd Program's stdout in C

I read an exec'd program's stdout using a pipe:
int pipes[2];
pipe(pipes);
if (fork() == 0) {
dup2(pipes[1], 1);
close(pipes[1]);
execlp("some_prog", "");
} else {
char* buf = auto_read(pipes[0]);
}
To read from stdout, I have a function auto_read which automatically allocates more memory as needed.
char* auto_read(int fp) {
int bytes = 1000;
char* buf = (char*)malloc(bytes+1);
int bytes_read = read(fp, buf, bytes);
int total_reads = 1;
while (bytes_read != 0) {
realloc(buf, total_reads * bytes + 1);
bytes_read = read(fp, buf + total_reads * bytes, bytes);
total_reads++;
}
buf[(total_reads - 1) * bytes + bytes_read] = 0;
return buf;
}
The reason I do it this way is I don't know how much text the program is going to spew out ahead of time, and I don't want to create an overly large buffer and be a memory hog. I'm wondering if there is:
A cleaner way to write this.
A more memory or speed-efficient way of doing this.
Use popen if you only need to read from a process and are on a *NIX platform:
FILE *programStdout = popen("command", "r");
// read from programStdout (fread(), fgets(), etc.)
char buffer[1024];
while (fgets(buffer, 1024, programStdout))
{
puts(buffer);
}
EDIT: You asked for a way to map a programs output to a file, so here you go:
#import <stdio.h>
#import <unistd.h>
#import <sys/mman.h>
void *dataWithContentsOfMappedProgram(const char *command, size_t *len)
{
// read the data
char template[] = "/tmp/tmpfile_XXXXXX";
int fd = mkstemp(template);
FILE *output = fdopen(fd, "w+");
FILE *input = popen(command, "r");
#define BUF_SIZ 1024
char buffer[BUF_SIZ];
size_t readSize = 0;
while ((readSize = fread(buffer, 1, BUF_SIZ, input)))
{
fwrite(buffer, 1, readSize, output);
}
fclose(input);
input = NULL;
#undef BUF_SIZ
// now we map the file
long fileLength = ftell(output);
fseek(output, 0, SEEK_SET);
void *data = mmap(NULL, fileLength, PROT_READ | PROT_WRITE, MAP_FILE | MAP_PRIVATE, fd, 0);
close(fd);
if (data == MAP_FAILED)
return NULL;
return data;
}
int main()
{
size_t fileLen = 0;
char *mapped = dataWithContentsOfMappedProgram("echo Hello World!", &fileLen);
puts(mapped);
munmap(mapped, fileLen);
}

How can I get a file's size in C? [duplicate]

This question already has answers here:
How do you determine the size of a file in C?
(15 answers)
Closed 3 years ago.
How can I find out the size of a file I opened with an application written in C ?
I would like to know the size, because I want to put the content of the loaded file into a string, which I allocate using malloc(). Just writing malloc(10000*sizeof(char)); is IMHO a bad idea.
You need to seek to the end of the file and then ask for the position:
fseek(fp, 0L, SEEK_END);
sz = ftell(fp);
You can then seek back, e.g.:
fseek(fp, 0L, SEEK_SET);
or (if seeking to go to the beginning)
rewind(fp);
Using standard library:
Assuming that your implementation meaningfully supports SEEK_END:
fseek(f, 0, SEEK_END); // seek to end of file
size = ftell(f); // get current file pointer
fseek(f, 0, SEEK_SET); // seek back to beginning of file
// proceed with allocating memory and reading the file
Linux/POSIX:
You can use stat (if you know the filename), or fstat (if you have the file descriptor).
Here is an example for stat:
#include <sys/stat.h>
struct stat st;
stat(filename, &st);
size = st.st_size;
Win32:
You can use GetFileSize or GetFileSizeEx.
If you have the file descriptor fstat() returns a stat structure which contain the file size.
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
// fd = fileno(f); //if you have a stream (e.g. from fopen), not a file descriptor.
struct stat buf;
fstat(fd, &buf);
off_t size = buf.st_size;
I ended up just making a short and sweet fsize function(note, no error checking)
int fsize(FILE *fp){
int prev=ftell(fp);
fseek(fp, 0L, SEEK_END);
int sz=ftell(fp);
fseek(fp,prev,SEEK_SET); //go back to where we were
return sz;
}
It's kind of silly that the standard C library doesn't have such a function, but I can see why it'd be difficult as not every "file" has a size(for instance /dev/null)
How to use lseek/fseek/stat/fstat to get filesize ?
#include <fcntl.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
void
fseek_filesize(const char *filename)
{
FILE *fp = NULL;
long off;
fp = fopen(filename, "r");
if (fp == NULL)
{
printf("failed to fopen %s\n", filename);
exit(EXIT_FAILURE);
}
if (fseek(fp, 0, SEEK_END) == -1)
{
printf("failed to fseek %s\n", filename);
exit(EXIT_FAILURE);
}
off = ftell(fp);
if (off == -1)
{
printf("failed to ftell %s\n", filename);
exit(EXIT_FAILURE);
}
printf("[*] fseek_filesize - file: %s, size: %ld\n", filename, off);
if (fclose(fp) != 0)
{
printf("failed to fclose %s\n", filename);
exit(EXIT_FAILURE);
}
}
void
fstat_filesize(const char *filename)
{
int fd;
struct stat statbuf;
fd = open(filename, O_RDONLY, S_IRUSR | S_IRGRP);
if (fd == -1)
{
printf("failed to open %s\n", filename);
exit(EXIT_FAILURE);
}
if (fstat(fd, &statbuf) == -1)
{
printf("failed to fstat %s\n", filename);
exit(EXIT_FAILURE);
}
printf("[*] fstat_filesize - file: %s, size: %lld\n", filename, statbuf.st_size);
if (close(fd) == -1)
{
printf("failed to fclose %s\n", filename);
exit(EXIT_FAILURE);
}
}
void
stat_filesize(const char *filename)
{
struct stat statbuf;
if (stat(filename, &statbuf) == -1)
{
printf("failed to stat %s\n", filename);
exit(EXIT_FAILURE);
}
printf("[*] stat_filesize - file: %s, size: %lld\n", filename, statbuf.st_size);
}
void
seek_filesize(const char *filename)
{
int fd;
off_t off;
if (filename == NULL)
{
printf("invalid filename\n");
exit(EXIT_FAILURE);
}
fd = open(filename, O_RDONLY, S_IRUSR | S_IRGRP);
if (fd == -1)
{
printf("failed to open %s\n", filename);
exit(EXIT_FAILURE);
}
off = lseek(fd, 0, SEEK_END);
if (off == -1)
{
printf("failed to lseek %s\n", filename);
exit(EXIT_FAILURE);
}
printf("[*] seek_filesize - file: %s, size: %lld\n", filename, (long long) off);
if (close(fd) == -1)
{
printf("failed to close %s\n", filename);
exit(EXIT_FAILURE);
}
}
int
main(int argc, const char *argv[])
{
int i;
if (argc < 2)
{
printf("%s <file1> <file2>...\n", argv[0]);
exit(0);
}
for(i = 1; i < argc; i++)
{
seek_filesize(argv[i]);
stat_filesize(argv[i]);
fstat_filesize(argv[i]);
fseek_filesize(argv[i]);
}
return 0;
}
Have you considered not computing the file size and just growing the array if necessary? Here's an example (with error checking ommitted):
#define CHUNK 1024
/* Read the contents of a file into a buffer. Return the size of the file
* and set buf to point to a buffer allocated with malloc that contains
* the file contents.
*/
int read_file(FILE *fp, char **buf)
{
int n, np;
char *b, *b2;
n = CHUNK;
np = n;
b = malloc(sizeof(char)*n);
while ((r = fread(b, sizeof(char), CHUNK, fp)) > 0) {
n += r;
if (np - n < CHUNK) {
np *= 2; // buffer is too small, the next read could overflow!
b2 = malloc(np*sizeof(char));
memcpy(b2, b, n * sizeof(char));
free(b);
b = b2;
}
}
*buf = b;
return n;
}
This has the advantage of working even for streams in which it is impossible to get the file size (like stdin).
If you're on Linux, seriously consider just using the g_file_get_contents function from glib. It handles all the code for loading a file, allocating memory, and handling errors.
#include <stdio.h>
#define MAXNUMBER 1024
int main()
{
int i;
char a[MAXNUMBER];
FILE *fp = popen("du -b /bin/bash", "r");
while((a[i++] = getc(fp))!= 9)
;
a[i] ='\0';
printf(" a is %s\n", a);
pclose(fp);
return 0;
}
HTH

Resources