C read big matrix from input file - c

Question

Im trying to read a 40000x40000 boolean (binary) matrix from a input file and store it in a variable. After I store it in the variable, I want to write it to a file. However, with the code I wrote it takes more than a hour. Can someone help me out? I think im doing something wrong.
Code
void get_grid_values_file(bool *grid, int n, int m, char *input_filename){
FILE *in_file;
in_file = fopen(input_filename, "r");
char buffer[1];
bool search = true;
int k=0;
while(search){
fseek(in_file, k, SEEK_SET);
fread(buffer, 1, 1, in_file);
if(*buffer == '\n')
search = false;
k++;
}
int i,j;
for(i=0; i<n; i++){
for(j=0; j<m; j++){
fseek(in_file, k, SEEK_SET);
fread(buffer, 1, 1, in_file);
*((grid+i*m) + j) = atof(buffer);
k+=2;
}
}
fclose(in_file);
}
void set_grid_values_file(bool *grid, int n, int m, char *output_filename){
FILE *out_file;
out_file = fopen(output_filename, "w");
char buffer[1] = " ";
//Set n,m and spaces
int length_n= (int) (log10 (abs (n))) + 1;
char char_n[length_n];
sprintf(char_n, "%d", n);
fseek(out_file, 0, SEEK_SET);
fwrite (char_n, length_n, 1, out_file);
fseek(out_file, length_n, SEEK_SET);
fwrite (" ", 1, 1, out_file);
int length_m= (int) (log10 (abs (m))) + 1;
char char_m[length_m];
sprintf(char_m, "%d", m);
fseek(out_file, length_n+1, SEEK_SET);
fwrite (char_m, length_m, 1, out_file);
fseek(out_file, length_n+1+length_m, SEEK_SET);
fwrite ("\n", sizeof(char), 1, out_file);
//Set grid
int i,j;
int k =length_n + length_m + 2;
for(i=0; i<n; i++){
for(j=0; j<m; j++){
fseek(out_file, k, SEEK_SET);
buffer[0] = (*((grid+i*m) + j) == true ? '1' : '0');
fwrite (buffer, 1, 1, out_file);
k++;
fseek(out_file, k, SEEK_SET);
fwrite (" ", 1, 1, out_file);
k++;
}
fseek(out_file, k, SEEK_SET);
fwrite ("\n", sizeof(char), 1, out_file);
k++;
}
}
int main(int argc, char *argv[])
{
char *input_filename = "gen0_40kx40k.in";
char *output_filename = "gol_output.out";
int n = 40000;
int m = 40000;
bool *grid = (bool *)malloc(n*m*sizeof(bool));
//Read
get_grid_values_file((bool *)grid, n, m, input_filename);
//Write
set_grid_values_file((bool *)grid, n, m, output_filename);
return 0;
}
Input format, the first line contains the dem of the 2d matrix:
20 20
1 0 1 0 0 1 0 0 1 0 1 0 1 0 1 1 0 1 0 0
1 1 0 0 0 0 0 1 0 1 1 1 1 0 0 1 0 0 1 1
0 1 1 0 1 0 1 1 0 0 0 1 1 0 0 1 1 0 0 1
1 0 1 1 0 1 0 0 1 0 1 1 1 0 1 0 1 1 1 1
1 1 0 0 0 0 0 1 0 0 0 0 0 1 1 1 0 1 1 1
1 1 1 0 1 0 0 0 1 0 1 0 1 0 1 1 0 1 0 1
1 1 1 0 0 0 1 1 1 0 0 1 1 1 1 1 0 0 1 1
0 1 0 1 1 1 1 1 0 1 0 1 0 1 1 1 1 0 0 1
1 0 0 0 0 1 1 0 1 1 1 1 0 1 1 1 0 0 0 0
1 1 0 0 0 1 1 0 0 1 1 1 1 1 1 1 0 1 1 0
0 1 1 0 0 0 1 1 0 1 1 0 0 1 0 1 1 1 1 1
0 0 0 1 1 1 1 1 0 0 1 0 1 1 0 0 1 1 1 0
1 0 0 1 0 0 0 1 1 1 0 1 1 0 0 0 1 1 0 0
0 1 0 1 0 1 1 0 0 0 0 1 1 0 1 1 0 1 0 1
1 0 0 0 0 0 1 1 0 1 1 0 0 0 1 1 1 1 1 1
1 1 1 0 1 0 1 1 1 1 0 0 1 0 1 1 0 0 1 0
1 0 0 1 0 0 0 1 1 1 1 0 0 0 1 1 0 0 0 1
0 0 0 1 0 1 1 1 1 0 1 0 0 0 1 1 0 0 0 1
1 1 1 0 1 0 1 1 1 0 1 1 0 1 0 1 0 1 0 1
1 1 1 0 1 0 1 1 0 1 0 0 1 0 0 0 0 1 0 1

Answer 1

Read larger chunks from the input file - not every char as a separate call. Eg a complete row of your matrix at once.
Why do you use in the inner loop of set_grid_values_file two fwrite calls. Better to combine them:
char buffer[2] = " ";
for(i=0; i<n; i++){
int base = grid+i*m;
for(j=0; j<m; j++){
fseek(out_file, k, SEEK_SET);
buffer[0] = (*(base + j) ? '1' : '0');
fwrite (buffer, 1, 2, out_file);
k+=2;
}

Answer 2

I would suggest removing the calls to fseek.
while(search){
// Make sure the read is successful. Otherwise, break out of the loop.
if ( fread(buffer, 1, 1, in_file) != 1 )
{
break;
}
if(*buffer == '\n')
search = false;
k++;
}
int i,j;
// Rewind the file
fseek(in_file, 0, SEEK_SET);
for(i=0; i<n; i++){
for(j=0; j<m; j++){
// Make sure the read is successful. Otherwise, break out of the loop.
if ( fread(buffer, 1, 1, in_file) != 1 )
{
break;
}
*((grid+i*m) + j) = atof(buffer);
k+=2;
}
}
Also atof(buffer) is a problem when buffer has only one element in it. Use at least a two element array.
char buffer[2] = {0};

Answer 3

This is about the simplest(and possibly the fastest) method.
stdio is buffered, getc() is most probably a macro
function calls are costly (they trash the instruction pipeline and cache) ; you use 1seek+1fread per boolean bit read.
you dont need to seek the file; just read it sequentially and put the values at the correct{row,col} positions.
setting and testing an indicator variable (your search) is a waste of time (often taught in Pascal and Java classes...) ;instead: just jump out of the loop (or continue)
I am assuming an ASCII file with spaces between the values, but without the{nrow,ncol} at the top (the{n,m}values are supplied as function arguments)
void get_grid_values_file(bool *grid, int n, int m, char *input_filename)
{
unsigned col,row;
FILE * fp;
fp= fopen (input_filename, "r" );
if(!fp)return;
for(row=col=0; ; ) {
int ch;
ch=getc(fp);
if (ch == EOF)break;
if (ch < '0' || ch > '1') continue;
grid[row*m+col++] = (ch == '0') ? False :True;
if (col == m) {col=0; row++; }
if (row == n) break;
}
fclose(fp);
return;
}