I managed to compile ncat. I am using -k option to keep server open. Instead of accepting data to STDOUT, my goal is to write to files instead. So far I was able to write to a file instead of STDOUT but my goal is to loop through new files on each new connection. Right now it is appending to the same filename_0 and f++ is not incrementing. Here is what I have so far. The original code will be below. The difference is in the else clause, basically if n is actually greater than 0. On each loop, n is 512 bytes until the last chunk. I just want to be able to have new files from each new connection. filename_0, filename_1, filename_3, etc.
MODIFIED CODE:
/* Read from a client socket and write to stdout. Return the number of bytes
read from the socket, or -1 on error. */
int read_socket(int recv_fd)
{
char buf[DEFAULT_TCP_BUF_LEN];
struct fdinfo *fdn;
int nbytes, pending;
int f = 0;
fdn = get_fdinfo(&client_fdlist, recv_fd);
ncat_assert(fdn != NULL);
nbytes = 0;
do {
int n, s;
n = ncat_recv(fdn, buf, 512, &pending);
if (n <= 0) {
if (o.debug)
logdebug("Closing fd %d.\n", recv_fd);
#ifdef HAVE_OPENSSL
if (o.ssl && fdn->ssl) {
if (nbytes == 0)
SSL_shutdown(fdn->ssl);
SSL_free(fdn->ssl);
}
#endif
close(recv_fd);
checked_fd_clr(recv_fd, &master_readfds);
rm_fd(&client_fdlist, recv_fd);
checked_fd_clr(recv_fd, &master_broadcastfds);
rm_fd(&broadcast_fdlist, recv_fd);
conn_inc--;
if (get_conn_count() == 0)
checked_fd_clr(STDIN_FILENO, &master_readfds);
return n;
}
else {
char filename[20];
snprintf(filename, sizeof(char) * 20, "filename_%i", f);
FILE *fp = fopen(filename, "a");
if (fp == NULL)
{
printf("Could not open file");
return 0;
}
//Write(STDOUT_FILENO, buf, n);
s = fwrite(buf, 1, n, fp);
fclose(fp);
f++;
nbytes += n;
}
} while (pending);
return nbytes;
}
ORIGINAL CODE:
int read_socket(int recv_fd)
{
char buf[DEFAULT_TCP_BUF_LEN];
struct fdinfo *fdn;
int nbytes, pending;
fdn = get_fdinfo(&client_fdlist, recv_fd);
ncat_assert(fdn != NULL);
nbytes = 0;
do {
int n;
n = ncat_recv(fdn, buf, sizeof(buf), &pending);
if (n <= 0) {
if (o.debug)
logdebug("Closing fd %d.\n", recv_fd);
#ifdef HAVE_OPENSSL
if (o.ssl && fdn->ssl) {
if (nbytes == 0)
SSL_shutdown(fdn->ssl);
SSL_free(fdn->ssl);
}
#endif
close(recv_fd);
checked_fd_clr(recv_fd, &master_readfds);
rm_fd(&client_fdlist, recv_fd);
checked_fd_clr(recv_fd, &master_broadcastfds);
rm_fd(&broadcast_fdlist, recv_fd);
conn_inc--;
if (get_conn_count() == 0)
checked_fd_clr(STDIN_FILENO, &master_readfds);
return n;
}
else {
Write(STDOUT_FILENO, buf, n);
nbytes += n;
}
} while (pending);
return nbytes;
}
I was able to figure out using the other functions involved. i passed a pointer into this function to write to it. the handler is a function i added the open() file pointer to.
Currently I am attempting to parse words from all text files in a directory (in this case it is safe to assume there will only be text files within the directory). It seems as though I am able to open the file within the threads function, however I am unable to grab the text within. No error messages are being presented but the printf within splitInput is not printing to the terminal.
Forgive my semantic work within the code, I am a fresh novice with C! Along with this there may be unused code within main as this will be part of a larger project. I appreciate the assistance in advance!
#include <stdlib.h>
#include <dirent.h>
#include <pthread.h>
#include <string.h>
#include <stdio.h>
#include <stdint.h>
#include "queue.h"
void* splitInput(void *filename) {
printf("Thread %s Created\n", (char*)filename);
FILE *file;
int i = 0;
char *cp;
char *bp;
char line[255];
char *array[5000];
file = fopen((char*)filename, "r");
if(file == NULL) {
perror("Error opening file");
}
printf("Opened File %s\n", (char*)filename);
while(fgets(line, sizeof(line), file) != NULL) {
bp = line;
while(1) {
cp = strtok(bp, ",.!? \n");
bp = NULL;
if(cp == NULL) {
break;
}
array[i++] = cp;
printf("Check print - word %i:%s:\n", i-1, cp);
}
}
fclose(file);
return 0;
}
int main(int argc, char *argv[]) {
DIR* d;
struct dirent* e;
// grab our queueSize and threadCount
int queueSize = atoi(argv[2]);
int threadCount = atoi(argv[3]);
// var for creating a thread each file
int i = 0;
// open the dir
d = opendir(argv[1]);
printf("Queue Size: %d\n", queueSize);
printf("Thread Count: %d\n", threadCount);
// set our thread count now that we know how many files are in dir
pthread_t threads[threadCount];
// read through our directory
while((e = readdir(d)) != NULL) {
// make sure we aren't reading . and ..
if(strcmp(e->d_name, ".") == 0) {
continue;
}
if(strcmp(e->d_name, "..") == 0) {
continue;
}
printf("entered file %s\n", e->d_name);
char *filename = strdup(e->d_name);
if(i < threadCount) {
// create our threads
pthread_create(&threads[i], NULL, splitInput, filename);
}
// increment i
i++;
}
// join our existing threads
for(int j = 0; j < i; j++) {
pthread_join(threads[j], NULL);
}
return 0;
}
Current Output
device#user:~/os/testdir$ ./output ~/os/testdir/test 10 10 output
Queue Size: 10
Thread Count: 10
entered file test
Thread test Created
Opened File test
Found the answer, I was attempting to open a file outside my working directory which cannot be done without the full path. Changing the working directory to the parameter given solved the issue. This can be done in this case with chdir(argv[1])
Revised code below.
#include <stdlib.h>
#include <dirent.h>
#include <pthread.h>
#include <string.h>
#include <stdio.h>
#include <stdint.h>
#include <unistd.h>
#include "queue.h"
void* splitInput(void *filename) {
printf("Thread %s Created\n", (char*)filename);
FILE *file;
int i = 0;
char *cp;
char *bp;
char line[255];
char *array[5000];
file = fopen((char*)filename, "r");
if(file == NULL) {
perror("Error opening file");
}
printf("Opened File %s\n", (char*)filename);
while(fgets(line, sizeof(line), file) != NULL) {
bp = line;
while(1) {
cp = strtok(bp, ",.!? \n");
bp = NULL;
if(cp == NULL) {
break;
}
array[i++] = cp;
printf("Check print - word %i:%s:\n", i-1, cp);
}
}
fclose(file);
return 0;
}
int main(int argc, char *argv[]) {
DIR* d;
struct dirent* e;
// grab our queueSize and threadCount
int queueSize = atoi(argv[2]);
int threadCount = atoi(argv[3]);
// var for creating a thread each file
int i = 0;
// open the dir
chdir(argv[1]);
d = opendir(argv[1]);
printf("Queue Size: %d\n", queueSize);
printf("Thread Count: %d\n", threadCount);
// set our thread count now that we know how many files are in dir
pthread_t threads[threadCount];
// read through our directory
while((e = readdir(d)) != NULL) {
// make sure we aren't reading . and ..
if(strcmp(e->d_name, ".") == 0) {
continue;
}
if(strcmp(e->d_name, "..") == 0) {
continue;
}
printf("entered file %s\n", e->d_name);
char *filename = strdup(e->d_name);
if(i < threadCount) {
// create our threads
pthread_create(&threads[i], NULL, splitInput, filename);
}
// increment i
i++;
}
// join our existing threads
for(int j = 0; j < i; j++) {
pthread_join(threads[j], NULL);
}
return 0;
}
I have a program that reads from a Random Access File and is to return the smallest and largest number in the file. One requirement is that this is done with 4 processes using fork() and piping the results. I divide the file up into 4 chunks and have each process evaluate a chunk of the file. I find the max and min of each chunk and write them to a pipe. At the end I will compare the piped values and find the largest and smallest of the values.
I am having trouble reading from the pipes as they are returning -1. Any insight on what I am doing wrong? Thanks!
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/wait.h>
int findMin(int start, int end, const char * filename);
int findMax(int start, int end, const char * filename);
//Calculates minimum and maximum of a number
int main(int argc, char * argv[])
{
const char * filename; // name of file to read
FILE * ft; // file handle for the file
int pid, // process id of this process
num, // the number of integer values in the file
i, // loop control variable for reading values
temp=0; // used to store each value read from the file
long size; // size in bytes of the input file
/*********************************************************************/
filename = argv[1]; // read the file named on the command line
ft= fopen(filename, "rb");
if (ft)
{
pid = getpid();
fseek (ft,0,SEEK_END); //go to end of file
size = ftell(ft); //what byte in file am I at?
fseek (ft,0,SEEK_SET); //go to beginning of file
num = (int)size / (int)sizeof(int); // number of integer values
printf("file size: %li bytes\n", size);
printf("sizeof(int) = %i bytes\n",(int) sizeof(int));
printf("how many integers = %i\n\n", num);
fclose(ft);
}
//Split file size into quarters to make 4 processes
int increment = num/4;
int num1 = increment;
int num2 = num1 + increment;
int num3 = num2 + increment;
int num4 = num;
int status;
int pid1 = -1;
int pid2 = -1;
//Pipes
int fdmin1[2];
int fdmax1[2];
int fdmin2[2];
int fdmax2[2];
int fdmin3[2];
int fdmax3[2];
int fdmin4[2];
int fdmax4[2];
//initializing pipes
if(pipe(fdmin1) == -1)
{
perror("Piping fd1 failed");
return 0;
}
if(pipe(fdmax1) == -1)
{
perror("Piping fd2 failed");
return 0;
}
if(pipe(fdmin2) == -1)
{
perror("Piping fd3 failed");
return 0;
}
if(pipe(fdmax2) == -1)
{
perror("Piping fd4 failed");
return 0;
}
if(pipe(fdmin3) == -1)
{
perror("Piping fd3 failed");
return 0;
}
if(pipe(fdmax3) == -1)
{
perror("Piping fd4 failed");
return 0;
}
if(pipe(fdmin4) == -1)
{
perror("Piping fd3 failed");
return 0;
}
if(pipe(fdmax4) == -1)
{
perror("Piping fd4 failed");
return 0;
}
//temp variables for pipes
int temp1;
int temp2;
int temp3;
int temp4;
int temp5;
int temp6;
int temp7;
int temp8;
pid1 = fork();
printf("pid1: %d \n", pid1);
if(pid1 > 0)
{
//Process 1
temp1 = findMin(0, num1, filename);
temp2 = findMax(0, num1, filename);
close(fdmin1[0]);
if(write(fdmin1[1], &temp1, sizeof(int)) == -1)
{
printf("Error writting to pipe");
}
close(fdmin1[1]);
close(fdmax1[0]);
if(write(fdmax1[1], &temp2, sizeof(int)) == -1)
{
printf("Error writting to pipe");
}
close(fdmax1[1]);
}
else if(pid1 == 0)
{
//Process 2
temp3 = findMin(num1, num2, filename);
temp4 = findMax(num1, num2, filename);
close(fdmin2[0]);
if(write(fdmin2[1], &temp3, sizeof(int)) == -1)
{
printf("Error writting to pipe");
}
close(fdmin2[1]);
close(fdmax2[0]);
if(write(fdmax2[1], &temp4, sizeof(int)) == -1)
{
printf("Error writting to pipe");
}
close(fdmax2[1]);
pid2 = fork();
printf("pid2: %d \n", pid2);
if(pid2 > 0)
{
//Process 3
temp5 = findMin(num2, num3, filename);
temp6 = findMax(num2, num3, filename);
close(fdmin3[0]);
if(write(fdmin3[1], &temp5, sizeof(int)) == -1)
{
printf("Error writting to pipe");
}
close(fdmin3[1]);
close(fdmax3[0]);
if(write(fdmax3[1], &temp6, sizeof(int)) == -1)
{
printf("Error writting to pipe");
}
close(fdmax3[1]);
}
else if(pid2 == 0)
{
//Process 4
temp7 = findMin(num3, num4, filename);
temp8 = findMax(num3, num4, filename);
close(fdmin4[0]);
if(write(fdmin4[1], &temp7, sizeof(int)) == -1)
{
printf("Error writting to pipe");
}
close(fdmin4[1]);
close(fdmax4[0]);
if(write(fdmax4[1], &temp8, sizeof(int)) == -1)
{
printf("Error writting to pipe");
}
close(fdmax4[1]);
}
}
//Close all pipe ends in all processes
close(fdmin1[0]);
close(fdmin1[1]);
close(fdmin2[0]);
close(fdmin2[1]);
close(fdmin3[0]);
close(fdmin3[1]);
close(fdmin4[0]);
close(fdmin4[1]);
close(fdmax1[0]);
close(fdmax1[1]);
close(fdmax2[0]);
close(fdmax2[1]);
close(fdmax3[0]);
close(fdmax3[1]);
close(fdmax4[0]);
close(fdmax4[1]);
//Wait for all processes to finish
int returnStatus;
waitpid(pid1, &returnStatus, 0);
int returnStatus2;
waitpid(pid2, &returnStatus2, 0);
//Make sure we are in parant process
if(pid1 > 0)
{
//Variables to compare min and max returned from processses
int min1;
int max1;
int min2;
int max2;
int min3;
int max3;
int min4;
int max4;
//read from pipe (error is occuring here)
close(fdmin1[1]);
if(read(fdmin1[0], &min1, sizeof(int)) == -1)
{
printf("Error reading");
}
close(fdmin1[0]);
printf("min1: %d \n", min1);
}
return 0;
}
//function to find the minimum in the file
int findMin(int start, int end, const char * filename)
{
int temp;
int smallestNum;
int i;
int length = end - start;
FILE * ft2;
ft2= fopen(filename, "rb");
fseek (ft2,start,SEEK_SET);
fread(&smallestNum,sizeof(int),1,ft2);
for(i = 0; i < length; i++)
{
fread(&temp,sizeof(int),1,ft2);
//printf("%d \n", temp);
if(temp < smallestNum)
{
smallestNum = temp;
}
/*
printf("%5i: %7i ",pid,temp);
if ((i+1)%5 == 0)
printf("\n");
*/
}
fclose(ft2);
printf("SmallestNum: %d \n", smallestNum);
return smallestNum;
}
//function to find maximum in file
int findMax(int start, int end, const char * filename)
{
int temp;
int largestNum;
int i;
int length = end - start;
FILE * ft3;
ft3= fopen(filename, "rb");
fseek (ft3,start,SEEK_SET);
fread(&largestNum,sizeof(int),1,ft3);
for(i = 0; i < length; i++)
{
fread(&temp,sizeof(int),1,ft3);
//printf("%d \n", temp);
if(temp > largestNum)
{
largestNum = temp;
}
/*
printf("%5i: %7i ",pid,temp);
if ((i+1)%5 == 0)
printf("\n");
*/
}
fclose(ft3);
printf("Largest Num: %d \n", largestNum);
return largestNum;
}
Here is the code for generating the Random Access File
/*
* This file generates a binary output file containing integers. It
* requires the output filename as a parameter and will take an
* argument indicating the number of values to generate as input.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#define BIAS 0 // a bias value added to the numbers to "bias" the file
// contents to provide an offset to the min and max
int main(int argc, char * argv[]) {
const char * filename; // name of the output file
FILE * ft; // file handle for output file
int numtogen = 1000000; // default is to generate 1,000,000 numbers
int randomnum, i; // variables used in the loop generating numbers
if (argc<2) { // not enough arguments, need output file name
printf("Usage: gendata <filename> [number of numbers]\n");
return 1;
}
if (argc == 3) // optional third argument for number of numbers
numtogen = atoi(argv[2]);
filename=argv[1]; // use the filename entered to store numbers
srand(time(NULL)); // seed the random number generator
ft= fopen(filename, "wb") ;
if (ft) {
for (i = 0; i < numtogen; i++){
randomnum = rand() % numtogen + BIAS;
fwrite(&randomnum,sizeof(int),1,ft);
}
fclose(ft);
}
return 0;
}
I am having trouble reading from the pipes as they are returning -1. Any insight on what I am doing wrong? Thanks!
this is because in the main process you close two times the pipe, doing
printf("pid1: %d \n", pid1);
if(pid1 > 0)
{
...
close(fdmin1[0]); <<< HERE
and
//Close all pipe ends in all processes
close(fdmin1[0]); <<< HERE
so it is closed when you do :
if(read(fdmin1[0], &min1, sizeof(int)) == -1)
do not close fdmin1[0] before to read in but the reverse.
Note you also close two times fdmin1[1] and fdmax1[0] and fdmax1[1].
The usage of the pipes is very strange and probably no what you want :
fdmin1 is a pipe between the main process and itself, the main process does if(write(fdmin1[1], &temp1, sizeof(int)) == -1)and later if(read(fdmin1[0], &min1, sizeof(int)) == -1) so that pipe is useless and min1 is temp1
the main process does if(write(fdmax1[1], &temp2, sizeof(int)) == -1) but nobody read that value, that pipe is useless and temp2 = findMax(0, num1, filename); is done for nothing.
the main process child does if(write(fdmin2[1], &temp3, sizeof(int)) == -1) and if(write(fdmax2[1], &temp4, sizeof(int)) == -1) and if(write(fdmin3[1], &temp5, sizeof(int)) == -1) and if(write(fdmax3[1], &temp6, sizeof(int)) == -1) but nobody read, these four pipes are useless and all the min/max computing are done for nothing.
it is the same for the third created process doing if(write(fdmin4[1], &temp7, sizeof(int)) == -1) and if(write(fdmax4[1], &temp8, sizeof(int)) == -1) but nobody read, these two pipes are useless and the min/max computing are done for nothing.
That means at the end you do not get the right min/max value in the main process, but only the min value of the first quarter computing by the main process and all other computing are lost.
The code
//Wait for all processes to finish
int returnStatus;
waitpid(pid1, &returnStatus, 0);
int returnStatus2;
waitpid(pid2, &returnStatus2, 0);
is executed by all the child processes, because you do not exit or return when you have to do.
You also have an undefined behavior because you have a race condition between your processes, the execution is not the same depending on where I had usleep in your code. A parent process must wait for the end of its child process when needed, you do not at the right moment. Note your process numbering is wrong, there are only the main process and two children, so 3 processes rather than 4, //process4 does not exist and that comment is in process 2.
Except in the main process you do not read from the right position in the file because for findMin and findMax the parameter start correspond to a rank of int rather than a position in the file, you must replace
fseek (ft2,start,SEEK_SET);
fseek (ft3,start,SEEK_SET);
by
fseek (ft2,start*sizeof(int),SEEK_SET);
fseek (ft3,start*sizeof(int),SEEK_SET);
You also (try to) read one int too many doing
int length = end - start;
...
fread(&smallestNum,sizeof(int),1,ft2);
for(i = 0; i < length; i++)
{
fread(&temp,sizeof(int),1,ft2);
for instance replace the loop to have
for(i = 1; i < length; i++)
There also are a lot of useless variables in your program, if I compile with option -Wall :
bruno#bruno-XPS-8300:/tmp$ gcc -Wall -g p.c -o p
p.c: In function ‘main’:
p.c:250:16: warning: unused variable ‘max4’ [-Wunused-variable]
int max4;
^
p.c:249:16: warning: unused variable ‘min4’ [-Wunused-variable]
int min4;
^
p.c:248:16: warning: unused variable ‘max3’ [-Wunused-variable]
int max3;
^
p.c:247:16: warning: unused variable ‘min3’ [-Wunused-variable]
int min3;
^
p.c:246:16: warning: unused variable ‘max2’ [-Wunused-variable]
int max2;
^
p.c:245:16: warning: unused variable ‘min2’ [-Wunused-variable]
int min2;
^
p.c:244:16: warning: unused variable ‘max1’ [-Wunused-variable]
int max1;
^
p.c:48:12: warning: unused variable ‘status’ [-Wunused-variable]
int status;
^
p.c:20:8: warning: unused variable ‘temp’ [-Wunused-variable]
temp=0; // used to store each value read from the file
^
p.c:19:8: warning: unused variable ‘i’ [-Wunused-variable]
i, // loop control variable for reading values
^
p.c:17:8: warning: variable ‘pid’ set but not used [-Wunused-but-set-variable]
int pid, // process id of this process
^
bruno#bruno-XPS-8300:/tmp$
Out of that
You must check the value of argc before to do filename = argv[1];.
If fopen(filename, "rb"); fails you must stop the execution, currently you continue with an undefined behavior.
Note also your program can be simplified using array of pipe rather than separated variables for them, allowing you to use a loop rather than the sequence of if(pipe(fdmin1) == -1) ... if(pipe(fdmax4) == -1) .... It is the same to start the child processes, rather than to duplicate the code use a function to write it only one time. Doing that you can have a definition allowing any number of child process rather than dedicated to 4 only.
Going back to the statement
I divide the file up into 4 chunks and have each process evaluate a chunk of the file
This is an extreme case but you have to manage the case the file is too small to be divided by 4, this is not the case in your proposal.
this is done with 4 processes
Considering the main process is count among the 4, 3 children must be created. But rather than to have each child creating an other one if needed, it is more simple to have the 3 children created by the main process and the parallelism is a little better.
A program must be simple, I already said you have a lot of variables for nothing and lot of code is duplicated, also :
It is useless to have so many pipes, only one is enough to allow each child to send the min/max it computed because the pipe reads and writes are guaranteed to be atomic up to PIPE_BUF (larger than the size of 2 int)
It is useless to read the file so many times, you can search for the min and the max at the same time.
And finally a proposal :
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/wait.h>
#define N 4 /* including the main process */
/* to send/receive result atomicaly through the pipe */
typedef struct {
int min, max;
} MinMax;
void findMinMax(long offset, long n, FILE * fp, MinMax * minmax);
//Calculates minimum and maximum of a number
int main(int argc, char * argv[])
{
const char * filename; // name of file to read
FILE * fp; // file handle for the file
long num; // the number of integer values in the file
long size; // size in bytes of the input file
long offset; // offset in file
int pp[2]; // the unique pipe
int pids[N-1];
MinMax minmax;
int i;
if (argc != 2) {
fprintf(stderr, "Usage: %s <filename>\n", *argv);
exit(-1);
}
filename = argv[1];
fp = fopen(filename, "rb");
if (fp == NULL) {
perror("cannot open file");
exit(-1);
}
/* get file size */
if (fseek(fp, 0, SEEK_END) == -1) { //go to end of file
perror("cannot fseek");
fclose(fp); /* also done automaticaly when exiting program */
exit(-1);
}
size = ftell(fp); //what byte in file am I at?
num = size / sizeof(int); // number of integer values
printf("file size: %li bytes\n", size);
printf("how many integers = %li\n\n", num);
if (num < N) {
fprintf(stderr, "the input file is too small, it must contains at least %i int\n", N);
fclose(fp); /* also done automaticaly when exiting program */
exit(-1);
}
//initializing pipe
if(pipe(pp) == -1) {
perror("Piping failed");
exit(-1);
}
offset = 0;
for (i = 0; i != N-1; ++i) {
pids[i] = fork();
switch (pids[i]) {
case 0:
/* child */
{
FILE * fp2 = fopen(filename, "rb");
if (fp2 == NULL) {
perror("child cannot open file");
exit(-1);
}
findMinMax(offset, num/N, fp2, &minmax);
printf("min max child %d : %d %d\n", i, minmax.min, minmax.max);
if (write(pp[1], &minmax, sizeof(minmax)) != sizeof(minmax)) {
perror("Error writting to pipe");
exit(-1);
}
}
exit(0);
case -1:
/* parent */
perror("Cannot fork");
exit(-1);
default:
/* parent, no error */
offset += (num/N)*sizeof(int);
}
}
findMinMax(offset, (size - offset)/sizeof(int), fp, &minmax);
printf("min max main : %d %d\n", minmax.min, minmax.max);
for (i = 0; i != N-1; ++i) {
int status;
MinMax mm;
if ((waitpid(pids[i], &status, 0) != -1) &&
(status == 0) &&
(read(pp[0], &mm, sizeof(mm)) == sizeof(mm))) {
if (mm.min < minmax.min)
minmax.min = mm.min;
if (mm.max > minmax.max)
minmax.max = mm.max;
}
else
fprintf(stderr, "cannot get result for child %d\n", i);
}
printf("global min max : %d %d\n", minmax.min, minmax.max);
return 0;
}
// function to find the minimum and maximum in the file
// n > 1
void findMinMax(long offset, long n, FILE * fp, MinMax * minmax)
{
int v;
if (fseek(fp, offset, SEEK_SET) == -1) {
perror("cannot fseek");
exit(-1);
}
if (fread(&minmax->min, sizeof(minmax->min), 1, fp) != 1) {
fclose(fp); /* also done automaticaly when exiting program */
perror("cannot read int");
exit(-1);
}
minmax->max = minmax->min;
while (--n) {
if (fread(&v, sizeof(v), 1, fp) != 1) {
fclose(fp); /* also done automaticaly when exiting program */
perror("cannot read int");
exit(-1);
}
if (v < minmax->min)
minmax->min = v;
if (v > minmax->max)
minmax->max = v;
}
fclose(fp); /* also done automaticaly when exiting program */
}
As you can see the code is much simple and I just have to modify #define N 4 to an other value to change the number of processes working in parallel.
Using your second program to generate 1000000 int in aze, compilation and execution of my proposal :
bruno#bruno-XPS-8300:/tmp$ gcc -g -Wall p.c
bruno#bruno-XPS-8300:/tmp$ ./a.out aze
file size: 4000000 bytes
how many integers = 1000000
min max main : 2 999995
min max child 0 : 10 999994
min max child 2 : 0 999998
min max child 1 : 3 999999
global min max : 0 999999
bruno#bruno-XPS-8300:/tmp$
I have this code that reads in two int numbers from a file. And stores it in a buffer[] to be taken in a second function to be used. I'm not sure if my stopping conditions in the first function are correct. It looks fine but when running the code it stops at the second function.
static int count = 0;
int buffer[5];
int requestNum = 1;
FILE* f;
int main(int argc, char** argv)
{
/*THREAD IDs*/
pthread_t liftR, lift1;
/*OPEN FILE*/
f = fopen("sim_input.txt", "r");
/*CREATE THREAD*/
pthread_create(&liftR, NULL, request, NULL);
pthread_create(&lift1, NULL, lift, NULL);
/*RUNS TILL THREADS FINISHED*/
pthread_join(liftR, NULL);
pthread_join(lift1, NULL);
/*CLEAN UP*/
fclose(f);
return 0;
}
void* request(void* data1)
{
int req1, req2, eof;
/*NOT EOF*/
while(eof != -1)
{
/*READ ONE REQUEST*/
eof = fscanf(f, "%d %d", &req1, &req2);
/*CHECK IF BUFFER FULLL*/
if(count < 5)
{
/*ADD REQUEST TO BUFFER*/
buffer[count] = req1;
buffer[count + 1] = req2;
count = count + 2;
printf("COUNT: %d\n", count);
/*PRINTING*/
printf("-------------------------------------------\n");
printf("From Buffer -> Item1: %d, Item2: %d\n", req1, req2);
printf("Request No: %d\n", requestNum);
printf("-------------------------------------------\n");
requestNum++;
}
}
return NULL;
}
void* lift(void* data2)
{
while(count > 0)
{
sleep(1);
printf("================\n");
printf("COUNT: %d\n", count);
printf("REMOVE ITEM FROM BUFFER - DO STUFF WITH IT\n");
printf("================\n");
count = count - 2;
}
return NULL;
}
OUTPUT:
Shows count 2, 4, 6. Only 3 request shown, In file it goes up to 10 request
There are multiple bugs in your program:
Reading uninitialized eof variable:
int int req1, req2, eof; // What is the value of eof? It could be -1 (or anything else).
/*NOT EOF*/
while(eof != -1)
You throw away the data you read:
if(count < 5)
{
/*ADD REQUEST TO BUFFER*/
}
If the first thread runs for a while before the second thread starts, then it will store the first two requests into the buffer, and throw away the rest.
To fix this, you need to wait for the second thread to drain the buffer when it is full.
You access count without any locking, which is a data race and undefined behavior. You must protect reading and writing shared (between threads) globals with a mutex.
I'm trying to test possible problems for reading from and writing to same files concurrently on an embedded device which has embedded Linux on it.
I have two processes: Writer and Reader. As the names suggest, the Writer constantly writes 1 of 3 const strings to 10 files one-by-one and at the same time the Reader reads the same files and compares the outputs to be sure they are correct. In order to overcome synchronization issues, I thought I could use mandatory file locking mechanism.
Note: Reader process is actually a placeholder for an user application which will read the files in a similar fashion. Since it will not be in my control, advisory file locking will not be applicable.
I have mounted a tmpfs with mandatory file locking enabled.
mount -t tmpfs -o mand,size=1m tmpfs /tmp2
The Writer creates 10 files with mandatory file locking enabled. And in each iteration one of the three strings below is written to files.
const char* s1 = "31415926535897932384626433832795028841971693993751058209749445923078164062862089986280348253421170679";
const char* s2 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
const char* s3 = "************************************************************************";
The Reader reads these files one-by-one and reports if there is a mismatch. And it occasionally (roughly once in every 1000 iterations, see the codes below) reads strings like:
"************************************************************************62862089986280348253421170679"
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz********************"
This is exactly the problem I was expecting to encounter. The smaller strings are written on top of the bigger ones and the remaining characters are not deleted.
I tried to use fsync(fn) after writing to file, but it did not work. It should not be needed anyway, since this is a tmpfs.
What is the reason for this and what can I do to avoid this?
Here are the codes for the Writer and the Reader:
The Writer
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <sys/fcntl.h>
#include <sys/stat.h>
#include <errno.h>
const char* s1 = "31415926535897932384626433832795028841971693993751058209749445923078164062862089986280348253421170679";
const char* s2 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
const char* s3 = "************************************************************************";
int main()
{
// create /tmp2/test directory
if (mkdir("/tmp2/test", 0777) == -1) {
if (errno != EEXIST) {
perror("mkdir");
return -1;
}
}
// create 10 input files input0, input1, ..., input9
for (int i = 0; i < 10; i++) {
char path[50] = "/tmp2/test/input";
sprintf(path + 16, "%d", i);
FILE* fp = fopen(path, "w");
if (fp == NULL) {
fprintf(stderr, "Unable to create file %s\n", path);
perror("create: ");
return -1;
}
chmod(path, 02644); //mandatory file locking enabled
fclose(fp);
}
// flock structure
struct flock fl;
fl.l_type = F_WRLCK;
fl.l_whence = SEEK_SET;
fl.l_start = 0;
fl.l_len = 0;
fl.l_pid = 0;
// loop
for(int step = 0; ; step++) {
usleep(50000);
for (int i = 0; i < 10; i++) {
char path[50] = "/tmp2/test/input";
sprintf(path + 16, "%d", i);
FILE* fp = fopen(path, "w+");
if (fp == NULL) {
fprintf(stderr, "Unable to open file %s\n", path);
perror("fopen: ");
return -1;
}
int fd = fileno(fp);
// place a write lock
fl.l_type = F_WRLCK;
if (fcntl(fd, F_SETLK, &fl) == -1) {
perror("lock: ");
return -1;
}
// write 1 of 3 strings
if (step % 3 == 0) {
write(fd, s1, strlen(s1));
} else if (step % 3 == 1) {
write(fd, s2, strlen(s2));
} else {
write(fd, s3, strlen(s3));
}
//fsync(fd); // fsync should not be needed since this is a tmpfs
// unlock
fl.l_type = F_UNLCK;
if (fcntl(fd, F_SETLK, &fl) == -1) {
perror("lock: ");
return -1;
}
fclose(fp);
}
}
return 0;
}
The Reader:
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <sys/stat.h>
#include <errno.h>
const char* s1 = "31415926535897932384626433832795028841971693993751058209749445923078164062862089986280348253421170679";
const char* s2 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
const char* s3 = "************************************************************************";
int main()
{
// counters for errors
int err_ctr = 0;
int read_err_ctr = 0;
// loop
for(int step = 0; ; step++) {
usleep(50000);
for (int i = 0; i < 10; i++) {
char path[50] = "/tmp2/test/input";
sprintf(path + 16, "%d", i);
FILE* fp = fopen(path, "r");
if (fp == NULL) {
fprintf(stderr, "Unable to open file %s\n", path);
perror("read: ");
return -1;
}
// read the file
char reading[150];
if (fgets(reading, 150, fp) == NULL) {
fclose(fp);
read_err_ctr++;
printf("Step = %d; ReadError = %d; from %s\n", step, read_err_ctr, path);
continue;
}
fclose(fp);
// continue if the string matches
if (strcmp(reading, s1) == 0) {
continue;
} else if (strcmp(reading, s2) == 0) {
continue;
} else if (strcmp(reading, s3) == 0) {
continue;
}
// print error
err_ctr++;
printf("Step = %d; Error = %d; I've read %s from %s\n", step, err_ctr, reading, path);
}
}
return 0;
}