Here are two problems in the program
First, is that when I uncomment the pthread_join() in the main function, there will be a seg fault, other wise the program will run...
Second, is that the output file will be missing the first letter of each word that has stored in the global variable words from last read file. So, for example, there are two files:
one has words "abc abc abc abc abc abc abc abc".
the second has words "def def"
if i input 5 for the second argument when calling a.out, the output in the output file will be
abc
abc
abc
abc
abc
bc
bc
bc
def
def
This is also a werid thing I could not figure out why.
/* main.c */
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <dirent.h>
#include <ctype.h>
#include <pthread.h>
#include "hw3.h"
int index_;
pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
typedef struct files
{
char *inputfile;
FILE * outputfile;
} files;
void * readFile( void *arg ){
files *info = (files *)arg;
char fileName[80];
strncat(fileName, (info->inputfile), 79);
fileName[80] = '\0';
FILE *outputfd = info->outputfile;
FILE* fd;
fd = fopen(fileName, "r");
if ( fd == NULL) {
fprintf(stderr, "ERROR:<open() failed>\n");
}
printf("TID %d: Opened \"%s\"\n", (unsigned int)pthread_self(), fileName);
fflush(stdout);
int rc;
char ch[1] = {0};
char word[80] = {0};
ch[0] = fgetc(fd);
pthread_mutex_lock(&mutex);
while( ch[0] != EOF){
if( isalnum(ch[0]) ){
// char str = ch[0];
strncat(word, ch, 1);
}
else{//it's a word
if( strlen( word ) >= 2 ){
words[index_] = word;
printf("TID %d: Stored \"%s\" in shared buffer at index [%d]\n",(unsigned int)pthread_self(), word, index_ );
if( index_+ 1 == maxwords ){
index_ = 0;
printf("MAIN: Buffer is full; writing %d words to output file\n", maxwords);
for( unsigned int i = 0; i<maxwords; i++ ){
rc = fwrite( words[i], 1, sizeof(words[i]), outputfd );
fwrite( "\n", 1, sizeof("\n"), outputfd );
if( rc == -1 ){
fprintf(stderr, "ERRPR:<write() failed>\n");
//return EXIT_FAILURE;
}
}
}
else{
index_ ++;
}
}
for(int i = 0; i< strlen(word); i++){
word[i] = '\0';
}
}
ch[0] = fgetc(fd);
}
pthread_mutex_unlock(&mutex);
printf("TID %d: Closed \"%s\"; and exiting\n", (unsigned int)pthread_self(), fileName );
fclose(fd);
pthread_exit( NULL );
}
int main( int argc, char * argv[] ){
if(argc != 4){
fprintf(stderr, "ERROR: Invalid arguments\nUSAGE: ./a.out <input-directory> <buffer-size> <output-file>\n");
return EXIT_FAILURE;
}
//dynamically allocated words buffer with argument 2
maxwords = atoi(argv[2]);
words = (char**)calloc(maxwords, sizeof(char*) );
if ( words == NULL)
{
fprintf( stderr, "ERROR:<word calloc() failed\n>" );
return EXIT_FAILURE;
}
printf("MAIN: Dynamically allocated memory to store %d words\n", maxwords);
fflush(stdout);
//open/create output file of the third argument
FILE* outputfd = fopen (argv[3], "w");
if ( outputfd == NULL )
{
perror( "open() failed" );
return EXIT_FAILURE;
}
DIR * dir = opendir( argv[1] );
if(dir == NULL){
perror("ERRPR:<opendir() failed>");
return EXIT_FAILURE;
}
chdir(argv[1]);
printf("MAIN: Opened \"%s\" directory\n", argv[1]);
fflush(stdout);
pthread_t tid[10];
index_ = 0;
int i = 0;//files index
struct dirent * file;
//files allfiles[20];
char fileName[80];
int rc;
//-----------------------------------------------------------------------
// while loop reads all files in the directory
while ( ( file = readdir( dir ) ) != NULL )
{
struct stat buf;
rc = lstat( file->d_name, &buf ); /* e.g., "xyz.txt" */
/* ==> "assignments/xyz.txt" */
if ( rc == -1 ){
fprintf(stderr, "ERRPR:<lstat() failed>\n");
return EXIT_FAILURE;
}
if ( S_ISREG( buf.st_mode ) )
{
// printf( " -- regular file\n" );
// fflush(stdout);
strncpy(fileName, file->d_name, 79);
files info;
info.inputfile = fileName;
info.outputfile = outputfd;
//printf("%d",i);
printf("MAIN: Created child thread for \"%s\"\n",fileName);
rc = pthread_create( &tid[i], NULL, readFile,(void *)&info );
sleep(1);
i++
}
else if ( S_ISDIR( buf.st_mode ) )
{
// printf( " -- directory\n" );
// fflush(stdout);
}
else
{
// printf( " -- other file\n" );
// fflush(stdout);
}
}
closedir(dir);
printf("MAIN: Closed \"%s\" directory\n", argv[1]);
fflush(stdout);
printf("MAIN: Created \"%s\" output file\n",argv[3]);
fflush(stdout);
//-----------------------------------------------------------------------
for( int j = 0; j<i; j++){
printf( "MAIN: Joined child thread: %u\n", (unsigned int)tid[j] );
pthread_join(tid[i], NULL);
}
for( unsigned int i = 0; i<index_; i++ ){
int rc = fwrite( words[i], 1, sizeof(words[i]), outputfd );
if( rc == -1 ){
fprintf(stderr, "ERRPR:<write() failed>\n");
return EXIT_FAILURE;
}
}
printf( "MAIN: All threads are done; writing %d words to output file\n", index_);
fflush(stdout);
free( words );
fclose( outputfd );
return EXIT_SUCCESS;
}
This here is the whole program, and there is a header file which is just two global variab
char ** words = NULL;
/* global/shared integer specifying the size */
/* of the words array (from argv[2]) */
int maxwords;
Thanks to everyone for the help!
You need separate info objects for each thread. Right now, all of the threads get the same info object, which you change in between creating threads, and therefore, for most of them, by the time they get a chance to look at the name of the file they are supposed to process, it has been changed.
The segmentation fault is being caused by code you have not shown us, so I can't help you with that except to suggest that you apply valgrind.
Here are two more bugs:
char fileName[80];
strncat(fileName, (info->inputfile), 79);
You can only concatenate onto a string, not an unitialized array of characters that may or may not contain a valid string.
char ch[1] = {0};
char word[80] = {0};
ch[0] = fgetc(fd);
pthread_mutex_lock(&mutex);
while( ch[0] != EOF){
The fgets function returns an integer that will be EOF on end of file, otherwise it returns the character value. You convert it to a char and then compare the char to EOF. But that makes no sense since EOF is the integer value that represents end of file. Once cast to a character, it is a valid character that could have been read from the file since the file can contain any characters and "end of file" is not a character.
Related
I am writing some C code to process some data in a file, but I just learned that the file is going to be constantly added to (about 1 time/second, maybe faster). So I'm wondering how do I keep reading from the file as its being added to. Then when I get to the end, wait until the next line is added and then process it. Then wait again and then process, and so on and so on. I have something like:
while(1){
fgets(line, sizeof(line), file);
while(line == NULL){
//wait ? then try to read again?
}
//tokenize line and do my stuff here
}
I thought I could maybe use inotify, but I am getting nowhere with that. Does anyone have any advice?
The most efficient way is using inotify, and the direct way is using the read() system call directly.
using inotify
The following code may give you some help, It works well on Debian 7.0, GCC 4.7:
/*This is the sample program to notify us for the file creation and file deletion takes place in “/tmp/test_inotify” file*/
// Modified from: http://www.thegeekstuff.com/2010/04/inotify-c-program-example/
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/inotify.h>
#define EVENT_SIZE ( sizeof (struct inotify_event) )
#define EVENT_BUF_LEN ( 1024 * ( EVENT_SIZE + 16 ) )
int main( )
{
int length, i = 0;
int fd;
int wd;
char buffer[EVENT_BUF_LEN];
/*creating the INOTIFY instance*/
fd = inotify_init();
/*checking for error*/
if ( fd < 0 ) {
perror( "inotify_init error" );
}
/* adding the “/tmp/test_inotify” test into watch list. Here,
* the suggestion is to validate the existence of the
* directory before adding into monitoring list.
*/
wd = inotify_add_watch( fd, "/tmp/test_inotify", IN_CREATE | IN_DELETE | IN_ACCESS | IN_MODIFY | IN_OPEN );
/* read to determine the event change happens on “/tmp/test_inotify” file.
* Actually this read blocks until the change event occurs
*/
length = read( fd, buffer, EVENT_BUF_LEN );
/* checking for error */
if ( length < 0 ) {
perror( "read" );
}
/* actually read return the list of change events happens.
* Here, read the change event one by one and process it accordingly.
*/
while ( i < length ) {
struct inotify_event *event = ( struct inotify_event * ) &buffer[ i ];
if( event->len == 0) {
// For a single file watching, the event->name is empty, and event->len = 0
printf(" Single file watching event happened\n");
} else if ( event->len ) {
if ( event->mask & IN_CREATE ) {
if ( event->mask & IN_ISDIR ) {
printf( "New directory %s created.\n", event->name );
} else {
printf( "New file %s created.\n", event->name );
}
} else if ( event->mask & IN_DELETE ) {
if ( event->mask & IN_ISDIR ) {
printf( "Directory %s deleted.\n", event->name );
} else {
printf( "File %s deleted.\n", event->name );
}
} else if( event->mask & IN_ACCESS ) {
if ( event->mask & IN_ISDIR ) {
printf( "Directory %s accessed.\n", event->name );
} else {
printf(" File %s accessed. \n", event->name );
}
} else if( event->mask & IN_MODIFY ) {
if ( event->mask & IN_ISDIR ) {
printf( "Directory %s modified.\n", event->name );
} else {
printf(" File %s modified. \n", event->name );
}
} else if( event->mask & IN_OPEN ) {
if ( event->mask & IN_ISDIR ) {
printf( "Directory %s opened.\n", event->name );
} else {
printf(" File %s opened. \n", event->name );
}
} else {
printf( "Directory or File is accessed by other mode\n");
}
}
i += EVENT_SIZE + event->len;
}
/* removing the “/tmp/test_inotify” directory from the watch list. */
inotify_rm_watch( fd, wd );
/* closing the INOTIFY instance */
close( fd );
}
When runing the above program. You could test it by create a file or directoy named /tmp/test_inotify.
A detailed explanation could be found here
Use read system call
If a file is open, and have read to the end of current file size. the read() system call will return 0. And if some writer wrote N bytes to this file later, and then the read() will just return min(N, buffersize).
So it works correctly for your circumstance. Following is an examples of the code.
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
typedef int FD ;
int main() {
FD filed = open("/tmp/test_inotify", O_RDWR );
char buf[128];
if( !filed ) {
printf("Openfile error\n");
exit(-1);
}
int nbytes;
while(1) {
nbytes = read(filed, buf, 16);
printf("read %d bytes from file.\n", nbytes);
if(nbytes > 0) {
split_buffer_by_newline(buf); // split buffer by new line.
}
sleep(1);
}
return 0;
}
Reference
Thanks to Jonathan Leffler's Comment
http://www.thegeekstuff.com/2010/04/inotify-c-program-example/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
int
main()
{
char ch;
FILE *fp;
long int nbytes_read = 0;
char str [128];
int j = 0;
int first_time = 1;
memset(str, '\0', 128);
fp = fopen("file.txt", "r");
while (1) {
if (first_time != 1) {
fp = fopen("file.txt", "r");
fseek(fp, nbytes_read, SEEK_SET);
sleep(10);
}
if (fp != NULL) {
while ((ch = fgetc(fp)) != EOF) {
if (ch == '\n') {
str[j++] = ch;
printf("%s", str);
memset(str, '\0', 128);
j = 0;
} else {
str[j++] = ch;
}
nbytes_read++;
}
//printf("%ld\n", nbytes_read);
first_time = 0;
}
fclose(fp);
}
return 0;
}
You can use select() with the fileno(file) as the file-descriptor. select will return either with a timeout (if you set a timeout) or when you can read from the file.
Using select can be a good choice but if you do not wish to use it, you can add a sleep for a small amount of milliseconds before reading value.
Here is my code, for my own shell in C. When compiling I get an error: use of undeclared identifier 'output'. Here are examples of some of the errors when compiling:
error: use of undeclared identifier 'output' char
input[100];output[100];
test3.c:53:15: error: use of undeclared identifier 'output'
strcpy(output,args[i+1]);
^
test3.c:53:15: error: use of undeclared identifier 'output'
test3.c:60:8: warning: implicit declaration of function 'open' is
invalid in C99
[-Wimplicit-function-declaration] j = open(input, O_RDONLY, 0);
^
test3.c:60:20: error: use of undeclared identifier 'O_RDONLY' j =
open(input, O_RDONLY, 0);
^
test3.c:61:29: error: use of undeclared identifier 'O_RDONLY'
if ((j = open(input, O_RDONLY, 0)) < 0) {
test3.c:70:12: warning: implicit declaration of function 'creat' is
invalid in
C99 [-Wimplicit-function-declaration] if ((i= creat(output , 0644)) < 0) {
test3.c:70:18: error: use of undeclared identifier 'output' if ((i=
creat(output , 0644)) < 0) {
Here is my code:
#include "stdio.h"
#include "stdlib.h"
#include "string.h"
#include "signal.h"
#include "unistd.h"
void prompt(char*);
void execute( char* );
char** parse( char* );
int main( int ac, char* av[] )
{
char input[255]; // buffer for supporting command
signal( SIGINT, SIG_IGN ); // ignore ctrl-c
while(1)
{
prompt(input);
execute( input );
}
};
void execute( char* str)
{
int fork_result, status, i = 0,j=0,in=0,out=0;
char input[100];output[100];
char** args = parse( str ); // splits the user command into arguments
fork_result = fork(); // attempt to fork
if ( fork_result == -1 ) // failure
{
perror("Failed to fork\n");
exit(1);
}
else if ( fork_result == 0 ) // I'm the child
{
for(i=0;args[i]!='\0';i++)
{
if(strcmp(args[i],"<")==0)
{
args[i]=NULL;
strcpy(input,args[i+1]);
in=2;
}
if(strcmp(args[i],">")==0)
{
args[i]=NULL;
strcpy(output,args[i+1]);
out=2;
}
}
if (in)
{
j = open(input, O_RDONLY, 0);
if ((j = open(input, O_RDONLY, 0)) < 0)
{
perror("Couldn't open input file");
exit(0);
}
dup2(j, 0);
close(j);
}
if (out)
{
if ((i= creat(output , 0644)) < 0)
{
perror("Couldn't open the output file");
exit(0);
}
dup2(i, STDOUT_FILENO);
close(i);
}
execvp( args[0], args );
perror("failed to exec\n");
exit(2);
}
else // I'm the parent
{
// wait here
wait(&status); // wait for child to finish
free( args ); // free dynamic memory
}
}
char** parse( char* str )
{
char** args = malloc( 256 );
int i = 0;
args[i] = strtok( str, " " );
while( args[i] )
{
i++;
args[i] = strtok( NULL, " " );
}
return args;
}
void prompt(char* input)
{
printf("$ "); // print prompt
fgets( input, 255, stdin );
input[strlen(input)-1] = '\0'; // overwrite \n with \0
if ( strcmp( input, "exit" ) == 0 ) // shell command
exit(0);
}
char input[100];output[100];
You want:
char input[100], output[100];
Also add: #include <fcntl.h>
In general, man open (and other functions you use) is your friend -- it tells you what #includes to add.
There are many more potential bugs and arbitrary limitations in your code. Some examples:
void execute( char* str)
{
char input[100], output[100];
...
if(strcmp(args[i],"<")==0)
{
args[i]=NULL;
strcpy(input,args[i+1]); // possible stack buffer overflow.
if(strcmp(args[i],">")==0)
{
args[i]=NULL;
strcpy(output,args[i+1]); // possible stack buffer overflow
char** parse( char* str )
{
char** args = malloc( 256 ); // limit of 256/sizeof(char*) parameters.
// on a 64-bit system, if more than 32 parameters are supplied ...
args[i] = strtok( NULL, " " ); // ... possible heap buffer overflow.
fgets( input, 255, stdin ); // arbitrary limit of 254 characters on command line.
There is no guarantee that the string ends with \n:
input[strlen(input)-1] = '\0'; // overwrite \n with \0
If I was grading this "shell", I'd give it an "F".
There are multiple errors in your code.
1. On line 27 you need to separate the two variable definitions of input and output with a comma instead of a semicolon char input[100], output[100]; or specify the type of output like char input[100]; char output[100]; You have done this already in the line above.
2. The compiler complains about missing definitions of the function open and the identifier O_RDONLY. This can be fixed by adding #include "fcntl.h" to your includes at the top of the file.
After these changes, the code compiles fine for me (with gcc 5.4.0):
#include "stdio.h"
#include "stdlib.h"
#include "string.h"
#include "signal.h"
#include "unistd.h"
#include "fcntl.h"
void prompt(char*);
void execute( char* );
char** parse( char* );
int main( int ac, char* av[] )
{
char input[255]; // buffer for supporting command
signal( SIGINT, SIG_IGN ); // ignore ctrl-c
while(1)
{
prompt(input);
execute( input );
}
};
void execute( char* str)
{
int fork_result, status, i = 0,j=0,in=0,out=0;
char input[100], output[100];
char** args = parse( str ); // splits the user command into arguments
fork_result = fork(); // attempt to fork
if ( fork_result == -1 ) // failure
{
perror("Failed to fork\n");
exit(1);
}
else if ( fork_result == 0 ) // I'm the child
{
for(i=0;args[i]!='\0';i++)
{
if(strcmp(args[i],"<")==0)
{
args[i]=NULL;
strcpy(input,args[i+1]);
in=2;
}
if(strcmp(args[i],">")==0)
{
args[i]=NULL;
strcpy(output,args[i+1]);
out=2;
}
}
if (in)
{
j = open(input, O_RDONLY, 0);
if ((j = open(input, O_RDONLY, 0)) < 0)
{
perror("Couldn't open input file");
exit(0);
}
dup2(j, 0);
close(j);
}
if (out)
{
if ((i= creat(output , 0644)) < 0)
{
perror("Couldn't open the output file");
exit(0);
}
dup2(i, STDOUT_FILENO);
close(i);
}
execvp( args[0], args );
perror("failed to exec\n");
exit(2);
}
else // I'm the parent
{
// wait here
wait(&status); // wait for child to finish
free( args ); // free dynamic memory
}
}
char** parse( char* str )
{
char** args = malloc( 256 );
int i = 0;
args[i] = strtok( str, " " );
while( args[i] )
{
i++;
args[i] = strtok( NULL, " " );
}
return args;
}
void prompt(char* input)
{
printf("$ "); // print prompt
fgets( input, 255, stdin );
input[strlen(input)-1] = '\0'; // overwrite \n with \0
if ( strcmp( input, "exit" ) == 0 ) // shell command
exit(0);
}
I created a program to get all files in a directory, find the individual checksums and then find the total checksums using multithreading.
I am receiving a segmentation fault so I ran gdb and saw that the error is on line 60 where open() is. After researching the seg fault on SO, and on other forums, I attempted to implement a few different approaches such as changing open() to fopen() with a FILE *handle rather than an int. That change proved incorrect.
After hours of debugging and searching, I am clueless and would greatly appreciate any insight.
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <stdarg.h>
#include <fcntl.h>
#include <time.h>
#include <sys/types.h>
#include <dirent.h>
#include <pthread.h> ///Compile with -pthread or -lpthread
#include <sys/stat.h>
#define BUFFER_SIZE (1<<16)
void cleanup();
void get_filenames();
void* get_checksum();
char **filenames;
int file_cnt;
DIR *dir;
//int handle;
FILE *handle;
unsigned int checksum;
unsigned char* ptr;
int length;
int count;
unsigned char* buffer;
int* sum;
unsigned int total = 0;
int main(int argc, char *argv[]){
int i;
pthread_t* file;
atexit(cleanup);
get_filenames();
printf("There are %d files:\n", file_cnt);
file = calloc(sizeof(pthread_t), file_cnt);
sum = calloc(sizeof(int), file_cnt);
for(i=0; i<file_cnt; i++){
printf("%s\n", filenames[i]);
pthread_create(&(file[i]), NULL, get_checksum, (void*)&filenames[i]);
}
for(i=0; i<file_cnt; i++){
total += sum[i];
}
printf("total is: %u\n", total);
return EXIT_SUCCESS;
}
void* get_checksum(void* a){
int b = *((int *)a);
//handle = open(filenames[b], O_RDONLY); //SEG FAULT HERE
handle = fopen(filenames[b], "r"); //SEG FAULT HERE
if( handle == NULL ){
printf( "Can't open file: %s\n", filenames[b]);
exit(1);
}
buffer = malloc(BUFFER_SIZE);
if( buffer == NULL ){
printf( "Can't get enough memory\n" );
exit(1);
}
checksum = 0;
do{
//length = read( handle, buffer, BUFFER_SIZE );
length = read( handle, buffer, (sizeof(char)));
if( length == -1 ){
printf( "Error reading file: %s\n", filenames[b]);
//return NULL;
exit(1);
}
ptr = buffer;
count = length;
while( count-- ){
checksum = checksum + (unsigned int)( *ptr++ );
sum[b] = checksum;
}
} while( length );
printf("Checksum= %d\nTimes at: %d\n", checksum, (int)clock());
}
void cleanup() {
if(filenames && file_cnt > 0) {
while(file_cnt-- > 0) {
if(filenames[file_cnt]) {
free(filenames[file_cnt]);
}
}
free(filenames);
}
if(dir) {
closedir(dir);
}
return;
}
void get_filenames() {
struct dirent *dir_entry;
if((dir = opendir(".")) == NULL) {
fprintf(stderr, "Couldn't open the directory entry for reading\n");
exit(1);
}
errno = 0;
file_cnt = 0;
while((dir_entry = readdir(dir)) != NULL) {
char **new_filenames = filenames;
static int realative_dirs = 0;
if(realative_dirs < 2 &&
(strcmp(".", dir_entry->d_name) == 0 || strcmp("..", dir_entry->d_name) == 0)
) {
realative_dirs++;
continue;
}
new_filenames = (char **)realloc(filenames, sizeof(char **) * (file_cnt + 1));
if(new_filenames == NULL) {
free(filenames[file_cnt]);
fprintf(stderr, "Could not allocate reference for filename[%d]\n", file_cnt);
exit(1);
}
filenames = new_filenames;
filenames[file_cnt] = (char *)calloc(strlen(dir_entry->d_name) + 1, sizeof(char));
if(filenames[file_cnt] == NULL) {
fprintf(stderr, "Could not allocate memory for filename[%d]'s string: \"%s\"\n",
file_cnt, dir_entry->d_name);
exit(1);
}
strcpy(filenames[file_cnt], dir_entry->d_name);
file_cnt++;
}
if(errno != 0) {
fprintf(stderr, "An error occured getting the filenam list\n");
exit(1);
}
return;
}
Below is the output and gdb debugging:
There are 24 files:
.windows
.xscreensaver
.alias
.cshrc
Segmentation fault
(gdb) run
Starting program: /home/nolooking/a.out
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
There are 24 files:
.windows
[New Thread 0x7ffff781e700 (LWP 15957)]
.xscreensaver
[New Thread 0x7ffff701d700 (LWP 15958)]
Program received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7ffff781e700 (LWP 15957)]
0x0000000000400d53 in get_checksum (a=0x60b610) at checksum.c:60
60 handle = open(filenames[b], O_RDONLY);
(gdb) backtrace
#0 0x0000000000400d53 in get_checksum (a=0x60b610) at checksum.c:60
#1 0x00007ffff7bc6374 in start_thread () from /lib64/libpthread.so.0
#2 0x00007ffff7907c3d in clone () from /lib64/libc.so.6
(gdb) quit
A debugging session is active.
UPDATE:
I took the advice of one user in the comments who suggested that I use:
handle=fopen((char*)a, "r");. I can successfully print out the file names when the if statement if(handle==NULL) is commented out. When I include that if statement I receive the following output:
There are 24 files:
.windows
.xscreensaver
.alias
.cshrc
Can't open file: p▒`
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <stdarg.h>
#include <fcntl.h>
#include <time.h>
#include <sys/types.h>
#include <dirent.h>
#include <pthread.h>
#include <sys/stat.h>
#define BUFFER_SIZE (1<<16)
void cleanup();
void get_filenames();
void* get_checksum();
char **filenames;
int file_cnt;
DIR *dir;
//int handle;
FILE *handle;
unsigned int checksum;
unsigned char* ptr;
int length;
int count;
unsigned char* buffer;
int* sum;
unsigned int total = 0;
int main(int argc, char *argv[]){
int i;
pthread_t* file;
atexit(cleanup);
get_filenames();
printf("There are %d files:\n", file_cnt);
file = calloc(sizeof(pthread_t), file_cnt);
sum = calloc(sizeof(int), file_cnt);
for(i=0; i<file_cnt; i++){
printf("%s\n", filenames[i]);
pthread_create(&(file[i]), NULL, get_checksum, (void*)&filenames[i]);
}
for(i=0; i<file_cnt; i++){
total += sum[i];
}
printf("total is: %u\n", total);
return EXIT_SUCCESS;
}
void* get_checksum(void* a){
int b = *((int *)a);
handle = fopen(((char*)a), "r");
if( handle == NULL ){
printf( "Can't open file: %s\n", ((char*)a));
exit(1);
}
buffer = malloc(BUFFER_SIZE);
if( buffer == NULL ){
printf( "Can't get enough memory\n" );
exit(1);
}
checksum = 0;
do{
length = read( handle, buffer, BUFFER_SIZE );
if( length == -1 ){
printf( "Error reading file: %s\n", ((char*)a));
//return NULL;
exit(1);
}
ptr = buffer;
count = length;
while( count-- ){
checksum = checksum + (unsigned int)( *ptr++ );
//sum[a] = checksum;
}
} while( length );
printf("Checksum= %d\nTimes at: %d\n", checksum, (int)clock());
}
void cleanup() {
if(filenames && file_cnt > 0) {
while(file_cnt-- > 0) {
if(filenames[file_cnt]) {
free(filenames[file_cnt]);
}
}
free(filenames);
}
if(dir) {
closedir(dir);
}
return;
}
void get_filenames() {
struct dirent *dir_entry;
if((dir = opendir(".")) == NULL) {
fprintf(stderr, "Couldn't open the directory entry for reading\n");
exit(1);
}
errno = 0;
file_cnt = 0;
while((dir_entry = readdir(dir)) != NULL) {
char **new_filenames = filenames;
static int realative_dirs = 0;
if(realative_dirs < 2 &&
(strcmp(".", dir_entry->d_name) == 0 || strcmp("..", dir_entry->d_name) == 0)
) {
realative_dirs++;
continue;
}
new_filenames = (char **)realloc(filenames, sizeof(char **) * (file_cnt + 1));
if(new_filenames == NULL) {
free(filenames[file_cnt]);
fprintf(stderr, "Could not allocate reference for filename[%d]\n", file_cnt);
exit(1);
}
filenames = new_filenames;
filenames[file_cnt] = (char *)calloc(strlen(dir_entry->d_name) + 1, sizeof(char));
if(filenames[file_cnt] == NULL) {
fprintf(stderr, "Could not allocate memory for filename[%d]'s string: \"%s\"\n",
file_cnt, dir_entry->d_name);
exit(1);
}
strcpy(filenames[file_cnt], dir_entry->d_name);
file_cnt++;
}
if(errno != 0) {
fprintf(stderr, "An error occured getting the filenam list\n");
exit(1);
}
return;
}
Why I am receiving that output once I uncomment the if statement?
Change this
pthread_create(&(file[i]), NULL, get_checksum, (void*)&filenames[i]);
to be
pthread_create(&(file[i]), NULL, get_checksum, (void*)i);
and this
int b = *((int *)a);
to be
int b = (int)a;
Also you cannot call read() on a FILE* as it is returned by fopen(). Use fread() instead.
Don't use &i. I'll explain in a bit. The argument you're passing to the thread is wrong a is not an integer. It's meant to be a pointer to a string...
Change the thread create to this...
pthread_create(&(file[i]), NULL, get_checksum, filenames[i]);
then print the string as follows...
void* get_checksum(void *a){
char *file_name = (char *)a;
printf("filename=%s\n", file_name);
You're passing the string as a pointer to the called function. In your code you're trying to use this as an index into the array.
If you want to pass the index as an integer beware... this won't work..
pthread_create(&(file[i]), NULL, get_checksum, &i);
This is multithreaded and the value pointed to by &i is changing as the loop runs. Pass the pointer to the string and do not under any circumstances change filenames as the threads run.
I think your problem is simply because you are passing &filenames[i] instead of simply &i.
Then in void* get_checksum(void* a) you are trying to use a char* as an int.
The code would be more like :
for(i=0; i<file_cnt; i++){
printf("%s\n", filenames[i]);
pthread_create(&(file[i]), NULL, get_checksum, (void*)&i);
}
and in void* get_checksum(void* a) :
int b = *((int *)a);
handle = fopen(filenames[b], "r");
if( handle == NULL ){
printf( "Can't open file: %s\n", filenames[b]);
exit(1);
}
Hello i want to dynamically initialize an array based on a text file, but for some reason im doing it wrong. i get an error at line "malloc" that the "texto" is not being initialized.
char nome[] = "partidas.txt";
f = fopen(nome, "rt");
int size = fsize(f);
char **texto;
**texto = (char)malloc(size);
int i = 0;
while ((fgets(texto[i], sizeof(texto), f) != NULL))
{
printf("%s\n", texto[i++]);
}
//remember to include the right header files
#include <stdio.h>
#include <string.h>
#include <errno.h>
#define READ_LENGTH 1024
char* pFileContents = NULL;
int iContentsIndex = 0;
long int sz = 0;
FILE* pFD = NULL;
int readCount = 0;
int stat = 0;
// note: all errors are printed on stderr, success is printed on stdout
// to find the size of the file:
// You need to seek to the end of the file and then ask for the position:
pFD = fopen( "filename", "rt" );
if( NULL == pFD )
{
perror( "\nfopen file for read: %s", strerror(errno) );
exit(1);
}
stat = fseek(pFD, 0L, SEEK_END);
if( 0 != stat )
{
perror( "\nfseek to end of file: %s", strerror(errno) );
exit(2);
}
sz = ftell(pFD);
// You can then seek back to the beginning
// in preparation for reading the file contents:
stat = fseek(pFD, 0L, SEEK_SET);
if( 0 != stat )
{
perror( "\nfseek to start of file: %s", strerror(errno) );
exit(2);
}
// Now that we have the size of the file we can allocate the needed memory
// this is a potential problem as there is only so much heap memory
// and a file can be most any size:
pFileContents = malloc( sz );
if( NULL == pFileContents )
{
// handle this error and exit
perror( "\nmalloc failed: %s", strerror(errno) );
exit(3);
}
// then you can perform the read loop
// note, the following reads directly into the malloc'd area
while( READ_LENGTH ==
( readCount = fread( pFileContents[iContentsIndex], READ_LENGTH, 1, pFD) )
)
{
iContentsIndex += readCount;
readCount = 0;
}
if( (iContentsIndex+readCount) != sz )
{
perror( "\nfread: end of file or read error", strerror(errno) );
free( pFileContents );
exit(4);
}
printf( "\nfile read successful\n" );
free( pFileContents );
return(0);
I am writing some C code to process some data in a file, but I just learned that the file is going to be constantly added to (about 1 time/second, maybe faster). So I'm wondering how do I keep reading from the file as its being added to. Then when I get to the end, wait until the next line is added and then process it. Then wait again and then process, and so on and so on. I have something like:
while(1){
fgets(line, sizeof(line), file);
while(line == NULL){
//wait ? then try to read again?
}
//tokenize line and do my stuff here
}
I thought I could maybe use inotify, but I am getting nowhere with that. Does anyone have any advice?
The most efficient way is using inotify, and the direct way is using the read() system call directly.
using inotify
The following code may give you some help, It works well on Debian 7.0, GCC 4.7:
/*This is the sample program to notify us for the file creation and file deletion takes place in “/tmp/test_inotify” file*/
// Modified from: http://www.thegeekstuff.com/2010/04/inotify-c-program-example/
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/inotify.h>
#define EVENT_SIZE ( sizeof (struct inotify_event) )
#define EVENT_BUF_LEN ( 1024 * ( EVENT_SIZE + 16 ) )
int main( )
{
int length, i = 0;
int fd;
int wd;
char buffer[EVENT_BUF_LEN];
/*creating the INOTIFY instance*/
fd = inotify_init();
/*checking for error*/
if ( fd < 0 ) {
perror( "inotify_init error" );
}
/* adding the “/tmp/test_inotify” test into watch list. Here,
* the suggestion is to validate the existence of the
* directory before adding into monitoring list.
*/
wd = inotify_add_watch( fd, "/tmp/test_inotify", IN_CREATE | IN_DELETE | IN_ACCESS | IN_MODIFY | IN_OPEN );
/* read to determine the event change happens on “/tmp/test_inotify” file.
* Actually this read blocks until the change event occurs
*/
length = read( fd, buffer, EVENT_BUF_LEN );
/* checking for error */
if ( length < 0 ) {
perror( "read" );
}
/* actually read return the list of change events happens.
* Here, read the change event one by one and process it accordingly.
*/
while ( i < length ) {
struct inotify_event *event = ( struct inotify_event * ) &buffer[ i ];
if( event->len == 0) {
// For a single file watching, the event->name is empty, and event->len = 0
printf(" Single file watching event happened\n");
} else if ( event->len ) {
if ( event->mask & IN_CREATE ) {
if ( event->mask & IN_ISDIR ) {
printf( "New directory %s created.\n", event->name );
} else {
printf( "New file %s created.\n", event->name );
}
} else if ( event->mask & IN_DELETE ) {
if ( event->mask & IN_ISDIR ) {
printf( "Directory %s deleted.\n", event->name );
} else {
printf( "File %s deleted.\n", event->name );
}
} else if( event->mask & IN_ACCESS ) {
if ( event->mask & IN_ISDIR ) {
printf( "Directory %s accessed.\n", event->name );
} else {
printf(" File %s accessed. \n", event->name );
}
} else if( event->mask & IN_MODIFY ) {
if ( event->mask & IN_ISDIR ) {
printf( "Directory %s modified.\n", event->name );
} else {
printf(" File %s modified. \n", event->name );
}
} else if( event->mask & IN_OPEN ) {
if ( event->mask & IN_ISDIR ) {
printf( "Directory %s opened.\n", event->name );
} else {
printf(" File %s opened. \n", event->name );
}
} else {
printf( "Directory or File is accessed by other mode\n");
}
}
i += EVENT_SIZE + event->len;
}
/* removing the “/tmp/test_inotify” directory from the watch list. */
inotify_rm_watch( fd, wd );
/* closing the INOTIFY instance */
close( fd );
}
When runing the above program. You could test it by create a file or directoy named /tmp/test_inotify.
A detailed explanation could be found here
Use read system call
If a file is open, and have read to the end of current file size. the read() system call will return 0. And if some writer wrote N bytes to this file later, and then the read() will just return min(N, buffersize).
So it works correctly for your circumstance. Following is an examples of the code.
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
typedef int FD ;
int main() {
FD filed = open("/tmp/test_inotify", O_RDWR );
char buf[128];
if( !filed ) {
printf("Openfile error\n");
exit(-1);
}
int nbytes;
while(1) {
nbytes = read(filed, buf, 16);
printf("read %d bytes from file.\n", nbytes);
if(nbytes > 0) {
split_buffer_by_newline(buf); // split buffer by new line.
}
sleep(1);
}
return 0;
}
Reference
Thanks to Jonathan Leffler's Comment
http://www.thegeekstuff.com/2010/04/inotify-c-program-example/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
int
main()
{
char ch;
FILE *fp;
long int nbytes_read = 0;
char str [128];
int j = 0;
int first_time = 1;
memset(str, '\0', 128);
fp = fopen("file.txt", "r");
while (1) {
if (first_time != 1) {
fp = fopen("file.txt", "r");
fseek(fp, nbytes_read, SEEK_SET);
sleep(10);
}
if (fp != NULL) {
while ((ch = fgetc(fp)) != EOF) {
if (ch == '\n') {
str[j++] = ch;
printf("%s", str);
memset(str, '\0', 128);
j = 0;
} else {
str[j++] = ch;
}
nbytes_read++;
}
//printf("%ld\n", nbytes_read);
first_time = 0;
}
fclose(fp);
}
return 0;
}
You can use select() with the fileno(file) as the file-descriptor. select will return either with a timeout (if you set a timeout) or when you can read from the file.
Using select can be a good choice but if you do not wish to use it, you can add a sleep for a small amount of milliseconds before reading value.