parallel download - c

Question

Hi I need a little help with parallel download program.
Currently, it is downloading the same file in parallel instead of downloading multiple files at the same time.
Something is wrong with the fork and fgets, not sure how to fix them. Thank you.
#include <sys/wait.h>
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
FILE *file; /*declare the file pointer*/
#define LINE_MAX 1000
char line [LINE_MAX];
//Parent process
int main()
{
pid_t pid;
file= fopen ("urls.txt", "rt"); /*open file and read it*/
if(!file)
{
perror("fopen");
exit(-1);
}
int numberOfChildren = 0;
while (!feof (file)) {
memset (line,'\0',1000);
char *urlPtr;
while (!feof (file))
{
urlPtr= fgets (line,LINE_MAX, file);
if(urlPtr)
{
int lineLen = strlen(urlPtr);
urlPtr[lineLen-1] = '\0';
pid = fork();
++numberOfChildren;
if (pid == 0) { /* child process */
execlp("/usr/bin/wget", "wget", urlPtr, NULL);
}
else if (pid < 0) { /* error occurred */
fprintf(stderr, "Fork Failed");
exit(-1);
}
}
}
while (numberOfChildren>0) { /* parent process */
/* parent will wait for the child to complete */
wait (NULL);
--numberOfChildren;
printf ("Child Complete");
}
}
fclose (file); /*close file command*/
return 0;
}

Answer 1

You have the fork() check outside the URL reading loop. You first read lots of URLs and spawn a lot of children, and then do the pid check. Try
while (!feof (file))
{
urlPtr= fgets (line,LINE_MAX, file);
pid = fork();
if (pid == 0) { /* child process */
execlp("/usr/bin/wget", "wget", urlPtr, NULL);
}
else if (pid < 0) { /* error occurred */
fprintf(stderr, "Fork Failed");
exit(-1);
}
++numberOfChildren;
}

Answer 2

You should put a diagnostic print and exit after the execlp() (but in the child code after the if). You should probably also close the input file before you execute wget; the program doesn't need it open. No huge harm done this time, but it's good to be tidy. Your parent probably shouldn't exit just because one child failed to fork(); you have other children, in general, that you should wait for. You might stop processing the file at that point, though. And you should definitely forget about feof(); use while (fgets(line, sizeof(line), file) != 0), though that means you don't need urlPtr. The memset() is superfluous; fgets() initializes the string correctly.
Adaptation of code in question
#include <sys/wait.h>
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
FILE *file; /*declare the file pointer*/
#define LINE_MAX 1000
char line [LINE_MAX];
//Parent process
int main(void)
{
pid_t pid;
file = fopen("urls.txt", "rt"); /*open file and read it*/
if (!file)
{
perror("fopen");
exit(-1);
}
int numberOfChildren = 0;
memset(line,'\0',1000);
char *urlPtr;
while (!feof(file))
{
urlPtr= fgets(line, sizeof(line), file);
if (urlPtr)
{
int lineLen = strlen(urlPtr);
urlPtr[lineLen-1] = '\0';
pid = fork();
++numberOfChildren;
if (pid == 0)
{ /* child process */
execlp("/usr/bin/wget", "wget", urlPtr, NULL);
fprintf(stderr, "%d: wget failed\n", (int)getpid());
exit(1);
}
else if (pid < 0)
{ /* error occurred */
fprintf(stderr, "Fork Failed\n");
exit(-1);
}
else
printf("%d: %s\n", (int)pid, urlPtr);
}
}
/* JL: Moved block of code */
while (numberOfChildren>0)
{ /* parent process */
/* parent will wait for the child to complete */
int status;
int corpse = wait(&status);
--numberOfChildren;
printf("Child %d Complete (0x%04X)\n", corpse, status);
}
fclose(file); /*close file command*/
return 0;
}
Note that a while (!feof(file)) loop has been removed, but there is more unnecessary code that could go. Given data file
ftp://ftp.iana.org/tz/releases/tzcode2012f.tar.gz
ftp://ftp.iana.org/tz/releases/tzdata2012f.tar.gz
The code above works fetching the two files in parallel.
Alternative code
I like to use functions, even for relatively short stretches of code that are used once. Hence the be_childish() function added below. The error reporting is a bit tedious to write out, but that is no excuse for not doing it.
I briefly introduced a minimal function that does error reporting, based on an elaborate library of my own, but it would only be used twice in this code (for the file open error and after execlp() returns, which always and unconditionally indicates failure), but decided to leave it out. I have functions such as err_setarg0(), err_error(), err_remark() and err_usage() and using those would reduce each error report to a single line (and some more complex functions that could be told to include the PID automatically, etc). To me, it is worth having such a library as it makes error checking much, much simpler and therefore less painful and less likely to be skimped on.
#include <sys/wait.h>
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
static void be_childish(const char *urlPtr)
{
const char *wget = "/usr/bin/wget";
char *nl = strchr(urlPtr, '\n');
if (nl != 0)
*nl = '\0';
printf("%d: %s\n", (int)getpid(), urlPtr);
execlp(wget, "wget", urlPtr, NULL);
fprintf(stderr, "%d: Failed to execute %s\n", (int)getpid(), wget);
exit(EXIT_FAILURE);
}
int main(int argc, char **argv)
{
FILE *file;
char line [1024];
pid_t pid;
const char *name = "urls.txt";
int rc = EXIT_SUCCESS;
if (argc == 2)
name = argv[1];
else if (argc > 2)
{
fprintf(stderr, "Usage: %s [filename]\n", argv[0]);
exit(EXIT_FAILURE);
}
file = fopen(name, "rt"); /* Undefined behaviour per POSIX */
int numberOfChildren = 0;
if (file == 0)
{
fprintf(stderr, "Failed to open file %s\n", name);
exit(EXIT_FAILURE);
}
while (fgets(line, sizeof(line), file) != 0)
{
if ((pid = fork()) == 0)
{
fclose(file);
be_childish(line);
}
else if (pid < 0)
{
fprintf(stderr, "Fork Failed");
rc = EXIT_FAILURE;
break;
}
++numberOfChildren;
}
fclose(file);
/* Parent waits for the children to complete */
while (numberOfChildren > 0)
{
int status;
const char *result = "OK";
pid = wait(&status);
--numberOfChildren;
if (status != 0)
{
result = "Failed";
rc = EXIT_FAILURE;
}
printf("Child %d %s\n", pid, result);
}
return rc;
}
Note that the code takes a file name on the command line, defaulting to your "urls.txt". The "rt" open mode is not a POSIX or standard C mode; it will likely work, but "r" is sufficient to open a text file on all systems ("rb" to open a binary file works on all systems too, and is POSIX and standard C compliant). It reports which child process is processing each file listed. It reports the status (success or failure) of each child; it's own exit status is only success if all the children were successful.
You could probably control the verboseness from the command line. You might also want to keep a record of which child was processing each file so that you could report on files successfully downloaded, rather than on the processes which the user doesn't care about, really. That complicates the processing since you need to make a copy of each URL as you read it.
Note that you do need to trim the newlines off the end of the string (URL) before passing it to wget.
This code now tested (after adding the newline amendment), and it produced two files. The screen display is a bit of a mess; that's because each copy of wget thinks it is the sole user:
80334: ftp://ftp.iana.org/tz/releases/tzcode2012f.tar.gz
80335: ftp://ftp.iana.org/tz/releases/tzdata2012f.tar.gz
--2012-09-23 19:19:44-- ftp://ftp.iana.org/tz/releases/tzcode2012f.tar.gz
=> “tzcode2012f.tar.gz”
Resolving ftp.iana.org... --2012-09-23 19:19:44-- ftp://ftp.iana.org/tz/releases/tzdata2012f.tar.gz
=> “tzdata2012f.tar.gz”
Resolving ftp.iana.org... 192.0.32.8192.0.32.8, , 2620:0:2d0:200::82620:0:2d0:200::8
Connecting to ftp.iana.org|192.0.32.8|:21... Connecting to ftp.iana.org|192.0.32.8|:21... connected.
Logging in as anonymous ... connected.
Logging in as anonymous ... Logged in!
==> SYST ... Logged in!
==> SYST ... done. ==> PWD ... done. ==> PWD ... done.
==> TYPE I ... done.
==> TYPE I ... done. ==> CWD (1) /tz/releases ... done. ==> CWD (1) /tz/releases ... done.
==> SIZE tzdata2012f.tar.gz ... done.
==> SIZE tzcode2012f.tar.gz ... 206404
==> PASV ... 135543
==> PASV ... done. ==> RETR tzdata2012f.tar.gz ... done. ==> RETR tzcode2012f.tar.gz ... done.
Length: 206404 (202K) (unauthoritative)
0% [ ] 0 --.-K/s done.
Length: 135543 (132K) (unauthoritative)
100%[==============================================================================>] 135,543 72.7K/s in 1.8s
100%[==============================================================================>] 206,404 81.4K/s in 2.5s
2012-09-23 19:19:48 (72.7 KB/s) - “tzcode2012f.tar.gz” saved [135543]
Child 80334 OK
2012-09-23 19:19:48 (81.4 KB/s) - “tzdata2012f.tar.gz” saved [206404]
Child 80335 OK