I wrote my parallel version of unix grep program. In this program you search for patter in files given as program arguments.
In root process I calculate number of files per process and first file ID for each process. After calculations I send this values to each of root processes.
In not root process I'm receiving num of Files and first file ID and analyzing each file in analyzeFile method. In that method I send back string, which contains all lines with pattern, to root process. After collecting all data from not root precesses I print out final result.
The problem is when I try to run program with more than 3 processes. It ends up with segmentation fault.
I'm out of ideas. Valgrind does not show any error reports on base scanning.
PS. I know that code needs refactoring, but first I want to get rid of this bug.
Thanks for any help.
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/mman.h>
#include <string.h>
#include <stdbool.h>
#include <stdlib.h>
#include <mpi.h>
#define YEL "\x1B[33m"
#define BLU "\x1B[34m"
#define MAG "\x1B[35m"
#define RESET "\x1B[0m"
#define send_data_tag 1000
#define return_data_tag 1001
struct linesRange
{
int lineFrom;
int lineTo;
int posFrom;
int posTo;
};
int countNumOfLines(char *str)
{
int numOfLines = 0;
int i = 0;
while(str[i] != '\0')
{
if(str[i] == '\n')
{
++numOfLines;
}
++i;
}
return numOfLines;
}
long int findLinePos(char *str, int lineNumToFind)
{
long int linePos = 0;
long int lineNum = 0;
long int i = 0;
if(lineNumToFind == 0)
{
return linePos;
}
while(str[i] != '\0')
{
if(str[i] == '\n')
{
++lineNum;
if(lineNum == lineNumToFind)
{
linePos = i;
break;
}
}
++i;
}
return linePos;
}
long int findNextLinePos(char *str, int startPos)
{
long int i = startPos;
long int nextLinePos = 0;
while(str[i] != '\0')
{
if(str[i] == '\n')
{
nextLinePos = i;
break;
}
++i;
}
return nextLinePos;
}
bool strstrn(char *str, char *pattern, long int posStart, long int posEnd)
{
long int pos_search = 0;
long int pos_text = 0;
int len_search = strlen(pattern);
for (pos_text = posStart; pos_text < posEnd; ++pos_text)
{
if(str[pos_text] == pattern[pos_search])
{
++pos_search;
if(pos_search == len_search)
{
return true;
}
}
else
{
pos_text -= pos_search;
pos_search = 0;
}
}
return false;
}
char * searchPatternInText(char *text, char *inputPattern, char *fileName, int proc_id)
{
char *bufToAdd;
int lineNum = 0;
char * resultBuf = (char*)malloc(sizeof(char));
memset(resultBuf, 0, sizeof(char));
char * curLine = text;
while(curLine)
{
++lineNum;
char * nextLine = strchr(curLine, '\n');
if (nextLine)
*nextLine = '\0';
if(strstr(curLine, inputPattern))
{
bufToAdd = (char*)malloc(sizeof(char));
memset(bufToAdd, 0, sizeof(char));
asprintf (&bufToAdd, BLU "Ścieżka:%s\tProces:%d\n" MAG "Numer linii:%d\t" RESET "%s\n", fileName, proc_id, lineNum, curLine);
resultBuf = (char*)realloc(resultBuf, strlen(resultBuf) + strlen(bufToAdd) + 1);
strcat(resultBuf, bufToAdd);
free(bufToAdd);
}
if (nextLine)
*nextLine = '\n';
curLine = nextLine ? (nextLine+ 1) : NULL;
}
return resultBuf;
}
void analyzeFile(struct stat sb, int fd, char *fileName, char *pattern, int proc_id)
{
char *text = mmap (0, sb.st_size, PROT_READ, MAP_SHARED, fd, 0);
if (text == MAP_FAILED) {
perror ("mmap");
return;
}
if (close (fd) == -1)
{
perror ("close");
return;
}
char *tempTxt;
tempTxt = (char*)malloc(sizeof(char) * (strlen(text) + 1));
strcpy(tempTxt, text);
tempTxt[strlen(text)] = '\0';
//munmap(text, sb.st_size);
char *result;
result = searchPatternInText(tempTxt, pattern, fileName, proc_id);
free(tempTxt);
if(proc_id != 0)
{
MPI_Send(result, strlen(result), MPI_CHAR,
0, return_data_tag, MPI_COMM_WORLD);
}
else
{
printf(result);
}
free(result);
}
int main (int argc, char *argv[])
{
int patternLen = 10;
char pattern [patternLen + 1];
float elapsed;
int numOfFiles = argc - 2;
int proc_id, num_procs, procIterator, filesIterator, root = 0;
int *numOfFilesPerProcess, *firstFileIdPerProcess;
if (argc < 3) {
fprintf (stderr, "usage: %s <file>\n", argv[0]);
return 1;
}
else
{
strncpy(pattern, argv[1], patternLen);
pattern[patternLen] = '\0';
}
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &proc_id);
MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
MPI_Barrier(MPI_COMM_WORLD);
double startTime = MPI_Wtime();
if(proc_id == root)
{
MPI_Status statusMaster;
numOfFilesPerProcess = (int*)malloc(sizeof(int) * num_procs);
firstFileIdPerProcess = (int*)malloc(sizeof(int) * num_procs);
for(int i = 0; i < num_procs; ++i)
{
numOfFilesPerProcess[i] = 0;
firstFileIdPerProcess[i] = 0;
}
filesIterator = 0;
procIterator = 0;
while(filesIterator < numOfFiles)
{
if(procIterator == num_procs)
{
procIterator = 0;
}
++numOfFilesPerProcess[procIterator];
++procIterator;
++filesIterator;
}
firstFileIdPerProcess[0] = 2;
//pierwszy numer argumentu argv[] per proces
for(int jj = 1; jj < num_procs; ++jj)
{
firstFileIdPerProcess[jj] = firstFileIdPerProcess[jj - 1] + numOfFilesPerProcess[jj - 1];
}
for(int j = 1; j < num_procs; ++j)
{
MPI_Send(&firstFileIdPerProcess[j], 1, MPI_UNSIGNED,
j, send_data_tag, MPI_COMM_WORLD);
MPI_Send(&numOfFilesPerProcess[j], 1, MPI_UNSIGNED,
j, send_data_tag, MPI_COMM_WORLD);
}
int firstFileIdForProcess = firstFileIdPerProcess[0];
int numOfFilesForProcess = numOfFilesPerProcess[0];
int fd;
struct stat sb;
for(int i = 0; i < numOfFilesForProcess; ++i)
{
fd = open (argv[firstFileIdForProcess + i], O_RDONLY);
if ( fd == -1
|| fstat (fd, &sb) == -1
|| !S_ISREG (sb.st_mode))
{
perror("file");
}
else
{
analyzeFile(sb, fd, argv[firstFileIdForProcess + i], pattern, proc_id);
}
}
for(int ii = numOfFilesPerProcess[0]; ii < numOfFiles; ++ii)
{
int resultLen;
char *result;
MPI_Probe(MPI_ANY_SOURCE, return_data_tag, MPI_COMM_WORLD, &statusMaster);
MPI_Get_count(&statusMaster, MPI_CHAR, &resultLen);
result = (char *)malloc(sizeof(char) * resultLen + 1);
MPI_Recv( result, resultLen, MPI_CHAR,
MPI_ANY_SOURCE, return_data_tag, MPI_COMM_WORLD, &statusMaster);
result[resultLen] = '\0';
printf(result);
free(result);
}
free(numOfFilesPerProcess);
free(firstFileIdPerProcess);
}
else
{
MPI_Status statusSlave;
int firstFileIdForProcess = 0;
int numOfFilesForProcess = 0;
int fd;
struct stat sb;
MPI_Recv( &firstFileIdForProcess, 1, MPI_UNSIGNED,
root, send_data_tag, MPI_COMM_WORLD, &statusSlave);
MPI_Recv( &numOfFilesForProcess, 1, MPI_UNSIGNED,
root, send_data_tag, MPI_COMM_WORLD, &statusSlave);
for(int i = 0; i < numOfFilesForProcess; ++i)
{
fd = open (argv[firstFileIdForProcess + i], O_RDONLY);
if ( fd == -1
|| fstat (fd, &sb) == -1
|| !S_ISREG (sb.st_mode))
{
perror("file");
}
else
{
analyzeFile(sb, fd, argv[firstFileIdForProcess + i], pattern, proc_id);
}
}
}
MPI_Barrier(MPI_COMM_WORLD);
double endTime = MPI_Wtime();
if(proc_id == root)
{
printf("Czas wykonania: %fs\n", endTime - startTime);
}
MPI_Finalize();
return 0;
}
Related
If I set O_NONBLOCK when open file, everything works OK. But read blocks when I set O_NONBLOCK via fcntl. What's wrong?
#include <fcntl.h>
#include <sys/epoll.h>
#include <unistd.h>
#include <stdio.h>
void
make_nonblock(size_t N, int in[N]) {
for (size_t i = 0; i < N; ++i) {
int flags = fcntl(in[i], F_GETFL);
flags |= O_NONBLOCK;
fcntl(in[i], F_SETFL, flags);
}
}
extern size_t
read_data_and_count(size_t N, int in[N]) {
make_nonblock(N, in);
char buffer[4096];
size_t total = 0;
for (size_t i = 0; i < N; ++i) {
ssize_t current = 0;
while ((current = read(in[i], buffer, sizeof(buffer))) > 0) {
total += current;
}
}
return total;
}
int
main(int argc, char** argv) {
int fds[1024];
for (int i = 1; i < argc; ++i) {
fds[i - 1] = open(argv[i], O_RDONLY);
}
size_t result = read_data_and_count(argc - 1, fds);
printf("Result: %ld\n", result);
return 0;
}
I'm currently writing a program in C that reads in from a CSV file, I have a defined buffer size but am having trouble separating each line from the buffer. I can see where the line ends by checking for a '\n' char. I cannot extract that line from the buffer for parsing however. Anybody have some ideas?
#ifndef BUFFSIZE
#define BUFFSIZE 4096
#endif
int main() {
int fd;
int fdBin;
char * buf = malloc(BUFFSIZE);
int count = 0;
bool EOFFlag = false;
fd = open("SongCSV.csv", O_RDONLY);
fdBin = open("BinarySongData.bin", O_CREAT | O_WRONLY, "0600");
if (fd == -1) {
printf("failed to open a file\n");
exit(1);
}
off_t offset = 0;
off_t offsetOld = 0;
int readBytes;
while (!EOFFlag) {
offsetOld = offset;
offset = lseek(fd, offset - offsetOld, SEEK_CUR);
readBytes = read(fd, buf, BUFFSIZE);
printf("\n\n%lld\n\n", (offset));
int i = 0;
int commaCounter = 0;
while (i < readBytes) {
if (buf[i] != '\n') {
}
if (buf[i] == '\n') {
printf("\t\t THIS IS END OF LINE \t%d", i);
commaCounter = 0;
}
if (buf[i] == ',') {
commaCounter++;
if (commaCounter == 4) {
printf("****Album Name****");
}
}
write(fdBin, buf, BUFFSIZE);
printf("%c", buf[i]);
i++;
}
if (readBytes < BUFFSIZE) {
EOFFlag = true;
printf("\nREACHED END OF FILE");
}
printf("\n");
printf("AA: END OF LINE ****%d*****", count);
count++;
}
close(fd);
close(fdBin);
return 0;
}
I do it this way, easy and simple. I just did it quickly, any doubts just ask me, Cheers.
#include <sys/types.h>
#include <sys/stat.h>
#include <stdio.h>
#include <fcntl.h>
#include <stdlib.h>
#include <unistd.h>
int main()
{
int len = sending();
char *firstline;
int i = 0;
char buf[0];
int rd ;
int fd = open("hey.csv", O_RDONLY);
rd = read(fd, buf, 1);
firstline = malloc(sizeof(char) * len);
while (i != len)
{
firstline[i] = buf[0];
i++;
rd = read(fd, buf, 1);
}
firstline[i] = '\0';
printf("%s\n", firstline);
return (0);
}
int sending()
{
int fd = open("hey.csv", O_RDONLY);
char buf[1];
int r = 0;
r = read(fd, buf, 1);
int len = 0;
while (buf[0] != '\n')//getting exact size to malloc
{
len++;
r = read(fd, buf, 1);
}
return len;
}
I am currently working on a producer-consumer implementation using C.
First, I create a buffer on the shared memory of a variable length that is given by the user in the consumer process.
Then, in the producer process, I need to access the shared memory and puts new data to the buffer so the consumer can consume.
Below is the consumer code:
#include "common.h"
#include <unistd.h>
int fd;
int errno;
int MY_LEN = 0;
Shared* shared_mem;
char *job[4];
int setup_shared_memory(){
fd = shm_open(MY_SHM, O_CREAT | O_RDWR, 0666);
if(fd == -1){
printf("shm_open() failed\n");
exit(1);
}
ftruncate(fd, sizeof(Shared) + MY_LEN*sizeof(char *));
}
int attach_shared_memory(){
shared_mem = (Shared*) mmap(NULL, sizeof(Shared) + MY_LEN*sizeof(char *), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
if(shared_mem == MAP_FAILED){
printf("mmap() failed\n");
exit(1);
}
return 0;
}
int init_shared_memory() {
shared_mem->data = 0;
int i;
for(i = 0; i < shared_mem->length; i++)
{
shared_mem->arr[i] = 0;
// shared_mem->arr[i] = (char *)calloc(1, sizeof(char*));
}
sem_init(&(shared_mem->mutex), 1, 1);
}
int init_job(){
int i;
for(i = 0; i < 4; i++)
{
job[i] = (char *)malloc(sizeof(char *));
}
}
int take_a_job(int index){
init_job();
char *ds = strdup(shared_mem->arr[index]);
job[0] = strtok(ds, "-");
int i = 1;
while(i < 4)
{
job[i] = strtok(NULL, "-");
i++;
}
// remove the job from the buffer
shared_mem->arr[index] = NULL;
}
int consume_job(int index){
printf("\nPrinter starts printing the job %s, %s pages from Buffer[%d]. The duration is %s seconds and the source is %s.\n",job[3], job[2], index, job[1], job[0]);
sleep(atoi(job[1])); // sleep for job[1] seconds.
}
int main(int args, char *argv[]) {
setup_shared_memory();
attach_shared_memory();
init_shared_memory();
MY_LEN = atoi(argv[1]); // the first parameter following ./printer = the length of the buffer
shared_mem->length = MY_LEN;
//shared_mem->arr = (int*) &shared_mem->arr;
int index = 1;
*(shared_mem->arr) = "1-10-5-6";
*(shared_mem->arr + 1) = "2-5-2-7";
*(shared_mem->arr + 2) = "3-20-10-8";
*(shared_mem->arr + 3) = "4-7-4-9";
take_a_job(index);
int i;
for(i = 0; i < shared_mem->length; i++){
printf("\n\n%d set %s\n", i, shared_mem->arr[i]);
}
consume_job(index);
printf("\n\nHello second check\n\n");
while (1) {}
return 0;
}
Here is the producer code:
#include "common.h"
int fd;
Shared* shared_mem;
char *job;
int setup_shared_memory(){
fd = shm_open(MY_SHM, O_RDWR, 0666);
if(fd == -1){
printf("shm_open() failed\n");
exit(1);
}
}
int attach_shared_memory(){
shared_mem = (Shared*) mmap(NULL, sizeof(Shared), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
if(shared_mem == MAP_FAILED){
printf("mmap() failed\n");
exit(1);
}
return 0;
}
int create_a_job(int args, char *argv[]){
int i;
job = (char *)calloc(8, sizeof(char *));
if(args != 5)
return 0; //the parameters are not correctly formatted
else{
for(i = 1; i < args; i++)
{
if(i > 1)
strcat(job, "-");
strcat(job, argv[i]);
}
}
strcat(job, "\0");
printf("\nthe job is %s\n", job);
}
int put_a_job(){
printf("shared_mem->length is %d\n\n", shared_mem->length);
int i;
for(i = 0; i < shared_mem->length; i++)
{
if(*(shared_mem->arr + i) == 0)
{
//shared_mem->arr[i] = (char *)malloc(sizeof(job));
//strcpy(shared_mem->arr[i], job);
*(shared_mem->arr + i) = (char *)job;
printf("\n\nThe index is %d\n", i);
//printf("\n\nthe argument is %s at %d\n", job, i);
return i;
}
}
printf("\n\nThe index is %d\n", i);
}
int main(int args, char *argv[]) {
setup_shared_memory();
attach_shared_memory();
// create a job with the parameters
int result = create_a_job(args, argv);
if(result == 0)
{
printf("Not the right parameters.\n");
printf("Plase enter client ID, job duration, number of pages and job ID.\n");
return 0;
}
int i;
put_a_job();
for (i=0; i < shared_mem->length; i++) {
printf("the argument is %s at %d\n", (char *)(shared_mem->arr + i), i);
}
printf("\n\n");
return 0;
}
The common.h file is
#ifndef _INCLUDE_COMMON_H_
#define _INCLUDE_COMMON_H_
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
// from `man shm_open`
#include <sys/mman.h>
#include <sys/stat.h> /* For mode constants */
#include <fcntl.h> /* For O_* constants */
#include <string.h>
#include <semaphore.h>
#define MY_SHM "/JIT"
typedef struct {
sem_t mutex;
int data;
int length; // the length of the buffer
char *arr[0];
} Shared;
#endif //_INCLUDE_COMMON_H_
I first run ./consumer 10 & to allocate a buffer of length 10 and after, I run ./producer 1 2 3 4 to put the job to the buffer and print the buffer, I got garbage values
Any help would be really appreciated! Thank you!
Instruction
*(shared_mem->arr + i) = (char *)job;
is storing the pointer job into the shared mem, not the pointed value.
Maybe you want to use a strncpy.
You cannot share memory address between processes, because of Linux uses virtual memory. To make the story short an address in a process is not valid for a different process.
Be aware that you have a memory leakage because you never call free() for the allocated job.
Trying to give a Minimal Viable Example of the problem. Basically the method send_chars_to_reducers sends a character to the proper reducer_pipe. The fork_reducers function remains in its while loop until it receives EOF but it never does even though I close all reducer pipes in send_chars_to_reducers. I know it doesnt exit the while loop because it never prints exiting reducers.
C Code
void send_chars_to_reducers(char * line) {
printf("SEND_CHARS_TO_REDUCERS read: %s\n\n", line);
int i;
int ob_size = 1;
int wlen = 0;
for (i = 0; i < strlen(line); i++) {
if (line[i] >= ALPHA_OFFSET && line[i] < ALPHA_OFFSET + LETTERS) {
int pipe_num = line[i] - ALPHA_OFFSET;
printf("SENDING %c TO REDUCER PIPE %d\n", line[i], pipe_num);
wlen = print_if_err(write(reducer_pipes[pipe_num][PIPE_WRITE_END], &line[i], ob_size), "write");
printf("WROTE %s to REDUCER %d\n", line[i], i);
}
}
close_reducer_pipes();
}
void close_reducer_pipes(void) {
int i;
for (i = 0; i < NUM_OF_REDUCERS; i++) {
close(reducer_pipes[i][PIPE_WRITE_END]);
close(reducer_pipes[i][PIPE_READ_END]);
}
}
void fork_mappers(void) {
/* Constants useful to all children */
char ibuf[PIPE_BUFFER_SIZE]; // input pipe buffer
int rlen = 0;
int i;
for (i=0; i<NUM_OF_MAPPERS; i++) {
pid_t mapper_pid = print_if_err(fork(), "fork");
if (mapper_pid == 0) {
int j;
for (j=0; j < NUM_OF_MAPPERS; j++) {
close(mapper_pipes[i][PIPE_WRITE_END]);
if (j != i) {
close(mapper_pipes[j][PIPE_READ_END]);
}
}
rlen = print_if_err(read(mapper_pipes[i][PIPE_READ_END], ibuf, 1000), "read");
send_chars_to_reducers(ibuf);
close_reducer_pipes();
//printf("forked mapper%d read: %s\n\n", i, ibuf);
close(mapper_pipes[i][PIPE_READ_END]);
_exit(0);
}
}
}
void fork_reducers(void) {
printf("HELLLOOOO FROM REDUCER\n");
char ibuf[PIPE_BUFFER_SIZE]; // input pipe buffer
int rlen = 0;
int i;
for (i = 0; i < NUM_OF_REDUCERS; i++) {
pid_t reducer_pid = print_if_err(fork(), "fork");
if (reducer_pid == 0) {
while (1) {
rlen = print_if_err(read(reducer_pipes[i][PIPE_READ_END], ibuf, 1), "read");
if (rlen > 0) {
printf("REDUCER #%d, read %s\n", i, ibuf);
} else {
break;
}
}
printf("exiting reducer\n");
_exit(0);
}
}
}
Entire C Code
#include <sys/wait.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <time.h>
#include <errno.h>
#define BUFFER_SIZE 1024
#define ALPHA_OFFSET 97
#define LETTERS 26
const int NUM_OF_MAPPERS = 4;
const int NUM_OF_REDUCERS = 26;
const int PIPE_READ_END = 0;
const int PIPE_WRITE_END = 1;
const int PIPE_BUFFER_SIZE = 1000;
int mapper_pipes[4][2];
int reducer_pipes[26][2];
int letter_count[26];
void init_letter_count(void) {
int i;
for (i =0; i < 26; i++) {
letter_count[i] = 0;
}
}
void pipe_wrapper(int pipefd[]) {
int ret = pipe(pipefd);
if (ret == -1) {
perror("Error. Failed when trying to create pipes.");
exit(EXIT_FAILURE);
}
}
void create_mapper_pipes(void) {
int i;
for (i = 0; i < NUM_OF_MAPPERS; i++) {
pipe_wrapper(mapper_pipes[i]);
}
}
void create_reducer_pipes(void) {
int i;
for (i=0; i < NUM_OF_REDUCERS; i++) {
pipe_wrapper(reducer_pipes[i]);
}
}
// Prints an error msg and exits if one occurs. Else, returns the system call value.
int print_if_err(int syscall_val, const char* syscall_name) {
if (syscall_val < 0) {
perror(syscall_name);
exit(errno);
} else {
//No syscall error we can return
return syscall_val;
}
}
void send_chars_to_reducers(char * line) {
printf("SEND_CHARS_TO_REDUCERS read: %s\n\n", line);
int i;
int ob_size = 1;
int wlen = 0;
for (i = 0; i < strlen(line); i++) {
if (line[i] >= ALPHA_OFFSET && line[i] < ALPHA_OFFSET + LETTERS) {
int pipe_num = line[i] - ALPHA_OFFSET;
printf("SENDING %c TO REDUCER PIPE %d\n", line[i], pipe_num);
wlen = print_if_err(write(reducer_pipes[pipe_num][PIPE_WRITE_END], &line[i], ob_size), "write");
printf("WROTE %c to REDUCER %d\n", line[i], pipe_num);
}
}
printf("END OF SEND CHAR FOR LOOP");
close_reducer_pipes();
}
void close_reducer_pipes(void) {
int i;
for (i = 0; i < NUM_OF_REDUCERS; i++) {
print_if_err(close(reducer_pipes[i][PIPE_WRITE_END]), "close");
print_if_err(close(reducer_pipes[i][PIPE_READ_END]), "close");
}
}
void fork_mappers(void) {
/* Constants useful to all children */
char ibuf[PIPE_BUFFER_SIZE]; // input pipe buffer
int rlen = 0;
int i;
for (i=0; i<NUM_OF_MAPPERS; i++) {
pid_t mapper_pid = print_if_err(fork(), "fork");
if (mapper_pid == 0) {
int j;
for (j=0; j < NUM_OF_MAPPERS; j++) {
close(mapper_pipes[i][PIPE_WRITE_END]);
if (j != i) {
close(mapper_pipes[j][PIPE_READ_END]);
}
}
rlen = print_if_err(read(mapper_pipes[i][PIPE_READ_END], ibuf, 1000), "read");
send_chars_to_reducers(ibuf);
//printf("forked mapper%d read: %s\n\n", i, ibuf);
close(mapper_pipes[i][PIPE_READ_END]);
_exit(0);
}
}
}
void fork_reducers(void) {
printf("HELLLOOOO FROM REDUCER\n");
char ibuf[PIPE_BUFFER_SIZE]; // input pipe buffer
int rlen = 0;
int i;
for (i = 0; i < NUM_OF_REDUCERS; i++) {
pid_t reducer_pid = print_if_err(fork(), "fork");
if (reducer_pid == 0) {
while (1) {
rlen = print_if_err(read(reducer_pipes[i][PIPE_READ_END], ibuf, 1), "read");
printf("RLEN = %d\n", rlen);
if (rlen > 0) {
int letter_count_i = ibuf[0] - ALPHA_OFFSET;
printf("REDUCER #%d, read %s, letter_count_i = %d\n", i, ibuf, letter_count_i);
letter_count[letter_count_i]++;
} else {
break;
}
}
printf("REDUCER EXITING\n");
_exit(0);
}
}
}
void send_lines_to_mappers(void) {
int wlen = 0;
char obuf[PIPE_BUFFER_SIZE];
int ob_size;
int count = 0;
char buff[BUFFER_SIZE]; // a buffer for each line of the file
FILE *input_file = fopen("input.txt", "r");
// read the input file line by line
while(fgets(buff, BUFFER_SIZE, input_file) > 0) {
//printf("send_lines_to_mappers read: %s\n\n", buff);
ob_size = sizeof buff;
switch(count) {
case 0 :
write(mapper_pipes[0][PIPE_WRITE_END], buff, ob_size);
close(mapper_pipes[0][PIPE_WRITE_END]);
close(mapper_pipes[0][PIPE_READ_END]);
break;
case 1 :
write(mapper_pipes[1][PIPE_WRITE_END], buff, ob_size);
close(mapper_pipes[1][PIPE_WRITE_END]);
close(mapper_pipes[1][PIPE_READ_END]);
break;
case 2 :
write(mapper_pipes[2][PIPE_WRITE_END], buff, ob_size);
close(mapper_pipes[2][PIPE_WRITE_END]);
close(mapper_pipes[2][PIPE_READ_END]);
break;
case 3 :
write(mapper_pipes[3][PIPE_WRITE_END], buff, ob_size);
close(mapper_pipes[3][PIPE_WRITE_END]);
close(mapper_pipes[3][PIPE_READ_END]);
break;
default :
printf("you did something wrong in send_lines_to_mappers loop");
}
count++;
}
fclose(input_file);
}
int main(void) {
init_letter_count();
// Setup the mapper pipes
create_mapper_pipes();
create_reducer_pipes();
fork_reducers();
fork_mappers();
send_lines_to_mappers();
return 0;
}
When you create a pipe, there are two ends ("file descriptors"), the reading end and the writing end.
When you fork(), the child process inherits ALL the open file descriptors, including both ends of ANY open pipe.
So, if you want:
[child1] >===pipe1===> [parent] >===pipe2===> [child2]
for example, then you have a several file descriptors to close.
In child1, you need to close the READ end of pipe1 AND BOTH ends of
pipe2.
In child2, you need to close the WRITE end of pipe2 and BOTH ends of
pipe1.
In the parent, after you are done forking, you need to close the
WRITE end of pipe1 and the READ end of pipe2.
As you have lots of pipes, you will have even more closing to do.
If you are not sure you have everything closed, the program lsof ("list open file-descriptors") can be helpful.
I am trying to make a simple shell program with the C language which have the options of redirecting stdin and stdout and making a pipe but it's giving me a segmentation fault error. Maybe the problem is in the getline but I'm not sure. Here is the code:
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <sys/wait.h>
#include <stdlib.h>
#include <fcntl.h>
#include <sys/stat.h>
#define R 0
#define W 1
#define LINE_LEN 25
struct Job {
char* command;
char** argv;
int stdin;
int stdout;
} typedef Job;
int tokens_number = 0;
int sign_place = 0;
int contain_left = 0;
int contain_right = 0;
int contain_line = 0;
char** parse_cmdline (char * cmdline ){
char** arg = calloc(15, sizeof(char*));
char temp_cmd[LINE_LEN*10];
strcpy(temp_cmd, cmdline);
char * tmp;
tmp = strtok(temp_cmd, " ");
while(tmp != NULL) {
arg[tokens_number] = (char*) malloc(LINE_LEN * sizeof(char*));
strcpy(arg[tokens_number],tmp);
tmp = strtok(NULL, " ");
tokens_number++;
}
//LAST ELEMENT IS NULL
arg[tokens_number+1] = NULL;
return arg;
}
void check_for_special_signs(char** argv){
int i;
for(i=0; i<tokens_number; i++){
if(strcmp(argv[i], "<") == 0){
contain_left = 1;
sign_place = i;
return;
}else if(strcmp(argv[i], ">") == 0){
contain_right = 1;
sign_place = i;
return;
}else if(strcmp(argv[i], "|") == 0){
contain_line = 1;
sign_place = i;
return;
}
}
}
void fork_child(Job* my_job) {
pid_t pid = fork();
if (pid == 0) {
execv(my_job -> command, my_job -> argv);
perror(my_job -> command);
} else if (pid > 0) {
int status;
wait(&status);
} else
perror("fork");
}
char** create_argv(char** argv){
int i;
int j = 0;
char** argvs = calloc(sign_place,sizeof(char*));
if(sign_place!=0){
for(i=0; i < sign_place ; i++){
argvs[i] = (char*) malloc(sizeof(char*));
strcpy(argvs[i],argv[i]);
}
return argvs;
}else{
return argv;
}
}
void close_job(Job* my_job) {
if (my_job -> stdin != STDIN_FILENO)
close(my_job -> stdin);
if (my_job -> stdout != STDOUT_FILENO)
close(my_job -> stdout);
free(my_job);
}
int main() {
size_t s = 512;
char* buffer = malloc(s * sizeof(char));
char** sep_cmd = malloc(s * sizeof(char));
while (getline(&buffer, &s, stdin) != EOF) {
Job* my_job;
int my_pipe[2];
int in = 0;
int out = 1;
sep_cmd = parse_cmdline(buffer);
my_job->command = sep_cmd[0];
my_job->argv = sep_cmd;
my_job->stdin = in;
my_job->stdout = out;
check_for_special_signs(my_job->argv);
pid_t pid = fork();
if (pid == 0) {
if(contain_left == 1){
in = open(my_job->argv[sign_place + 1], O_RDONLY);
if(in < 0){
perror("open()");
}
my_job->argv = create_argv(my_job->argv);
my_job->stdin = in;
}else if(contain_right == 1){
out = open(my_job->argv[sign_place + 1], O_WRONLY | O_CREAT,
S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
if (out < 0)
perror("open()");
my_job->argv = create_argv(my_job->argv);
my_job->stdout = out;
}else if(contain_line == 1){
pipe(my_pipe);
if (my_job -> stdin == my_pipe[R])
close(my_pipe[W]);
else
close(my_pipe[R]);
}
execv(my_job -> command, my_job -> argv);
perror(my_job -> command);
} else if (pid > 0) {
int status;
wait(&status);
} else{
perror("fork");
}
close_job(my_job);
free(buffer);
buffer = (char*) malloc(s * sizeof(char));
}
free(buffer);
return 0;
}
That way I can't see if there are more mistakes in the code. Please if you see more mistakes list them too.
Thank you.
You forgot to allocate memory for my_job in main function