After producing SIZE items, the producer process should write to the fd_pipe_p2c pipe and be read by the consumer process, which then outputs what it is consuming. After that the consumer should write back to the producer via the pipe_fd_c2p pipe to let it know there's more room in the buffer.
The cnt int doesn't seem to reach SIZE, which I expect it to do after the producer function produces SIZE items.
Therefore the consumer function never receives what is produced.
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#define SIZE 10
int shared_arr[SIZE];
int cnt = 0, in = 0, out = 0;
int fd_pipe_c2p[2], fd_pipe_p2c[2];
void consumer();
void producer();
int main() {
pipe(fd_pipe_c2p); // consumer to producer
pipe(fd_pipe_p2c); // producer to consumer
if (fork() == 0) {
// child process
consumer();
} else {
// parent process
producer();
sleep(3);
}
exit(0);
}
void consumer() {
// consumer process
close(fd_pipe_c2p[0]);
while (1) {
while (cnt == 0) {
// give up everything
write(fd_pipe_c2p[1], &cnt, 1);
write(fd_pipe_c2p[1], shared_arr, sizeof(shared_arr));
// read in cnt?
read(fd_pipe_p2c[0], &cnt, 1);
if (cnt > 0) { // if we have something, break out of inner loop
break;
}
}
read(fd_pipe_p2c[0], shared_arr, sizeof(shared_arr)); // read in array
int n = shared_arr[out]; // get value to consume
fprintf(stderr, "I am consuming.......%d\n", n, out);
out = (out + 1) % SIZE; // move out forward?
cnt--; // decrease count
}
}
void producer() {
// producer process
close(fd_pipe_p2c[0]);
while (1) {
// read in cnt, if it's not == SIZE then produce something
read(fd_pipe_c2p[0], &cnt, 1);
while (cnt == SIZE) {
// write to consumer, we've produced something for it:
write(fd_pipe_p2c[1], &cnt, 1);
write(fd_pipe_p2c[1], shared_arr, sizeof(shared_arr));
}
// cnt != SIZE, so let's produce something:
shared_arr[in] = rand() % 100;
fprintf(stderr, "I am producing......%d...%d\n", shared_arr[in], in);
in = (in + 1) % SIZE;
cnt++;
}
}
}
Perhaps an example of the expected output would explain better:
I am producing......26...0.
I am producing......34...1.
I am producing......6...2.
I am producing......1...3.
I am producing......84...4.
I am producing......35...5.
I am producing......89...6.
I am producing......38...7.
I am producing......65...8.
I am producing......78...9.
Check to see something available.
I am producing......80...0.
I am consuming......26 0
I am producing......67...1.
I am consuming......34 1
I am producing......59...2.
I am consuming......6 2
I am producing......52...3.
I am consuming......1 3
I am producing......85...4.
I am consuming......84 4
I am producing......79...5.
I am consuming......35 5
I am producing......94...6.
I am consuming......89 6
I am producing......58...7.
I am consuming......38 7
I am producing......74...8.
I am consuming......65 8
I am producing......67...9.
I am consuming......78 9
Check to see something available.
Instead, this is my current output:
I am producing......83...0
I am producing......86...1
I am producing......77...2
I am producing......15...3
I am producing......93...4
I am producing......35...5
I am producing......86...6
I am producing......92...7
I am producing......49...8
I am producing......21...9
I am producing......62...0
I am producing......27...1
I am producing......90...2
I am producing......59...3
I am producing......63...4
I am producing......26...5
I am producing......40...6
I am producing......26...7
I am producing......72...8
I am producing......36...9
I am producing......11...0
I am producing......68...1
I am producing......67...2
I am producing......29...3
I am producing......82...4
I am producing......30...5
I am producing......62...6
I am producing......23...7
I am producing......67...8
I am producing......35...9
I am producing......29...0
I am producing......2...1
I am producing......22...2
I am producing......58...3
I am producing......69...4
I am producing......67...5
I am producing......93...6
I am producing......56...7
I am producing......11...8
I am producing......42...9
I am producing......29...0
Related
Essentially, my program creates 3 threads. The "server" and 2 "workers." The workers are meant to sum the 3 digit positive integers in a 1000 line file (500 numbers per thread). After each worker has summed its part, the server prints each workers total. The only problem is my semaphores are not seeming to work.
Here is a version of my program:
// simple c program to simulate POSIX thread and semaphore
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <semaphore.h>
// define semaphores
sem_t s1;
FILE *file;
int sum1 = 0, sum2 = 0, num1 = 0, num2 = 0;
// file name
char fileName[10] = "data1.dat";
// server routine
void* server_routine()
{
printf("Server sent signal to worker thread 1\n");
printf("Server sent signal to worker thread 2\n");
sem_wait(&s1);
printf("Server recieved completion signal from worker thread 1\n");
sem_wait(&s1);
printf("Server recieved completion signal from worker thread 2\n\n");
// print the final results
printf("The sum of the first 500 numbers in the file is: %d\n", sum1);
printf("The sum of the last 500 numbers in the file is: %d\n\n", sum2);
pthread_exit(NULL);
}
// thread 1 reoutine
void* t1_routine()
{
printf("Thread 1 recieved signal from server\n");
file = fopen(fileName, "r");
for(int i = 0; i < 500; i++)
{
fscanf(file, "%d", &num1);
sum1 += num1;
}
printf("sum in thread 1: %d\n", sum1);
printf("Thread 1 sends completion signal to server\n");
sem_post(&s1);
pthread_exit(NULL);
}
// thread 2 routine
void* t2_routine()
{
printf("Thread 2 recieved signal from server\n");
file = fopen(fileName, "r");
fseek(file, 500 * 5, SEEK_SET);
for(int i = 0; i < 500; i++)
{
fscanf(file, "%d", &num2);
sum2 += num2;
}
printf("sum in thread 2: %d\n", sum2);
printf("Thread 2 sends completion signal to server\n");
sem_post(&s1);
pthread_exit(NULL);
}
// main function
int main(int argc, char *argv[])
{
// define threads
pthread_t server, t1, t2;
// initialize the semaphore
sem_init(&s1, 0, 0);
if(pthread_create(&server, NULL, &server_routine, NULL) != 0)
{
return 1;
}
if(pthread_create(&t1, NULL, &t1_routine, NULL) != 0)
{
return 2;
}
if(pthread_create(&t2, NULL, &t2_routine, NULL) != 0)
{
return 3;
}
if(pthread_join(server, NULL) != 0)
{
return 4;
}
if(pthread_join(t1, NULL) != 0)
{
return 5;
}
if(pthread_join(t2, NULL) != 0)
{
return 6;
}
// destroy semaphores
sem_close(&s1);
// exit thread
pthread_exit(NULL);
// end
return 0;
}
I've tested with less threads more semaphores as well, with non luck. I've tried different initial semaphore values. The only time I can get the correct output is when I manually wait with sleep(5); but that defeats the purpose of this project.
A few issues ...
Each client thread does its own/private fopen but FILE *file; is global so they overwrite each others values.
We need to make this variable function scoped so each thread has its own private pointer.
There are no fclose calls.
pthread_exit should not be done by the main thread. It is only for threads created with pthread_create.
Otherwise ...
Whichever thread does the fopen last will set the final value.
So, there is a race condition and the effect is the same as if the main thread (prior to pthread_create calls) had done a single fopen, defeating the purpose of each thread doing its own fopen.
Worse yet, a given thread may do the first fopen, then start with fscanf and have its file value changed when the second thread replaces the file value, so weird stuff happens to each thread because they are doing fseek/fscanf on the same FILE * instance.
Having the aforementioned fclose calls would have made the issue more evident.
Here is the refactored code. It is annotated:
// simple c program to simulate POSIX thread and semaphore
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <semaphore.h>
// define semaphores
sem_t s1;
// NOTE/BUG: each thread opens a different stream, so this must be function
// scoped
#if 0
FILE *file;
#endif
int sum1 = 0,
sum2 = 0,
num1 = 0,
num2 = 0;
// file name
char fileName[10] = "data1.dat";
// server routine
void *
server_routine()
{
printf("Server sent signal to worker thread 1\n");
printf("Server sent signal to worker thread 2\n");
sem_wait(&s1);
printf("Server recieved completion signal from worker thread 1\n");
sem_wait(&s1);
printf("Server recieved completion signal from worker thread 2\n\n");
// print the final results
printf("The sum of the first 500 numbers in the file is: %d\n", sum1);
printf("The sum of the last 500 numbers in the file is: %d\n\n", sum2);
pthread_exit(NULL);
}
// thread 1 reoutine
void *
t1_routine()
{
// NOTE/FIX: this must be function scoped (i.e. private to this thread)
#if 1
FILE *file;
#endif
printf("Thread 1 recieved signal from server\n");
file = fopen(fileName, "r");
for (int i = 0; i < 500; i++) {
fscanf(file, "%d", &num1);
sum1 += num1;
}
printf("sum in thread 1: %d\n", sum1);
printf("Thread 1 sends completion signal to server\n");
sem_post(&s1);
#if 1
fclose(file);
#endif
pthread_exit(NULL);
}
// thread 2 routine
void *
t2_routine()
{
// NOTE/FIX: this must be function scoped (i.e. private to this thread)
#if 1
FILE *file;
#endif
printf("Thread 2 recieved signal from server\n");
file = fopen(fileName, "r");
fseek(file, 500 * 5, SEEK_SET);
for (int i = 0; i < 500; i++) {
fscanf(file, "%d", &num2);
sum2 += num2;
}
printf("sum in thread 2: %d\n", sum2);
printf("Thread 2 sends completion signal to server\n");
sem_post(&s1);
#if 1
fclose(file);
#endif
pthread_exit(NULL);
}
// main function
int
main(int argc, char *argv[])
{
// define threads
pthread_t server, t1, t2;
// initialize the semaphore
sem_init(&s1, 0, 0);
if (pthread_create(&server, NULL, &server_routine, NULL) != 0) {
return 1;
}
if (pthread_create(&t1, NULL, &t1_routine, NULL) != 0) {
return 2;
}
if (pthread_create(&t2, NULL, &t2_routine, NULL) != 0) {
return 3;
}
if (pthread_join(server, NULL) != 0) {
return 4;
}
if (pthread_join(t1, NULL) != 0) {
return 5;
}
if (pthread_join(t2, NULL) != 0) {
return 6;
}
// destroy semaphores
sem_close(&s1);
// exit thread
// NOTE/BUG: only a subthread should do this
#if 0
pthread_exit(NULL);
#endif
// end
return 0;
}
In the code above, I've used cpp conditionals to denote old vs. new code:
#if 0
// old code
#else
// new code
#endif
#if 1
// new code
#endif
Note: this can be cleaned up by running the file through unifdef -k
UPDATE:
Thank you for the response Craig. I have implemented your suggestions to my own code but nothing seemed to change. I then decided to copy paste your updated code into a c file to test it out and I got the same result. It is as follows (in a separate comment since the output is too long): –
Max
It's hard to compare results because we're using different datasets. I created a perl script to create some data.
Most important is that the sum reported by the given worker matches what the server sees for that worker task.
Then, if we know what each per thread section of the file should sum to, that is another matter.
The per line termination is critical (e.g.): CRLF vs LF (see below)
The actual order of worker sem_post and termination doesn't really matter. It can vary system to system or, even, invocation to invocation. What matters is that the server thread waits for N threads (i.e.) N sem_wait calls before printing any sums.
I've produced an updated version below.
Server does not "signal" a worker. The worker "signals" the server by doing sem_post and the server "receives" it by doing sem_wait
I've create a task/thread struct to hold the sums, thread IDs, etc.
I've generalized the code to allow N threads.
Added check of \n placement (i.e. line width). That is, under linux/POSIX a four digit number would be followed by LF (newline) and length would be 5. But, under windows, it would be CRLF (carriage return/newline) and length would be 6.
Added check of file size to ensure it is exactly the desired/expected length.
Some additional diagnostics.
Here is the updated code:
// simple c program to simulate POSIX thread and semaphore
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <semaphore.h>
#include <sys/stat.h>
// number of bytes per line
// 5: 4 digits + LF
// 6: 4 digits + CRLF
#ifndef LINEWID
#define LINEWID (4 + 1)
#endif
// number of items / task
#ifndef COUNT
#define COUNT 500
#endif
// define semaphores
sem_t s1;
#if 0
int sum1 = 0,
sum2 = 0,
num1 = 0,
num2 = 0;
#endif
// file name
#if 0
char fileName[10] = "data1.dat";
#else
const char *fileName = "data1.dat";
#endif
// task control
typedef struct {
pthread_t tid; // thread ID
int tno; // thread index/offset
int sum; // sum
} tsk_t;
#define TSKMAX 50
int tskmax; // actual number of tasks
tsk_t tsklist[TSKMAX]; // list of tasks
// loop through all task blocks
#define TSKFORALL \
tsk_t *tsk = &tsklist[0]; tsk < &tsklist[tskmax]; ++tsk
// server routine
void *
server_routine(void *vp)
{
// NOTE/BUG: server does _not_ signal worker
#if 0
printf("Server sent signal to worker thread 1\n");
printf("Server sent signal to worker thread 2\n");
#endif
for (TSKFORALL) {
printf("Server waiting ...\n");
sem_wait(&s1);
printf("Server complete ...\n");
}
// print the final results
for (TSKFORALL)
printf("The sum of task %d is %d\n",tsk->tno,tsk->sum);
return (void *) 0;
}
// thread 1 reoutine
void *
worker_routine(void *vp)
{
FILE *file;
tsk_t *tsk = vp;
printf("Thread %d running ...\n",tsk->tno);
file = fopen(fileName, "r");
fseek(file,tsk->tno * COUNT * LINEWID,SEEK_SET);
tsk->sum = 0;
int num1;
int first = -1;
int last = -1;
for (int i = 0; i < COUNT; i++) {
if (fscanf(file, "%d", &num1) != 1) {
printf("Thread %d fscan error\n",tsk->tno);
break;
}
if (i == 0)
first = num1;
if (i == (COUNT - 1))
last = num1;
tsk->sum += num1;
}
printf("sum in thread %d: %d (first %d, last %d)\n",
tsk->tno, tsk->sum, first, last);
sem_post(&s1);
#if 1
fclose(file);
#endif
return (void *) 0;
}
// main function
int
main(int argc, char **argv)
{
--argc;
++argv;
setlinebuf(stdout);
setlinebuf(stderr);
if (argc != 1)
tskmax = 2;
else
tskmax = atoi(*argv);
if (tskmax > TSKMAX)
tskmax = TSKMAX;
// define threads
pthread_t server;
printf("main: %d tasks\n",tskmax);
printf("main: %d count\n",COUNT);
FILE *file = fopen(fileName,"r");
if (file == NULL) {
printf("main: fopen failure\n");
exit(96);
}
// check alignment
char chr;
fseek(file,LINEWID - 1,0);
fread(&chr,1,1,file);
if (chr != '\n') {
printf("main: newline mismatch -- chr=%2.2X\n",chr);
exit(97);
}
// get the file size
struct stat st;
if (fstat(fileno(file),&st) < 0) {
printf("main: fstat fault\n");
exit(97);
}
// ensure the file has the correct size
off_t size = tskmax * LINEWID * COUNT;
if (st.st_size != size)
printf("main: wrong file size -- st_size=%llu size=%llu\n",
(unsigned long long) st.st_size,
(unsigned long long) size);
fclose(file);
// initialize the semaphore
sem_init(&s1, 0, 0);
// set the offsets
int tno = 0;
for (TSKFORALL, ++tno)
tsk->tno = tno;
if (pthread_create(&server, NULL, &server_routine, NULL) != 0)
return 98;
for (TSKFORALL) {
if (pthread_create(&tsk->tid,NULL,worker_routine,tsk) != 0)
return 1 + tsk->tno;
}
if (pthread_join(server, NULL) != 0) {
return 99;
}
for (TSKFORALL) {
if (pthread_join(tsk->tid, NULL) != 0) {
return 5;
}
}
// destroy semaphores
sem_close(&s1);
// end
return 0;
}
Here is the perl script output that I used to generate the data:
number of tasks 2
element count per task 500
line separater 0A
section 0 sum 124750
section 1 sum 125250
Here is the program output:
main: 2 tasks
Server waiting ...
Thread 0 running ...
Thread 1 running ...
sum in thread 1: 125250 (first 1, last 500)
sum in thread 0: 124750 (first 0, last 499)
Server complete ...
Server waiting ...
Server complete ...
The sum of task 0 is 124750
The sum of task 1 is 125250
Here is the perl script:
#!/usr/bin/perl
# gendata -- generate data
#
# arguments:
# 1 - number of tasks (DEFAULT: 2)
# 2 - number of items / task (DEFAULT: 500)
# 3 - line separater (DEFAULT: \n)
master(#ARGV);
exit(0);
# master -- master control
sub master
{
my(#argv) = #_;
$tskmax = shift(#argv);
$tskmax //= 2;
printf(STDERR "number of tasks %d\n",$tskmax);
$count = shift(#argv);
$count //= 500;
printf(STDERR "element count per task %d\n",$count);
$sep = shift(#argv);
$sep //= "\n";
printf(STDERR "line separater");
foreach $chr (split(//,$sep)) {
$hex = ord($chr);
printf(STDERR " %2.2X",$hex);
}
printf(STDERR "\n");
for ($itsk = 0; $itsk < $tskmax; ++$itsk) {
$val = $itsk;
$sum = 0;
for ($lno = 1; $lno <= $count; ++$lno, ++$val) {
printf("%4d%s",$val,$sep);
$sum += $val;
}
printf(STDERR "section %d sum %d\n",$itsk,$sum);
}
}
void producer(char* line){
/* Produtor */
pthread_mutex_lock(&buffer_mutex);
while (occupied >= MAX_COMMANDS){
printf("produzi wait\n");
pthread_cond_wait(&less, &buffer_mutex);
}
printf("insere no index: %d\n",buff_write_idx);
insertCommand(line, buff_write_idx);
buff_write_idx = (buff_write_idx + 1) % MAX_COMMANDS;
occupied++;
printf("produzi. occupied = %d\n", occupied);
pthread_cond_signal(&more);
pthread_mutex_unlock(&buffer_mutex);
}
/******************************************************/
char* consumer(){
pthread_mutex_lock(&buffer_mutex);
while(occupied <= 0 ){
printf("consumidor wait\n");
pthread_cond_wait(&more, &buffer_mutex);
}
printf("retira do index: %d\n",buff_read_idx);
char* command = removeCommand(buff_read_idx);
buff_read_idx = (buff_read_idx + 1) % MAX_COMMANDS;
occupied--;
printf("consumidor. occupied = %d\n", occupied);
pthread_cond_signal(&less);
pthread_mutex_unlock(&buffer_mutex);
return command;
}
My problem is that after I run my program in the end, after the buffer done the operations necessary, in the CONSUMER(consumer while waiting for signal of producer) is always waiting for a producer but the producer is already empty. Suggestions to exit the loop?
I have a large file (around 1,000,000 characters) in the format "AATACGTAGCTA" and a subsequent file, such as "CGTATC" (10,240 characters). I want to find the largest match of the subsequence within the main sequence. A full, 100% subsequence match may not exist, this is not guaranteed. For the sake of a smaller example, the above would output: Longest match is 4/6 starting at position 5.
I'm working on my C basics, and would like to implement it like so:
The user chooses how many processes they would like to split the work
into.
Each process does 1/nth of the work and updates the shared memory
values located in the struct.
The longest match (it may not be all characters) is reflected in the
struct, as well as it's starting position, and how many
characters were matched. See output below.
Code
#define _GNU_SOURCE
#include <limits.h>
#include <stdio.h>
#include <errno.h>
#include <semaphore.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/wait.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/shm.h>
typedef struct memoryNeeded {
int start_pos, total_correct;
char sequence[1038336];
char subsequence[10240];
sem_t *sem;
} memoryNeeded;
// Used to check all arguments for validity
int checkArguments(char* p, int argc) {
char *prcs;
errno = 0;
int num;
long conv = strtol(p, &prcs, 10);
if (errno!= 0 || *prcs != '\0' || conv > INT_MAX || conv > 50) {
puts("Please input a valid integer for number of processes. (1-50)");
exit(1);
} else {
num = conv;
if (argc != 4) {
puts("\nPlease input the correct amount of command line arguments (4) in"
"the format: \n./DNA (processes) (sequence) (subsequence)\n");
exit(1);
} else
printf("Looking for string using %d processes...\n", num);
return(num);
}
}
int main (int argc, char* argv[]) {
int processes = checkArguments(argv[1], argc);
key_t shmkey;
int procNumber, shmid, pid;
FILE *sequence;
FILE *subsequence;
char *buf1, *buf2;
// Create shared memory
size_t region_size = sizeof(memoryNeeded);
shmkey = ftok("ckozeny", 5);
shmid = shmget(shmkey, region_size, 0644 | IPC_CREAT);
if (shmid < 0) {
perror("shmget\n");
exit(1);
}
// Create structure in shared memory, attach memory and open semaphore
memoryNeeded *mn;
mn = (memoryNeeded *)shmat(shmid, NULL, 0);
mn->sem = sem_open("sem", O_CREAT | O_EXCL, 0644, 1);
sequence = fopen(argv[2], "r");
subsequence = fopen(argv[3], "r");
// Get file sizes
fseek(sequence, 0L, SEEK_END);
int sz1 = ftell(sequence);
rewind(sequence);
fseek(subsequence, 0L, SEEK_END);
int sz2 = ftell(subsequence);
rewind(subsequence);
// Read files into 2 buffers, which are put into struct mn
buf1 = malloc(sz1);
buf2 = malloc(sz2);
if (sz1 != fread(buf1, 1, sz1, sequence)) {
free(buf1);
}
if (sz2 != fread(buf2, 1, sz2, subsequence)) {
free(buf2);
}
// Initialize struct with necessary values
mn->start_pos = 0;
mn->total_correct = 0;
strncpy(mn->sequence, buf1, sz1);
strncpy(mn->subsequence, buf2, sz2);
fclose(sequence);
fclose(subsequence);
// Begin n forks
for (procNumber = 0; procNumber < processes; procNumber++) {
pid = fork();
if (pid < 0) {
sem_unlink("sem");
sem_close(mn->sem);
printf ("Fork error.\n");
} else if (pid == 0)
break;
}
if (pid != 0) {
while ((pid = waitpid (-1, NULL, 0))){
if (errno == ECHILD)
break;
}
printf("Best match is at position %d with %d/10240 correct.", mn->start_pos, mn->total_correct);
printf ("\nParent: All children have exited.\n");
sem_unlink("sem");
sem_close(mn->sem);
shmdt(mn);
shmctl(shmid, IPC_RMID, 0);
exit(0);
} else {
// this child process will do its 1/nth of the work
sem_wait(mn->sem);
printf ("Child(%d) is in critical section.\n", procNumber);
sleep(1);
int i = 0;
int longest, count = 0;
for (i = 0; i < sz1; i += processes) {
for (int j = 0; j < sz2; j += processes) {
count = 0;
while (mn->sequence[i+j] == mn->subsequence[j]) {
count++;
j++;
}
if (count > longest) {
longest = count;
}
}
}
// If local match is longer than that of the struct, update and unlock
if (longest > mn->total_correct) {
mn->total_correct = count;
mn->start_pos = (i - count);
sem_post(mn->sem);
} else
// If not - unlock and let next process go
sem_post(mn->sem);
exit(0);
}
return 1;
}
The current child code is more or less "pseudocode". I've put it together how it makes sense in my head. (I'm aware this may not be correct or function as intended.) My question is in regard to the child code algorithm near the bottom.
How do I implement this so each child does 1/nth of the work, and finds the longest match, even though it may not match 100%?
Final output would be:
./DNA 6 sequence1 subsequence1
Looking for string using 6 processes...
Best match is at position 123456 with 9876/10240 correct.
Thanks.
Generalities and functioning of my program
NB : you will be able to test my program (only one file, containing the main function). I give you the entire code at the end of this question.
I wrote a program which can be used to illustrate the producer-consumer algorithm, with UNIX-Processes. The producer creates some value, for example 5, writes it into a RAM shared_buffer and then writes the latter into a file test.txt. The consumer assigns to this shared_buffer the content of the file test.txt and takes some value from the RAM buffer, shared_buffer.
I use functions to convert my shared_buffer into the file, and reciprocally : arrayToFile and fileToArray. Both are presented at the end of this question.
The shared_buffer has a size of 1 + 10 cases : the first one contains the number of full cases (ie. : with a 5 writen) and the 10 others can be filled either with 5 or nothing.
The first case is useful for the producer, to know where to write next value (ie. : in which case).
The file of course has also 1 + 10 cases. The file is needed because I use processes and not threads (not-shared memory, thus).
shared_buffer's initialisation is contained in the main function. shared_buffer's accesses (in reading and in writing) are contained in consumer's function and in producer's function, respectively. These codes are presented at the end of this question.
Access to shared_buffer and overall to the file are of course under mutual exclusion and three semaphores are used. The mutexe impedes producer and consumer to access it at the same time, and the two other semaphores are used to guarantee that the producer won't try to put a new element if there isn't enough place, and that the consumer won't try to take an element if there isn't any element. Well, it's just the producer-consumer algorithm.
Finally, producer's process runs until the end of time, and so does the consumer's process.
The declaration and initialisation of these three semaphores are presented at the end of this question.
My problem
There is only one problem : when both producer's process and consumer's process are running until the end of times while(TRUE), arrayToFile and fileToArray tell me that the file's opening failed. If I erase one or both while(TRUE), this error disapears (but thus, my program doesn't make its job).
So this problem appears only when both while(TRUE) are writen.
I think it's because I don't make good use of the mutexe. But I couldn't give you more explanations.
Source
Code is highly commented.
#include <unistd.h>
#include <fcntl.h>
#include <stdio.h>
#include <sys/sem.h>
#include <stdlib.h>
#define TRUE 1
#define ERR_NBR_ARGS "ERROR. Argument needed. Use as follows.\n"
int fileToArray(char *, int *, int);
int arrayToFile(char *, int *, int);
void check_if_command_arguments_are_correct(int);
void mutual_exclusion_produce(int, char*, int*, int);
void mutual_exclusion_consume(int, char*, int*, int);
int main(int argc, char* argv[]) {
check_if_command_arguments_are_correct(argc);
// FILE'S PATH
char* file_path = argv[1];
// DECLARATION AND INITIALISATION OF THE SHARED RESOURCE
int shared_buffer_number_of_elements = 10;
int shared_buffer[shared_buffer_number_of_elements + 1];
shared_buffer[0] = 0;
arrayToFile(file_path, shared_buffer, shared_buffer_number_of_elements);
// FILE'S KEY (used to make processes able to use the same semaphores)
key_t key = ftok(argv[0], '0');
if(key == - 1) {
perror("ftok");
exit(EXIT_FAILURE);
}
// DECLARATION AND INITIALISATION OF THE THREE SEMAPHORES
int semid = semget(key, 3, IPC_CREAT|IPC_EXCL|0666); // Declaration of the semaphores
if(semid == -1) {
perror("semget");
exit(EXIT_FAILURE);
}
int array_semaphores_values[3];
array_semaphores_values[0] = 1;
array_semaphores_values[1] = shared_buffer_number_of_elements;
array_semaphores_values[2] = 0;
int sem_controller = semctl(semid, 3, SETALL, array_semaphores_values); // Initialisation of the semaphores - 2th parameter is the array's size
if(sem_controller == -1) {
perror("semctl");
exit(EXIT_FAILURE);
}
// THE TWO FORKS - CREATION OF BOTH PRODUCER AND CONSUMER
pid_t producer = fork();
if(producer == -1) {
perror("fork");
exit(EXIT_FAILURE);
} else if(producer == 0) { // The producer process
mutual_exclusion_produce(semid, file_path, shared_buffer, shared_buffer_number_of_elements);
} else { // The main process
pid_t consumer = fork();
if(consumer == - 1) {
perror("fork");
exit(EXIT_FAILURE);
} else if(consumer == 0) { // The consumer process
mutual_exclusion_consume(semid, file_path, shared_buffer, shared_buffer_number_of_elements);
}
}
semctl(semid, 0, IPC_RMID, 0); // The semaphores are freed
}
void mutual_exclusion_produce(int semid, char* file_path, int* buffer, int size) {
/* The producer does the following :
* 1. It decrements the free cases semaphore ;
* 2. It decrements the mutex ;
* 3. It writes the buffer
* 4. It increments the mutex ;
* 5. It increments the full cases semaphore ;
* */
while(TRUE) {
// DECREMENTS FREE CASES SEMAPHORE AND DECREMENTS MUTEX
struct sembuf operation_decrement_free_cases;
operation_decrement_free_cases.sem_num = 2;
operation_decrement_free_cases.sem_op = -1;
operation_decrement_free_cases.sem_flg = 0;
struct sembuf operation_decrement_mutex;
operation_decrement_mutex.sem_num = 0;
operation_decrement_mutex.sem_op = -1;
operation_decrement_mutex.sem_flg = 0;
semop(semid, &operation_decrement_free_cases, 0);
semop(semid, &operation_decrement_mutex, 0);
// WRITES THE BUFFER INTO THE FILE
buffer[++buffer[0]] = 5;
arrayToFile(file_path, buffer, size);
// INCREMENTS THE MUTEX AND INCREMENTS THE FULL CASES SEMAPHORE
struct sembuf operation_increment_full_cases;
operation_decrement_free_cases.sem_num = 1;
operation_decrement_free_cases.sem_op = +1;
operation_decrement_free_cases.sem_flg = 0;
struct sembuf operation_increment_mutex;
operation_decrement_mutex.sem_num = 0;
operation_decrement_mutex.sem_op = +1;
operation_decrement_mutex.sem_flg = 0;
semop(semid, &operation_increment_mutex, 0);
semop(semid, &operation_increment_full_cases, 0);
}
}
void mutual_exclusion_consume(int semid, char* file_path, int* buffer, int size) {
/*
* The consumer does the following :
* 1. It decrements the full cases semaphore ;
* 2. It decrements the mutex ;
* 3. It reads the buffer ;
* 4. It increments the mutex ;
* 5. It increments the free cases semaphore ;
* */
while(TRUE) {
// DECREMENTS FULL CASES SEMAPHORE AND DECREMENTS MUTEX
struct sembuf operation_decrement_full_cases;
operation_decrement_full_cases.sem_num = 1;
operation_decrement_full_cases.sem_op = -1;
operation_decrement_full_cases.sem_flg = 0;
struct sembuf operation_decrement_mutex;
operation_decrement_mutex.sem_num = 0;
operation_decrement_mutex.sem_op = -1;
operation_decrement_mutex.sem_flg = 0;
semop(semid, &operation_decrement_full_cases, 0);
semop(semid, &operation_decrement_mutex, 0);
// READS THE FILE AND PUT THE CONTENTS INTO THE BUFFER
fileToArray(file_path, buffer, size);
// INCREMENTS THE MUTEX AND INCREMENTS THE FREE CASES SEMAPHORE
struct sembuf operation_increment_free_cases;
operation_decrement_full_cases.sem_num = 2;
operation_decrement_full_cases.sem_op = +1;
operation_decrement_full_cases.sem_flg = 0;
struct sembuf operation_increment_mutex;
operation_decrement_mutex.sem_num = 0;
operation_decrement_mutex.sem_op = +1;
operation_decrement_mutex.sem_flg = 0;
semop(semid, &operation_increment_mutex, 0);
semop(semid, &operation_increment_free_cases, 0);
}
}
void check_if_command_arguments_are_correct(int argc) {
if(argc != 2) {
fprintf(stderr, ERR_NBR_ARGS);
fprintf(stderr, "program_command <file_buffer>\n");
exit(EXIT_FAILURE);
}
}
int fileToArray(char *pathname, int *tab, int size) {
int cible;
if ( (cible = open(pathname,O_RDONLY)) < 0){
fprintf(stderr,"fileToArray - impossible to open the file\n");
return -1;
}
if (read(cible,tab,(size+1) * sizeof(int)) !=(size+1) * sizeof(int)) {
fprintf(stderr,"fileToArray - impossible to read the file\n");
return -1;
}
close(cible);
return 0;
}
int arrayToFile(char *pathname, int *tab, int size) {
int cible;
if ( (cible = open(pathname,O_WRONLY|O_CREAT|O_TRUNC,0666)) < 0){
fprintf(stderr,"arrayToFile - impossible to open the file\n");
return -1;
}
if (write(cible,tab,(size+1) * sizeof(int)) !=(size+1) * sizeof(int)) {
fprintf(stderr,"arrayToFile - impossible to write the file\n");
return -1;
}
close(cible);
return 0;
}
I have the bounded buffer, producer consumer problem to deal with and can only modify the prod and cons functions. This code runs without issues with only one consumer and producer threads. But with multiple of each it always gives me the same problem, sooner or later:
p5p1: p5p2a.c:207: bb_remove: Assertion `bbp->cnt > 0' failed.
What I don't get is how can this error occur when I check the bbp->cnt variable before calling the bbp_remove() function.
EDIT: The problem has been solved. I was not checking the variable within the lock, in either of the functions.
#include <sys/times.h>
#include <unistd.h>
#include <assert.h>
#include <stdlib.h>
#include <pthread.h>
#include <time.h>
#include <errno.h>
#include <stdio.h>
#define DEBUG 0
#define BUF_SIZE 100000
#define ITER 10000000
#define PROD_THRD 3
#define CONS_THRD 3
#define USAGE_STRING "Usage: %s\n"
extern int errno;
/* This is the bounded buffer type */
typedef struct {
int cnt, in, out;
pthread_mutex_t lock; /* Mutex to avoid race conditions */
int buf[BUF_SIZE]; /* The data passed is the id of the
* producer */
} bbuf_t;
typedef struct {
bbuf_t *bbp;
int id;
} parg_t;
/*
* yield()
* Because there is no yield system call in Linux, what we
* do is to put the thread to sleep for 1 ms. Actually, it
* will sleep for at least 1/HZ, which is 10 ms in Linux/386.
*/
#define YIELD_s 0
#define YIELD_ns 1000000
void yield() {
struct timespec st = {YIELD_s, YIELD_ns};
if( (nanosleep(&st, NULL)) == -1 && (errno == EINVAL)) {
perror("nanosleep");
pthread_exit(NULL);
}
}
/* Initialize bounded buffer */
int bbuf_init(bbuf_t *bbp) {
if(bbp == NULL)
return 0;
bbp->in = 0;
bbp->out = 0;
bbp->cnt = 0;
pthread_mutex_init(&(bbp->lock), NULL); /* I do not understand, but the
* compiler complains when I use
* PTHREAD_MUTEX_INITIALIZER */
return 1;
}
/* Print the bounded buffer members that matter */
void print_bbuf(bbuf_t *bbp) {
printf("bbp->in = %d bbp->out = %d bbp_cnt = %d \n",
bbp->in, bbp->out, bbp->cnt);
}
/* To validate the value of the members in, out and cnt of bbuf_t */
int val(int n, int min, int max) {
return( (min <= n) && (n <= max));
}
/* Ensure that the values of the members in, out and cnt are consistent */
int consist(int cnt, int in, int out) {
return ( in == ((out + cnt) % BUF_SIZE) );
}
/* This is the code of the checker thread. It is used to ensure that
* the bounded buffer has not been corrupted.
* Every 100 ms it checks the values of: in, out and cnt.
* This thread exits either if it detects the buffer has been corrupted
* or if it does not detect any activity in 50 consecutive iterations,
* i.e. approximately 5s. */
/* These constants are used with nanosleep() and
* put a process to sleep for 100 ms */
#define SLEEP_s 0
#define SLEEP_ns 100000000
#define MAX_IDLE 50
void *check(void *arg) {
bbuf_t *bbp = arg;
int cnt[2], in[2], out[2]; /* values read */
int idle;
struct timespec st = {SLEEP_s, SLEEP_ns}; /* 100 ms */
while(1) {
pthread_mutex_lock( &(bbp->lock) );
in[1] = bbp->in;
out[1] = bbp->out;
cnt[1] = bbp->cnt;
pthread_mutex_unlock( &(bbp->lock) );
if(in[1] == in[0] && out[1] == out[0] && cnt[1] == cnt[0] ) {
idle++;
if( idle >= MAX_IDLE ) {
printf("Checking thread exiting:");
print_bbuf(bbp);
printf("\t no activity detected for some time.\n");
pthread_exit(NULL);
}
} else {
idle = 0;
}
if( !val(in[1], 0, BUF_SIZE - 1) ) {
printf("Invalid value in = %d \n", in[1]);
pthread_exit(NULL);
} else if ( !val(out[1], 0, BUF_SIZE - 1) ) {
printf("Invalid value out = %d \n", out[1]);
pthread_exit(NULL);
} else if ( !val(cnt[1], 0, BUF_SIZE) ) {
printf("Invalid value cnt = %d \n", cnt[1]);
pthread_exit(NULL);
} else if ( !consist(cnt[1], in[1], out[1]) ) {
printf("Inconsistent buffer: cnt = %d in = %d out = %d \n",
cnt[1], in[1], out[1]);
pthread_exit(NULL);
}
if( (nanosleep(&st, NULL) == -1) && (errno == EINVAL)) {
perror("nanosleep");
pthread_exit(NULL);
}
in[0] = in[1];
out[0] = out[1];
cnt[0] = cnt[1];
}
}
/* The producer threads may use this code to
* enter one item into the buffer */
void bb_enter(bbuf_t *bbp, int me) {
assert( bbp->cnt < BUF_SIZE);
(bbp->buf)[bbp->in] = me;
(bbp->in)++;
(bbp->in) %= BUF_SIZE;
(bbp->cnt)++;
//printf("%d\n",bbp->cnt);
}
/* This is the code for the producer threads.
*
* To avoid busy waiting (or at least too much busy waiting) the producers
* should yield, using the yield() defined above, if the buffer is
* full. In that case, they should print a debugging message as well.
*
* Each producer should produce ITER (10 M) items: an integer with
* the id it receives in prod()'s argument.
*/
void *prod(void *arg) {
parg_t *parg = (parg_t *)arg;
bbuf_t *bbp = parg->bbp;
int me = parg->id;
/* Add variables and code, if necessary */
printf("I am a producer and have started\n");
int gcnt = 0;
while( gcnt <= ITER ){
if(bbp->cnt < BUF_SIZE){
pthread_mutex_lock(&(bbp->lock));
bb_enter(bbp,me);
gcnt++;
pthread_mutex_unlock(&(bbp->lock));}
else if( bbp->cnt == (BUF_SIZE-1)) {printf("I shall produce yield()\n"); yield();}
}
printf("I am a producer and have ended\n");
return;
}
/* The consumer threads may use this function to
* remove an item */
int bb_remove(bbuf_t *bbp) {
int val;
assert(bbp->cnt > 0);
val = (bbp->buf)[bbp->out];
(bbp->out)++;
(bbp->out) %= BUF_SIZE;
(bbp->cnt)--;
return val;
}
/* This is the code for the consumer threads.
* To avoid busy waiting (or at least too much busy waiting) consumers
* should yield, using the yield() defined above, if the buffer is
* empty. In that case, they should print a debugging message as well.
*
* Each consumer should consume ITER (10 M) items, and keep track of the
* producers of the items it consumes: use the cnt[] below.
*/
void *cons(void *arg) {
bbuf_t *bbp = (bbuf_t *)arg;
int cnt[PROD_THRD];
/* Add variables and code, if necessary:
* do not forget to initialize cnt */
printf("I am a consumer and have started\n");
int i;
for(i = 0; i < PROD_THRD; i++){
cnt[i] = 0;}
int temp;
int gcnt = 0;
while( gcnt <= ITER ){
if(bbp->cnt > 0){
pthread_mutex_lock(&(bbp->lock));
temp = bb_remove(bbp);
gcnt++;
cnt[temp]++;
pthread_mutex_unlock(&(bbp->lock));}
else if( bbp->cnt == 0) {printf("I shall consume yield()\n"); yield();}
}
printf("I am a consumer and have ended\n");
return;
}
int main(int argc, char *argv[]) {
int i;
pthread_t *tid, ctid;
parg_t *parg;
bbuf_t *bbp;
/* This is to measure the time it takes to run the program */
struct tms t;
clock_t start, end;
long ticks = sysconf(_SC_CLK_TCK);
start = times(&t);
if( argc != 1 ) {
printf(USAGE_STRING, argv[0]);
exit(1);
}
/* Array for pthread_join() */
if((tid = (pthread_t *) malloc((PROD_THRD + CONS_THRD) * sizeof(pthread_t)))
== NULL ) {
printf("Out of memory.\n");
exit(2);
}
/* Allocate Bounded Buffer */
if((bbp = (bbuf_t *) malloc(sizeof(bbuf_t))) == NULL ) {
printf("Out of memory. \n");
exit(2);
}
/* Initialize Bounded Buffer */
if( bbuf_init(bbp) == 0 ) {
printf("Failed to initialize bounded buffer\n");
exit(3);
}
/* Arguments for producer threads */
if((parg = (parg_t *) malloc( PROD_THRD * sizeof (parg_t))) == NULL ) {
printf("Out of memory.\n");
exit(2);
}
/* Create checker thread */
if( pthread_create(&ctid, NULL, check, bbp) )
perror("pthread_create");
printf("Created checker thread %u\n", (unsigned)ctid);
/* Create consumer threads */
for( i = 0; i < CONS_THRD; i++ ) {
/* We pass the same data structure, the bounded buffer,
* to each consumer: they need to synchronize to access it */
if( pthread_create(tid+i, NULL, cons, bbp) )
perror("pthread_create");
printf("Created consumer thread %u\n", (unsigned)tid[i]);
}
/* Create producer threads */
for( i = 0; i < PROD_THRD; i++ ) {
/* Because we want each consumer to keep track of the
* producer of the items it consumes, we assign an
* id to each producer thread */
parg[i].bbp = bbp;
parg[i].id = i;
if( pthread_create(tid+(i+CONS_THRD), NULL, prod, parg+i) )
perror("pthread_create");
printf("Created producer thread %u (%d)\n", (unsigned)tid[i+CONS_THRD], i);
}
/* Join consumer and producer threads */
for( i = 0; i < CONS_THRD + PROD_THRD; i ++ ) {
if( pthread_join(tid[i], NULL) == 0 ) {
printf("Joined thread %u.\n", (unsigned)tid[i]);
} else {
printf("Failed to join thread %u\n", (unsigned)tid[i]);
}
}
/* Join checker thread */
if( pthread_join(ctid, NULL) == 0 ) {
printf("Joined checker thread %u.\n", (unsigned)ctid);
} else {
printf("Failed to join checker thread %u\n", (unsigned)ctid);
}
/* How long did it take to run this ? */
end = times(&t);
printf("Wall time %2.4f s \n", (float)(end - start) / ticks);
return 0;
}
You should enter the mutex lock before you check bbp->cnt. Since you are checking it before you enter the mutex, another thread can reduce the value before you acquire the mutex and try to reduce the value yourself.