MPI Receive much slower when called in child thread rather than parent - c

Question

In the code below I try to receive an MPI message having two active threads. Passing 0 as command line argument the parent thread will use MPI_Recv and the child will send it. Passing 1 will cause the child to use MPI_Recv and the parent to send. I see huge differences in the times that I get between the 2 case as the size of the message becomes bigger. Do you have any idea why this could be happening?
#include <stdio.h>
#include <mpi.h>
#include <time.h>
#include <unistd.h>
#include <sys/time.h>
#include <stdlib.h>
#include <pthread.h>
pthread_t _comm_thread;
#define KILO 1024
#define MILLION 1000000
#define START_TIMER(timer) { gettimeofday(&(timer), NULL); }
#define STOP_TIMER(timer) { gettimeofday(&(timer), NULL); }
#define TIME_DIFF(timer1, timer2, total) { \
long long sec_diff = 0; \
long long usec_diff = 0; \
sec_diff = (timer2).tv_sec - (timer1).tv_sec; \
usec_diff = (timer2).tv_usec - (timer1).tv_usec; \
(total) += (sec_diff * MILLION) + usec_diff; \
}
short done_flag = 0;
short finished = 0;
int num_messages = 500;
int payload_sizes[] = {1, 2, 4 , 8, 16,32,64,128,256,512,1024,2*KILO,4*KILO,8*KILO,16*KILO,32*KILO,64*KILO,128*KILO,256*KILO,512*KILO,KILO*KILO};
// int payload_sizes[] = {KILO*KILO};
int num_payload_sizes = 21;
int loops=0;
void* receive(void* args){
int my_proc;
MPI_Comm_rank(MPI_COMM_WORLD,&my_proc);
long long total;
struct timeval start, stop;
if( my_proc==0 ){
int i;
char buffer [KILO*KILO];
for(i=0;i<num_payload_sizes;++i)
{
done_flag = 0;
total = 0;
int j;
START_TIMER(start)
for(j=0;j<num_messages;++j)
{
MPI_Send(buffer,payload_sizes[i],MPI_BYTE,1,0,MPI_COMM_WORLD);
}
STOP_TIMER(stop)
TIME_DIFF(start,stop,total)
printf("Payload size: %d : time : %lld usec\n",payload_sizes[i],(total)/num_messages);
}
}
else{
int cnt=0;
char* buffer;
while(1){
MPI_Status stat;
int size;
MPI_Probe(MPI_ANY_SOURCE,MPI_ANY_TAG,MPI_COMM_WORLD,&stat);
MPI_Get_count(&stat,MPI_BYTE,&size);
buffer = (char *)malloc(size);
MPI_Recv(buffer,size,MPI_BYTE,0,0,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
cnt++;
if(cnt == num_messages){
loops++;
cnt =0;
}
if(loops == num_payload_sizes){
free(buffer);
break;
}
free(buffer);
}
}
pthread_exit(0);
}
void * nothing(void *args){
pthread_exit(0);
}
int main(int argc, char* argv[]){
int provided;
if(argc != 2){
printf("Please pass as argument 0 to run the MPI_Recv in parent or 1 to run it in the child\n");
return -1;
}
int choice = atoi(argv[1]);
MPI_Init_thread( NULL, NULL, MPI_THREAD_SERIALIZED,&provided);
if( provided!= MPI_THREAD_SERIALIZED){
printf("MPI_THREAD_SERIALIZED is not provided\n");
return -1;
}
if( choice == 0){
pthread_create( &_comm_thread, NULL, nothing, NULL);
receive(NULL);
}else if(choice == 1){
pthread_create( &_comm_thread, NULL, receive, NULL);
}else{
printf("Must choose 0 or 1\n");
return -1;
}
pthread_join(_comm_thread,NULL);
MPI_Finalize();
return 0;
}