C - MPI - Bsend no work with buffer attach - c

I want the buffer to be full before I start receiving. It gives me the feeling that the buffer size is less than necessary
I followed the documentation to the letter but I can't see the error
https://www.mpich.org/static/docs/v3.1/www3/MPI_Buffer_attach.html
In some cases it gives me the feeling that with a specific buffer size I can perform more bsend than theoretically allowed
#include <stdio.h>
#include <mpi.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
int main(int argc, char *argv[])
{
int rank, size, tag=0;
MPI_Status status;
MPI_Request request;
int flag;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
int bsends_totales=1000000;
if (rank!=0) //Nodos esclavos
{
int size_bsend;
MPI_Pack_size( 1, MPI_INT, MPI_COMM_WORLD, &size_bsend );
int size_buffer=bsends_totales*(size_bsend+MPI_BSEND_OVERHEAD);
int * buffer = malloc(size_buffer);
memset(buffer,0,size_buffer);
MPI_Buffer_attach(buffer,size_buffer); //Buffer saliente
int enviar=4;
int sends_realizados=0;
for (int i=0;i<bsends_totales;i++)
{
printf("BSENDS realizados... %d\n",sends_realizados);
MPI_Bsend(&enviar,1,MPI_INT,0,tag,MPI_COMM_WORLD);
sends_realizados=sends_realizados+1;
}
printf("BSENDS TOTALES REALIZADOS: %d\n",sends_realizados);
MPI_Barrier(MPI_COMM_WORLD);
MPI_Buffer_detach(&buffer,&size_buffer);
free(buffer);
printf("TERMINE\n");
}
else //Master
{
int recibido;
MPI_Barrier(MPI_COMM_WORLD);
for (int i=0;i<bsends_totales;i++)
{
MPI_Recv(&recibido,1,MPI_INT,MPI_ANY_SOURCE,MPI_ANY_TAG,MPI_COMM_WORLD,&status);
}
}
MPI_Finalize();
return 0;
}
Before being able to fill the buffer an error appears
OUTPUT:
BSENDS realizados... 119696
BSENDS realizados... 119697
BSENDS realizados... 119698
BSENDS realizados... 119699
BSENDS realizados... 119700
code exit 11

Related

Send dynamic array inside a struct over MPI

In my C program I have a structure like the one below
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <mpi.h>
typedef struct Message
{
int elemNr;
char *elem;
} Msg;
I think I made all the steps to create custom data type in MPI
int main(int argc, char **argv) {
int size, rank;
int i;
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &size);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (size < 2) {
fprintf(stderr,"Requires at least two processes.\n");
exit(-1);
}
// just for simplicity
const int n = 5;
// create a new type for struct message
MPI_Datatype myType;
Msg msgSnd;
int block_length[2] = {1, n};
MPI_Aint elemNrAddr, elemAddr;
MPI_Aint displacement[2] = {0, 0};
MPI_Get_address(&msgSnd.elemNr, &elemNrAddr);
MPI_Get_address(&msgSnd.elem, &elemAddr);
// just displacement[1] because displacement[0] starts from 0
displacement[1] = elemAddr - elemNrAddr;
MPI_Datatype types[2] = {MPI_INT, MPI_CHAR};
MPI_Type_create_struct(2, block_length, displacement, types, &myType);
MPI_Type_commit(&myType);
// populate the message
msgSnd.elemNr = n;
msgSnd.elem = malloc(sizeof(char) * msgSnd.elemNr);
srand(time(NULL));
for (i = 0; i < msgSnd.elemNr; i++)
msgSnd.elem[i] = rand() % 26 + 'a';
if (rank != 0) {
printf("I'm sending\n");
MPI_Send(&msgSnd, 1, myType, 0, 0, MPI_COMM_WORLD);
printf("I sent\n");
} else {
MPI_Status status;
Msg msgRcv;
printf("I'm receiving\n");
MPI_Recv(&msgRcv, 1, myType, 1, 0, MPI_COMM_WORLD, &status);
printf("I received\n");
for (i = 0; i < msgRcv.elemNr; i++)
printf("element %d: %c\n", i, msgRcv.elem[i]);
if (msgRcv.elem != NULL)
free(msgRcv.elem);
}
if (msgSnd.elem != NULL)
free(msgSnd.elem);
MPI_Type_free(&myType);
MPI_Finalize();
return 0;
}
I ran the above code, but unfortunately when process 0 receives the message, the elem pointer points to null and the program ends with segmentation fault.
Can you help me to find the problem? Or, how can I send a dynamic array inside a struct on MPI?

MPI_Scatter produces write error, bad address (3)

I am receiving a writing error when trying to scatter a dynamically allocated matrix (it is contiguous), it happens when more than 5 cores are involved in the computation. I have placed printfs and it occurs in the scatter, the code is the next:
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <cblas.h>
#include <sys/time.h>
int main(int argc, char* argv[])
{
int err = MPI_Init(&argc, &argv);
MPI_Comm world;
world=MPI_COMM_WORLD;
int size = 0;
err = MPI_Comm_size(world, &size);
int rank = 0;
err = MPI_Comm_rank(world, &rank);
int n_rows=2400, n_cols=2400, n_rpc=n_rows/size;
float *A, *Asc, *B, *C; //Dyn alloc A B and C
Asc=malloc(n_rpc*n_cols*sizeof(float));
B=malloc(n_rows*n_cols*sizeof(float));
C=malloc(n_rows*n_cols*sizeof(float));
A=malloc(n_rows*n_cols*sizeof(float));
if(rank==0)
{
for (int i=0; i<n_rows; i++)
{
for (int j=0; j<n_cols; j++)
{
A[i*n_cols+j]= i+1.0;
B[i*n_cols+j]=A[i*n_cols+j];
}
}
}
struct timeval start, end;
if(rank==0) gettimeofday(&start,NULL);
MPI_Bcast(B, n_rows*n_cols, MPI_FLOAT, 0, MPI_COMM_WORLD);
if(rank==0) printf("Before Scatter\n"); //It is breaking here
MPI_Scatter(A, n_rpc*n_cols, MPI_FLOAT, Asc, n_rpc*n_cols, MPI_FLOAT, 0, MPI_COMM_WORLD);
if(rank==0) printf("After Scatter\n");
/* Some computation */
err = MPI_Finalize();
if (err) DIE("MPI_Finalize");
return err;
}
Upto 4 cores, it works correctly and performs the scatter, but with 5 or more it does not, and I can not find a clear reason.
The error message is as follows:
[raspberrypi][[26238,1],0][btl_tcp_frag.c:130:mca_btl_tcp_frag_send] mca_btl_tcp_frag_send: writev error (0xac51e0, 8)
Bad address(3)
[raspberrypi][[26238,1],0][btl_tcp_frag.c:130:mca_btl_tcp_frag_send] mca_btl_tcp_frag_send: writev error (0xaf197048, 29053982)
Bad address(1)
[raspberrypi:05345] pml_ob1_sendreq.c:308 FATAL
Thanks in advance!
Multiple errors, first of all, take care of using always the same type when defining variables. Then, when you use scatter, the send count and receive are the same, and you will be sending Elements/Cores. Also when receiving with gather you have to receive the same amount you sent, so again Elements/Cores.

MPI: Distributing segments of large file does not speedup execution

I have a large bioinformatics file (fasta), and I am using MPI to open the file at specific regions depending on the current program's ID. Then, I transcribe the amino acid sequence into their corresponding proteins.
#include <mpi.h>
int main(int argc, char* argv[]){
MPI_File in;
int id;
int p;
long buffersize = 3000000000/p;
MPI_Offset fileStart = buffersize * id;
char* nucleotides = (char*)malloc(buffersize);
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &id);
MPI_Comm_size(MPI_COMM_WORLD, &p);
MPI_File_open(MPI_COMM_WORLD, argv[1], MPI_MODE_RDONLY, MPI_INFO_NULL, &in);
MPI_File_read_at(in, fileStart, nucleotides, buffersize, MPI_CHAR, MPI_STATUS_IGNORE);
/* Calculations */
/* Write result */
MPI_File_close(&in);
free(nucleotides);
MPI_Finalize();
return 0;
}
I expect a speedup correlated with number of machines running the algorithm. However, I observe running my applications across multiple machines does not change execution time. Execution time appears to be independent of number of machines listed in my hostfile.
Any ideas how to get the expected behavior of more machines decreasing the read time?
To turn the comments into an answer:
Turn this
#include <mpi.h>
int main(int argc, char* argv[]){
MPI_File in;
int id;
int p;
//you are using p uninitialized here!!!
long buffersize = 3000000000/p;
//same applies to id
MPI_Offset fileStart = buffersize * id;
char* nucleotides = (char*)malloc(buffersize);
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &id);
MPI_Comm_size(MPI_COMM_WORLD, &p);
MPI_File_open(MPI_COMM_WORLD, argv[1], MPI_MODE_RDONLY, MPI_INFO_NULL, &in);
MPI_File_read_at(in, fileStart, nucleotides, buffersize, MPI_CHAR, MPI_STATUS_IGNORE);
/* Calculations */
/* Write result */
MPI_File_close(&in);
free(nucleotides);
MPI_Finalize();
return 0;
}
into this:
#include <mpi.h>
int main(int argc, char* argv[]){
MPI_File in;
MPI_Init(&argc, &argv);
int id;
int p;
MPI_Comm_rank(MPI_COMM_WORLD, &id);
MPI_Comm_size(MPI_COMM_WORLD, &p);
long buffersize = 3000000000/p;
MPI_Offset fileStart = buffersize * id;
char* nucleotides = (char*)malloc(buffersize);
MPI_File_open(MPI_COMM_WORLD, argv[1], MPI_MODE_RDONLY, MPI_INFO_NULL, &in);
MPI_File_read_at(in, fileStart, nucleotides, buffersize, MPI_CHAR, MPI_STATUS_IGNORE);
/* Calculations */
/* Write result */
MPI_File_close(&in);
free(nucleotides);
MPI_Finalize();
return 0;
}
No guarantee about whether the rest will work above.
I would highly recommend to make yourself comfortable with C programming before starting to write MPI code because you might end up lost. MPI issues are difficult to debug.

writing to text file in MPI

I'm trying to write to text file with MPI but the file is not created.
I need only to write at the master (rank = 0), but nothing works.
It only working when I running the program in console (and save corrupt element) and not in Mpich2 and
I attached the code.
Thanks for helping.
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* (C) 2001 by Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
/* This is an interactive version of cpi */
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>
int main(int argc,char *argv[])
{
int namelen, numprocs, rank;
char processor_name[MPI_MAX_PROCESSOR_NAME];
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
MPI_Comm_size(MPI_COMM_WORLD,&numprocs);
MPI_Get_processor_name(processor_name,&namelen);
MPI_Status status;
FILE* f = fopen("test.txt","wb+");
if (rank == 0) {
for (int i=0; i < 5; i++){
fprintf(f,"%d \n",i);
}
fclose(f);
}
else {
// do nothing
}
MPI_Finalize();
return 0;
}
In the sample code you posted, all processes open the file and only process 0 closes it. Could you try the following modification ?
if (rank == 0) {
FILE* f = fopen("test.txt","wb+");
if(f==NULL){printf("failed to open file: permission issue ?\n");exit(1);}
for (int i=0; i < 5; i++){
fprintf(f,"%d \n",i);
}
fclose(f);
}
Since your code seems to come from the Argonne National Laboratory, I suppose that it is ran on a cluster using a particular file system.
The following code is based on yours. It makes use of MPI_File_open() and MPI_File_write() on a single process, using MPI_COMM_SELF.
/* This is an interactive version of cpi */
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
int main(int argc,char *argv[])
{
int namelen, numprocs, rank;
char processor_name[MPI_MAX_PROCESSOR_NAME];
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
MPI_Comm_size(MPI_COMM_WORLD,&numprocs);
MPI_Get_processor_name(processor_name,&namelen);
MPI_Status status;
MPI_File fh;
if (rank == 0) {
MPI_File_open(MPI_COMM_SELF, "test.txt",MPI_MODE_CREATE | MPI_MODE_WRONLY,MPI_INFO_NULL,&fh);
//FILE* f = fopen("test.txt","wb+");
//if(f==NULL){
//printf("failed to open file\n");exit(1);
//}
for (int i=0; i < 5; i++){
char buf[42];
//fprintf(f,"%d \n",i);
snprintf(buf,42,"%d \n",i);
MPI_File_write(fh,buf,strlen(buf), MPI_CHAR,&status);
}
// fclose(f);
MPI_File_close(&fh);
}
else {
// do nothing
}
MPI_Finalize();
return 0;
}
Please make sure that you hae the permission to write in the considered folder. Make sure that all nodes can access this folder ! Try some folders like your folder in /tmp or /scratch... Your cluster may have some sort of documentation somewhere telling you where you can write files !

Callbacks provided in MPI_Comm_create_keyval are not called

I am reading "Using MPI" and try to execute the code myself. There is a nonblocking broadcast code in Chapter 6.2. I tried to run with my own callbacks instead of MPI_NULL_COPY_FN or MPI_NULL_DELETE_FN. Here is my code, it is very similar to the code in book, but the callbacks will not be called. I am not sure why. There are no warnings or errors when I compile with -Wall. Could you help me please? Thanks a lot.
#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>
static int ibcast_keyval=MPI_KEYVAL_INVALID;
typedef struct
{
MPI_Comm comm;
int ordering_tag;
} Ibcast_syshandle;
typedef struct
{
MPI_Request *req_array;
MPI_Status *stat_array;
int num_sends;
int num_recvs;
} Ibcast_handle;
int Ibcast_work(Ibcast_handle *handle)
{
if(handle->num_recvs==0)
MPI_Startall(handle->num_sends, handle->req_array);
else
MPI_Startall(handle->num_recvs, &(handle->req_array[handle->num_sends]));
return MPI_SUCCESS;
}
int Ibcast_copy(MPI_Comm oldcomm, int keyval, void *extra, void *attr_in, void *attr_out, int *flag)
{
Ibcast_syshandle *syshandle=(Ibcast_syshandle *)attr_in;
Ibcast_syshandle *new_syshandle;
printf("keyval=%d\n", keyval);
fflush(stdout);
if((keyval==MPI_KEYVAL_INVALID)||(keyval!=ibcast_keyval)||(syshandle==NULL))
return 1;
new_syshandle=(Ibcast_syshandle *)malloc(sizeof(Ibcast_syshandle));
new_syshandle->ordering_tag=0;
MPI_Comm_dup(syshandle->comm, &(new_syshandle->comm));
{
int rank;
MPI_Comm_rank(new_syshandle->comm, &rank);
printf("Ibcast_copy called from %d\n", rank);
fflush(stdout);
}
*(void **)attr_out=(void *)new_syshandle;
*flag=1;
return MPI_SUCCESS;
}
int Ibcast_delete(MPI_Comm comm, int keyval, void *attr_val, void *extra)
{
Ibcast_syshandle *syshandle=(Ibcast_syshandle *)attr_val;
{
int rank;
MPI_Comm_rank(syshandle->comm, &rank);
printf("Ibcast_delete called from %d\n", rank);
fflush(stdout);
}
if((keyval==MPI_KEYVAL_INVALID)||(keyval!=ibcast_keyval)||(syshandle==NULL))
return 1;
MPI_Comm_free(&(syshandle->comm));
free(syshandle);
return MPI_SUCCESS;
}
int Ibcast(void *buf, int count, MPI_Datatype datatype, int root, MPI_Comm comm, Ibcast_handle **handle_out)
{
Ibcast_syshandle *syshandle;
Ibcast_handle *handle;
int flag, mask, relrank;
int retn, size, rank;
int req_no=0;
MPI_Comm_size(comm, &size);
MPI_Comm_rank(comm, &rank);
if(size==1)
{
(*handle_out)=NULL;
return MPI_SUCCESS;
}
if(ibcast_keyval==MPI_KEYVAL_INVALID)
// MPI_Keyval_create(MPI_NULL_COPY_FN, MPI_NULL_DELETE_FN, &ibcast_keyval, NULL);
MPI_Comm_create_keyval(Ibcast_copy, Ibcast_delete, &ibcast_keyval, NULL);
MPI_Comm_get_attr(comm, ibcast_keyval, (void **)&syshandle, &flag);
if(flag==0)
{
syshandle=(Ibcast_syshandle *)malloc(sizeof(Ibcast_syshandle));
syshandle->ordering_tag=0;
MPI_Comm_dup(comm, &(syshandle->comm));
MPI_Comm_set_attr(comm, ibcast_keyval, (void *)syshandle);
}
handle=(Ibcast_handle *)malloc(sizeof(Ibcast_handle));
handle->num_sends=handle->num_recvs=0;
mask=0x1;
relrank=(rank-root+size)%size;
while((mask&relrank)==0 && mask<size)
{
if((relrank|mask)<size)
++handle->num_sends;
mask<<=1;
}
if(mask<size)
++handle->num_recvs;
handle->req_array=(MPI_Request *)malloc(sizeof(MPI_Request)*(handle->num_sends+handle->num_recvs));
handle->stat_array=(MPI_Status *)malloc(sizeof(MPI_Status)*(handle->num_sends+handle->num_recvs));
mask=0x1;
relrank=(rank-root+size)%size;
while((mask&relrank)==0 && mask<size)
{
if((relrank|mask)<size)
MPI_Send_init(buf, count, datatype, ((relrank|mask)+root)%size, syshandle->ordering_tag, syshandle->comm, &(handle->req_array[req_no++]));
mask<<=1;
}
if(mask<size)
MPI_Recv_init(buf, count, datatype, ((relrank & (~mask))+root)%size, syshandle->ordering_tag, syshandle->comm, &(handle->req_array[req_no++]));
retn=Ibcast_work(handle);
++(syshandle->ordering_tag);
(*handle_out)=handle;
return retn;
}
int Ibcast_wait(Ibcast_handle **handle_out)
{
Ibcast_handle *handle=(*handle_out);
int retn, i;
if(handle==NULL)
return MPI_SUCCESS;
if(handle->num_recvs!=0)
{
MPI_Waitall(handle->num_recvs, &handle->req_array[handle->num_sends], &handle->stat_array[handle->num_sends]);
MPI_Startall(handle->num_sends, handle->req_array);
}
retn=MPI_Waitall(handle->num_sends, handle->req_array, handle->stat_array);
for(i=0; i<(handle->num_sends+handle->num_recvs);i++)
MPI_Request_free(&(handle->req_array[i]));
free(handle->req_array);
free(handle->stat_array);
free(handle);
*handle_out=NULL;
return retn;
}
int main( int argc, char *argv[] )
{
int buf1[10], buf2[20];
int rank, i;
Ibcast_handle *ibcast_handle_1, *ibcast_handle_2;
MPI_Init( &argc, &argv );
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
if (rank == 0) {
for (i=0; i<10; i++) buf1[i] = i;
for (i=0; i<20; i++) buf2[i] = -i;
}
Ibcast( buf1, 10, MPI_INT, 0, MPI_COMM_WORLD, &ibcast_handle_1 );
Ibcast( buf2, 20, MPI_INT, 0, MPI_COMM_WORLD, &ibcast_handle_2 );
Ibcast_wait( &ibcast_handle_1 );
Ibcast_wait( &ibcast_handle_2 );
for (i=0; i<10; i++) {
if (buf1[i] != i) printf( "buf1[%d] = %d on %d\n", i, buf1[i], rank );
}
for (i=0; i<20; i++) {
if (buf2[i] != -i) printf( "buf2[%d] = %d on %d\n", i, buf2[i], rank );
}
MPI_Finalize();
return 0;
}
The callback functions are there to copy and delete the created attributes when a communicator is duplicated or deleted, or just when the attribute is deleted. The callback functions are necessary because the attributes can be completely arbitrary.
So here's a stripped down version of your code that does work (creating such a minimal example being a useful way to both track down problems and get help on sites like SO):
#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>
static int ibcast_keyval;
int Ibcast_copy(MPI_Comm oldcomm, int keyval, void *extra, void *attr_in, void *attr_out, int *flag)
{
printf("In ibcast_copy: keyval = %d\n", keyval);
*flag = 1;
return MPI_SUCCESS;
}
int Ibcast_delete(MPI_Comm comm, int keyval, void *attr_val, void *extra)
{
printf("In ibcast_delete: keyval = %d\n", keyval);
return MPI_SUCCESS;
}
int main( int argc, char *argv[] )
{
int rank, i;
int attr=2;
MPI_Init( &argc, &argv );
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
MPI_Comm duped_comm;
MPI_Comm_create_keyval(Ibcast_copy, Ibcast_delete, &ibcast_keyval, NULL);
MPI_Comm_set_attr( MPI_COMM_WORLD, ibcast_keyval, &attr);
MPI_Comm_dup( MPI_COMM_WORLD, &duped_comm);
MPI_Comm_free( &duped_comm );
MPI_Comm_delete_attr( MPI_COMM_WORLD, ibcast_keyval );
MPI_Finalize();
return 0;
}
Here we create the keyval with the callbacks, set the attribute corresponding to the key, and then dup MPI_COMM_WORLD (invoking the copy callback) and then free the dup'ed communicator and deleting the attribute from COMM_WORLD (invoking the delete callback both times:)
$ mpirun -np 1 ./comm-attr
In ibcast_copy: keyval = 10
In ibcast_delete: keyval = 10
In ibcast_delete: keyval = 10
In your code, you dup the communicator in Ibcast before setting the attribute, so that the copy callback is never invoked (as there is nothing to copy). You can fix that part by setting the attribute before the dup, but then there is another problem - you call dup and free within the callbacks, which is wrong-way around; those functions are to (indirectly) invoke the callbacks, not vice-versa.

Resources