I am receiving a writing error when trying to scatter a dynamically allocated matrix (it is contiguous), it happens when more than 5 cores are involved in the computation. I have placed printfs and it occurs in the scatter, the code is the next:
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <cblas.h>
#include <sys/time.h>
int main(int argc, char* argv[])
{
int err = MPI_Init(&argc, &argv);
MPI_Comm world;
world=MPI_COMM_WORLD;
int size = 0;
err = MPI_Comm_size(world, &size);
int rank = 0;
err = MPI_Comm_rank(world, &rank);
int n_rows=2400, n_cols=2400, n_rpc=n_rows/size;
float *A, *Asc, *B, *C; //Dyn alloc A B and C
Asc=malloc(n_rpc*n_cols*sizeof(float));
B=malloc(n_rows*n_cols*sizeof(float));
C=malloc(n_rows*n_cols*sizeof(float));
A=malloc(n_rows*n_cols*sizeof(float));
if(rank==0)
{
for (int i=0; i<n_rows; i++)
{
for (int j=0; j<n_cols; j++)
{
A[i*n_cols+j]= i+1.0;
B[i*n_cols+j]=A[i*n_cols+j];
}
}
}
struct timeval start, end;
if(rank==0) gettimeofday(&start,NULL);
MPI_Bcast(B, n_rows*n_cols, MPI_FLOAT, 0, MPI_COMM_WORLD);
if(rank==0) printf("Before Scatter\n"); //It is breaking here
MPI_Scatter(A, n_rpc*n_cols, MPI_FLOAT, Asc, n_rpc*n_cols, MPI_FLOAT, 0, MPI_COMM_WORLD);
if(rank==0) printf("After Scatter\n");
/* Some computation */
err = MPI_Finalize();
if (err) DIE("MPI_Finalize");
return err;
}
Upto 4 cores, it works correctly and performs the scatter, but with 5 or more it does not, and I can not find a clear reason.
The error message is as follows:
[raspberrypi][[26238,1],0][btl_tcp_frag.c:130:mca_btl_tcp_frag_send] mca_btl_tcp_frag_send: writev error (0xac51e0, 8)
Bad address(3)
[raspberrypi][[26238,1],0][btl_tcp_frag.c:130:mca_btl_tcp_frag_send] mca_btl_tcp_frag_send: writev error (0xaf197048, 29053982)
Bad address(1)
[raspberrypi:05345] pml_ob1_sendreq.c:308 FATAL
Thanks in advance!
Multiple errors, first of all, take care of using always the same type when defining variables. Then, when you use scatter, the send count and receive are the same, and you will be sending Elements/Cores. Also when receiving with gather you have to receive the same amount you sent, so again Elements/Cores.
Related
I want the buffer to be full before I start receiving. It gives me the feeling that the buffer size is less than necessary
I followed the documentation to the letter but I can't see the error
https://www.mpich.org/static/docs/v3.1/www3/MPI_Buffer_attach.html
In some cases it gives me the feeling that with a specific buffer size I can perform more bsend than theoretically allowed
#include <stdio.h>
#include <mpi.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
int main(int argc, char *argv[])
{
int rank, size, tag=0;
MPI_Status status;
MPI_Request request;
int flag;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
int bsends_totales=1000000;
if (rank!=0) //Nodos esclavos
{
int size_bsend;
MPI_Pack_size( 1, MPI_INT, MPI_COMM_WORLD, &size_bsend );
int size_buffer=bsends_totales*(size_bsend+MPI_BSEND_OVERHEAD);
int * buffer = malloc(size_buffer);
memset(buffer,0,size_buffer);
MPI_Buffer_attach(buffer,size_buffer); //Buffer saliente
int enviar=4;
int sends_realizados=0;
for (int i=0;i<bsends_totales;i++)
{
printf("BSENDS realizados... %d\n",sends_realizados);
MPI_Bsend(&enviar,1,MPI_INT,0,tag,MPI_COMM_WORLD);
sends_realizados=sends_realizados+1;
}
printf("BSENDS TOTALES REALIZADOS: %d\n",sends_realizados);
MPI_Barrier(MPI_COMM_WORLD);
MPI_Buffer_detach(&buffer,&size_buffer);
free(buffer);
printf("TERMINE\n");
}
else //Master
{
int recibido;
MPI_Barrier(MPI_COMM_WORLD);
for (int i=0;i<bsends_totales;i++)
{
MPI_Recv(&recibido,1,MPI_INT,MPI_ANY_SOURCE,MPI_ANY_TAG,MPI_COMM_WORLD,&status);
}
}
MPI_Finalize();
return 0;
}
Before being able to fill the buffer an error appears
OUTPUT:
BSENDS realizados... 119696
BSENDS realizados... 119697
BSENDS realizados... 119698
BSENDS realizados... 119699
BSENDS realizados... 119700
code exit 11
As a continuation to my previous question, I have modified the code for variable number of kernels. However, the way Gatherv is implemented in my code seems to be unreliable. Once in 3-4 runs the end sequence in the collecting buffer ends up being corrupted, it seems like, due to the memory leakage. Sample code:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <mpi.h>
int main (int argc, char *argv[]) {
MPI_Init(&argc, &argv);
int world_size,*sendarray;
int rank, *rbuf=NULL, count,total_counts=0;
int *displs=NULL,i,*rcounts=NULL;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &world_size);
if(rank==0){
displs = malloc((world_size+1)*sizeof(int));
for(int i=1;i<=world_size; i++)displs[i]=0;
rcounts=malloc(world_size*sizeof(int));
sendarray=malloc(1*sizeof(int));
for(int i=0;i<1;i++)sendarray[i]=1111;
count=1;
}
if(rank!=0){
int size=rank*2;
sendarray=malloc(size*sizeof(int));
for(int i=0;i<size;i++)sendarray[i]=rank;
count=size;
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_Gather(&count,1,MPI_INT,rcounts,1,MPI_INT,0,MPI_COMM_WORLD);
MPI_Barrier(MPI_COMM_WORLD);
if(rank==0){
displs[0]=0;
for(int i=1;i<=world_size; i++){
for(int j=0; j<i; j++)displs[i]+=rcounts[j];
}
total_counts=0;
for(int i=0;i<world_size;i++)total_counts+=rcounts[i];
rbuf = malloc(10*sizeof(int));
}
MPI_Gatherv(sendarray, count, MPI_INT, rbuf, rcounts,
displs, MPI_INT, 0, MPI_COMM_WORLD);
if(rank==0){
int SIZE=total_counts;
for(int i=0;i<SIZE;i++)printf("(%d) %d ",i, rbuf[i]);
free(rbuf);
free(displs);
free(rcounts);
}
if(rank!=0)free(sendarray);
MPI_Finalize();
}
Why is this happening and is there a way to fix it?
This becomes much worse in my actual project. Each sending buffer contains 150 doubles. The receiving buffer gets very dirty and sometimes I get an error of bed termination with exit code 6 or 11.
Can anyone at least reproduce my errors?
My guess: I am allocating memory for sendarray on each thread separately. If my virtual machine was 1-to-1 to the hardware, then, probably, there would be no such problem. But I have only 2 cores and run a process for 4 or more. Could it be the reason?
Change this line:
rbuf = malloc(10*sizeof(int));
to:
rbuf = malloc(total_counts*sizeof(int));
As a side note: each MPI process exists in its own process address space and they cannot stomp on eachothers data except through erroneous data explicitly passed through the MPI_XXX functions, which results in undefined behavior.
I am new to MPI and I am trying to manage arrays of different size in parallel and then pass them to the main thread, unsuccessfully so far.
I have learned that
MPI_Gatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
void *recvbuf, const int *recvcounts, const int *displs,
MPI_Datatype recvtype, int root, MPI_Comm comm)
is the way to go in this case.
Here is my sample code, which doesn't work because of memory issues (I think).
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <mpi.h>
int main (int argc, char *argv[]) {
MPI_Init(&argc, &argv);
int world_size,*sendarray;
int rank, *rbuf=NULL, count;
int *displs=NULL,i,*rcounts=NULL;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &world_size);
if(rank==0){
rbuf = malloc(10*sizeof(int));
displs = malloc(world_size*sizeof(int));
rcounts=malloc(world_size*sizeof(int));
rcounts[0]=1;
rcounts[1]=3;
rcounts[2]=6;
displs[0]=1;
displs[1]=3;
displs[2]=6;
sendarray=malloc(1*sizeof(int));
for(int i=0;i<1;i++)sendarray[i]=1;
count=1;
}
if(rank==1){
sendarray=malloc(3*sizeof(int));
for(int i=0;i<3;i++)sendarray[i]=2;
count=3;
}
if(rank==2){
sendarray=malloc(6*sizeof(int));
for(int i=0;i<6;i++)sendarray[i]=3;
count=6;
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_Gatherv(sendarray, count, MPI_INT, rbuf, rcounts,
displs, MPI_INT, 0, MPI_COMM_WORLD);
if(rank==0){
int SIZE=10;
for(int i=0;i<SIZE;i++)printf("(%d) %d ",i, rbuf[i]);
free(rbuf);
free(displs);
free(rcounts);
}
if(rank!=0)free(sendarray);
MPI_Finalize();
}
Specifically, when I run it, I get
(0) 0 (1) 1 (2) 0 (3) 2 (4) 2 (5) 2 (6) 3 (7) 3 (8) 3 (9) 3
Instead of something like this
(0) 1 (1) 2 (2) 2 (3) 2 (4) 3 (5) 3 (6) 3 (7) 3 (8) 3 (9) 3
Why is that?
What is even more interesting, is that it seems like missing elements are stored in 11th and 12th element of the rbuf, even though those are supposed to not even exist at the first place.
Your program is very close to working. If you change these lines:
displs[0]=1;
displs[1]=3;
displs[2]=6;
to this:
displs[0]=0;
displs[1]=displs[0]+rcounts[0];
displs[2]=displs[1]+rcounts[1];
you will get the expected output. The variable displs is the offset into the receiving buffer to place the data from process i.
I have the following code which works:
#include <mpi.h>
#include <stdio.h>
int main(int argc, char** argv) {
int world_rank, world_size;
MPI_Init(NULL, NULL);
MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
MPI_Comm_size(MPI_COMM_WORLD, &world_size);
int n = 10000;
int ni, i;
double t[n];
int x[n];
int buf[n];
int buf_size = n*sizeof(int);
MPI_Buffer_attach(buf, buf_size);
if (world_rank == 0) {
for (ni = 0; ni < n; ++ni) {
int msg_size = ni;
int msg[msg_size];
for (i = 0; i < msg_size; ++i) {
msg[i] = rand();
}
double time0 = MPI_Wtime();
MPI_Bsend(&msg, msg_size, MPI_INT, 1, 0, MPI_COMM_WORLD);
t[ni] = MPI_Wtime() - time0;
x[ni] = msg_size;
MPI_Barrier(MPI_COMM_WORLD);
printf("P0 sent msg with size %d\n", msg_size);
}
}
else if (world_rank == 1) {
for (ni = 0; ni < n; ++ni) {
int msg_size = ni;
int msg[msg_size];
MPI_Request request;
MPI_Barrier(MPI_COMM_WORLD);
MPI_Irecv(&msg, msg_size, MPI_INT, 0, 0, MPI_COMM_WORLD, &request);
MPI_Wait(&request, MPI_STATUS_IGNORE);
printf("P1 received msg with size %d\n", msg_size);
}
}
MPI_Buffer_detach(&buf, &buf_size);
MPI_Finalize();
}
As soon as I remove the print statements, the program crashes, telling me there is a MPI_ERR_BUFFER: invalid buffer pointer. If I remove only one of the print statements the other print statements are still executed, so I believe it crashes at the end of the program. I don't see why it crashes and the fact that it does not crash when I am using the print statements goes beyond my logic...
Would anybody have a clue what is going on here?
You are simply not providing enough buffer space to MPI. In buffered mode, all ongoing messages are stored in the buffer space which is used as a ring buffer. In your code, there can be multiple messages that need to be buffered, regardless of the printf. Note that not even 2*n*sizeof(int) would be enough buffer space - the barriers do not provide a guarantee that the buffer is locally freed even though the corresponding receive is completed. You would have to provide (n*(n-1)/2)*sizeof(int) memory to be sure, or something in-between and hope.
Bottom line: Don't use buffered mode.
Generally, use standard blocking send calls and write the application such that it doesn't deadlock. Tune the MPI implementation such that small messages regardless of the receiver - to avoid wait times on late receivers.
If you want to overlap communication and computation, use nonblocking messages - providing proper memory for each communication.
I have encountered a problem on using MPI_Gather for gathering indexed integers to a vector of integers. When I try to gather the integers without creating a new receive type, I get a MPI_ERR_TRUNCATE error.
*** An error occurred in MPI_Gather
*** on communicator MPI_COMM_WORLD
*** MPI_ERR_TRUNCATE: message truncated
*** MPI_ERRORS_ARE_FATAL: your MPI job will now abort
The minimal example replicating the issue is
#include <stdlib.h>
#include "mpi.h"
int i, comm_rank, comm_size, err;
int *send_data, *recv_data;
int *blocklengths, *displacements;
MPI_Datatype send_type;
int main ( int argc, char *argv[] ){
MPI_Init ( &argc, &argv );
MPI_Comm_rank(MPI_COMM_WORLD, &comm_rank);
MPI_Comm_size(MPI_COMM_WORLD, &comm_size);
unsigned int block = 1000;
unsigned int count = 1000;
send_data = malloc(sizeof(int)*block*count);
for (i=0; i<block*count; ++i) send_data[i] = i;
recv_data = 0;
if(comm_rank==0) recv_data = malloc(sizeof(int)*block*count*comm_size);
blocklengths = malloc(sizeof(int)*count);
displacements = malloc(sizeof(int)*count);
for (i=0; i<count; ++i) {
blocklengths[i] = block;
displacements[i] = i*block;
}
MPI_Type_indexed(count, blocklengths, displacements, MPI_INT, &send_type);
MPI_Type_commit(&send_type);
err = MPI_Gather((void*)send_data, 1, send_type, (void*)recv_data, block*count, MPI_INT, 0, MPI_COMM_WORLD);
if (err) MPI_Abort(MPI_COMM_WORLD, err);
free(send_data);
free(recv_data);
free(blocklengths);
free(displacements);
MPI_Finalize ( );
return 0;
}
I noticed that this error does not occur when I use data transfer size less than 6K bytes.
I found a workaround using MPI_Type_contiguous, although it seems I add extra overhead to my code.
MPI_Type_contiguous(block*count, MPI_INT, &recv_type);
MPI_Type_commit(&recv_type);
err = MPI_Gather((void*)send_data, 1, send_type, (void*)recv_data, 1, recv_type, 0, MPI_COMM_WORLD);
I have verified the error occurs in open-mpi v1.6 and v1.8.
Could anyone explain the source of this issue?