Callbacks provided in MPI_Comm_create_keyval are not called - c

Question

I am reading "Using MPI" and try to execute the code myself. There is a nonblocking broadcast code in Chapter 6.2. I tried to run with my own callbacks instead of MPI_NULL_COPY_FN or MPI_NULL_DELETE_FN. Here is my code, it is very similar to the code in book, but the callbacks will not be called. I am not sure why. There are no warnings or errors when I compile with -Wall. Could you help me please? Thanks a lot.
#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>
static int ibcast_keyval=MPI_KEYVAL_INVALID;
typedef struct
{
MPI_Comm comm;
int ordering_tag;
} Ibcast_syshandle;
typedef struct
{
MPI_Request *req_array;
MPI_Status *stat_array;
int num_sends;
int num_recvs;
} Ibcast_handle;
int Ibcast_work(Ibcast_handle *handle)
{
if(handle->num_recvs==0)
MPI_Startall(handle->num_sends, handle->req_array);
else
MPI_Startall(handle->num_recvs, &(handle->req_array[handle->num_sends]));
return MPI_SUCCESS;
}
int Ibcast_copy(MPI_Comm oldcomm, int keyval, void *extra, void *attr_in, void *attr_out, int *flag)
{
Ibcast_syshandle *syshandle=(Ibcast_syshandle *)attr_in;
Ibcast_syshandle *new_syshandle;
printf("keyval=%d\n", keyval);
fflush(stdout);
if((keyval==MPI_KEYVAL_INVALID)||(keyval!=ibcast_keyval)||(syshandle==NULL))
return 1;
new_syshandle=(Ibcast_syshandle *)malloc(sizeof(Ibcast_syshandle));
new_syshandle->ordering_tag=0;
MPI_Comm_dup(syshandle->comm, &(new_syshandle->comm));
{
int rank;
MPI_Comm_rank(new_syshandle->comm, &rank);
printf("Ibcast_copy called from %d\n", rank);
fflush(stdout);
}
*(void **)attr_out=(void *)new_syshandle;
*flag=1;
return MPI_SUCCESS;
}
int Ibcast_delete(MPI_Comm comm, int keyval, void *attr_val, void *extra)
{
Ibcast_syshandle *syshandle=(Ibcast_syshandle *)attr_val;
{
int rank;
MPI_Comm_rank(syshandle->comm, &rank);
printf("Ibcast_delete called from %d\n", rank);
fflush(stdout);
}
if((keyval==MPI_KEYVAL_INVALID)||(keyval!=ibcast_keyval)||(syshandle==NULL))
return 1;
MPI_Comm_free(&(syshandle->comm));
free(syshandle);
return MPI_SUCCESS;
}
int Ibcast(void *buf, int count, MPI_Datatype datatype, int root, MPI_Comm comm, Ibcast_handle **handle_out)
{
Ibcast_syshandle *syshandle;
Ibcast_handle *handle;
int flag, mask, relrank;
int retn, size, rank;
int req_no=0;
MPI_Comm_size(comm, &size);
MPI_Comm_rank(comm, &rank);
if(size==1)
{
(*handle_out)=NULL;
return MPI_SUCCESS;
}
if(ibcast_keyval==MPI_KEYVAL_INVALID)
// MPI_Keyval_create(MPI_NULL_COPY_FN, MPI_NULL_DELETE_FN, &ibcast_keyval, NULL);
MPI_Comm_create_keyval(Ibcast_copy, Ibcast_delete, &ibcast_keyval, NULL);
MPI_Comm_get_attr(comm, ibcast_keyval, (void **)&syshandle, &flag);
if(flag==0)
{
syshandle=(Ibcast_syshandle *)malloc(sizeof(Ibcast_syshandle));
syshandle->ordering_tag=0;
MPI_Comm_dup(comm, &(syshandle->comm));
MPI_Comm_set_attr(comm, ibcast_keyval, (void *)syshandle);
}
handle=(Ibcast_handle *)malloc(sizeof(Ibcast_handle));
handle->num_sends=handle->num_recvs=0;
mask=0x1;
relrank=(rank-root+size)%size;
while((mask&relrank)==0 && mask<size)
{
if((relrank|mask)<size)
++handle->num_sends;
mask<<=1;
}
if(mask<size)
++handle->num_recvs;
handle->req_array=(MPI_Request *)malloc(sizeof(MPI_Request)*(handle->num_sends+handle->num_recvs));
handle->stat_array=(MPI_Status *)malloc(sizeof(MPI_Status)*(handle->num_sends+handle->num_recvs));
mask=0x1;
relrank=(rank-root+size)%size;
while((mask&relrank)==0 && mask<size)
{
if((relrank|mask)<size)
MPI_Send_init(buf, count, datatype, ((relrank|mask)+root)%size, syshandle->ordering_tag, syshandle->comm, &(handle->req_array[req_no++]));
mask<<=1;
}
if(mask<size)
MPI_Recv_init(buf, count, datatype, ((relrank & (~mask))+root)%size, syshandle->ordering_tag, syshandle->comm, &(handle->req_array[req_no++]));
retn=Ibcast_work(handle);
++(syshandle->ordering_tag);
(*handle_out)=handle;
return retn;
}
int Ibcast_wait(Ibcast_handle **handle_out)
{
Ibcast_handle *handle=(*handle_out);
int retn, i;
if(handle==NULL)
return MPI_SUCCESS;
if(handle->num_recvs!=0)
{
MPI_Waitall(handle->num_recvs, &handle->req_array[handle->num_sends], &handle->stat_array[handle->num_sends]);
MPI_Startall(handle->num_sends, handle->req_array);
}
retn=MPI_Waitall(handle->num_sends, handle->req_array, handle->stat_array);
for(i=0; i<(handle->num_sends+handle->num_recvs);i++)
MPI_Request_free(&(handle->req_array[i]));
free(handle->req_array);
free(handle->stat_array);
free(handle);
*handle_out=NULL;
return retn;
}
int main( int argc, char *argv[] )
{
int buf1[10], buf2[20];
int rank, i;
Ibcast_handle *ibcast_handle_1, *ibcast_handle_2;
MPI_Init( &argc, &argv );
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
if (rank == 0) {
for (i=0; i<10; i++) buf1[i] = i;
for (i=0; i<20; i++) buf2[i] = -i;
}
Ibcast( buf1, 10, MPI_INT, 0, MPI_COMM_WORLD, &ibcast_handle_1 );
Ibcast( buf2, 20, MPI_INT, 0, MPI_COMM_WORLD, &ibcast_handle_2 );
Ibcast_wait( &ibcast_handle_1 );
Ibcast_wait( &ibcast_handle_2 );
for (i=0; i<10; i++) {
if (buf1[i] != i) printf( "buf1[%d] = %d on %d\n", i, buf1[i], rank );
}
for (i=0; i<20; i++) {
if (buf2[i] != -i) printf( "buf2[%d] = %d on %d\n", i, buf2[i], rank );
}
MPI_Finalize();
return 0;
}

Answer 1

The callback functions are there to copy and delete the created attributes when a communicator is duplicated or deleted, or just when the attribute is deleted. The callback functions are necessary because the attributes can be completely arbitrary.
So here's a stripped down version of your code that does work (creating such a minimal example being a useful way to both track down problems and get help on sites like SO):
#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>
static int ibcast_keyval;
int Ibcast_copy(MPI_Comm oldcomm, int keyval, void *extra, void *attr_in, void *attr_out, int *flag)
{
printf("In ibcast_copy: keyval = %d\n", keyval);
*flag = 1;
return MPI_SUCCESS;
}
int Ibcast_delete(MPI_Comm comm, int keyval, void *attr_val, void *extra)
{
printf("In ibcast_delete: keyval = %d\n", keyval);
return MPI_SUCCESS;
}
int main( int argc, char *argv[] )
{
int rank, i;
int attr=2;
MPI_Init( &argc, &argv );
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
MPI_Comm duped_comm;
MPI_Comm_create_keyval(Ibcast_copy, Ibcast_delete, &ibcast_keyval, NULL);
MPI_Comm_set_attr( MPI_COMM_WORLD, ibcast_keyval, &attr);
MPI_Comm_dup( MPI_COMM_WORLD, &duped_comm);
MPI_Comm_free( &duped_comm );
MPI_Comm_delete_attr( MPI_COMM_WORLD, ibcast_keyval );
MPI_Finalize();
return 0;
}
Here we create the keyval with the callbacks, set the attribute corresponding to the key, and then dup MPI_COMM_WORLD (invoking the copy callback) and then free the dup'ed communicator and deleting the attribute from COMM_WORLD (invoking the delete callback both times:)
$ mpirun -np 1 ./comm-attr
In ibcast_copy: keyval = 10
In ibcast_delete: keyval = 10
In ibcast_delete: keyval = 10
In your code, you dup the communicator in Ibcast before setting the attribute, so that the copy callback is never invoked (as there is nothing to copy). You can fix that part by setting the attribute before the dup, but then there is another problem - you call dup and free within the callbacks, which is wrong-way around; those functions are to (indirectly) invoke the callbacks, not vice-versa.