using MPI to share bulk data between processes - c

Question

My main question is how can I associate members of a custom-defined type/struct to that allocated via MPI_Win_allocate_shared(size,disp,...,&baseptr, &win). A help in either C or fortran is appreciated! Below I have included a sketch of what I wanted to do in both C and fortran.
An example in C is roughly as follows:
struct MyStruct{
int * ptr_int;
double * ptr_dble;
};
main(){
int n1,n2,n3;
struct * data;
// I am looking to use MPI to allocate a struct equivalent to the following:
// data = calloc(n3,sizeof(struct MyStruct))
// for (int i=0;i<n3;i++) {
// data[i].ptr_int = calloc(n1,sizeof(int));
// data[i].ptr_dble = calloc(n2,sizeof(double));}
int w_size,w_rank;
MPI_Init(NULL,NULL);
MPI_Comm_size(MPI_COMM_WORLD,&w_size);
MPI_Comm_rank(MPI_COMM_WORLD,&w_rank);
MPI_Win win;
MPI_Aint size;
void * baseptr;
if (w_rank==0){
size = n3*(sizeof(int)*n1 + sizeof(double)*n2);
MPI_Win_allocate_shared(size,1,MPI_INFO_NULL,MPI_COMM_WORLD,&baseptr,&win);
// Question: how to associate struct * data with win, baseptr?
// Can &win then be initialized by calling data[i].ptr_int[j] = ...?
}else{
MPI_Win_shared_query(...);
// Question: again, how to associated struct * data with win, baseptr?
}
}
Equivalently an example in fortran is follows:
type MyStruct
integer, allocatable :: ptr_int(:)
real, allocatable :: ptr_dble(:)
end type
program main
implicit none
use mpi
integer :: n1,n2,n3
type(MyStruct), allocatable :: data
integer :: w_rank, w_size, ierr
call mpi_init(ierr)
call mpi_comm_size(mpi_comm_world,w_size,ierr)
call mpi_comm_rank(mpi_comm_world,w_rank,ierr)
MPI_Win MPI_Win
MPI_Aint size
if (w_rank==0) then
size = n3*(sizeof(int)*n1 + sizeof(double)*n2)
call mpi_win_allocate_shared(size,1,MPI_INFO_NULL,MPI_COMM_WORLD,baseptr,win)
! Question: how to associate data with win, baseptr?
! Can win then be initialized by calling data(i)%ptr_int(j) = ...?
else
call mpi_win_shared_query(...);
! Question: again, how to associated type(mystruct) data with win, baseptr?
endif
end program main

Answer 1

I solved the C part of the problem, thanks to a similar question posted earlier at: MPI-3 Shared Memory for Array Struct. I still need to implement it in fortran which is more relevant to my current work.
The key aspect is that one can define a pointer to the struct in each MPI process, and use pointer arithmetics to associate the shared memory with the data structure. A complete C solution is given as follows:
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>
struct product{
int nint;
int ndble;
int * ptr_int;
double * ptr_dble;
};
int main(){
int n1,n2,nproduct;
n1 = 3;
n2 = 4;
nproduct = 2;
struct product * tmp = calloc(nproduct,sizeof(struct product));
// Initiate MPI
int world_size,world_rank;
int disp_unit;
MPI_Win win;
MPI_Aint size;
void * baseptr;
MPI_Init(NULL,NULL);
MPI_Comm_size(MPI_COMM_WORLD,&world_size);
MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
printf("Hello world from rank %d out of %d processors\n",world_rank,world_size);
if (world_rank==0){
size = (sizeof(int)*n1 + sizeof(double)*n2) * nproduct;
disp_unit = 1;
MPI_Win_allocate_shared(size,disp_unit,MPI_INFO_NULL,MPI_COMM_WORLD,&baseptr, &win);
printf("Success allocation\n");
}
else{
MPI_Win_allocate_shared(0,1,MPI_INFO_NULL,MPI_COMM_WORLD,&baseptr,&win);
MPI_Win_shared_query(win,0,&size,&disp_unit,&baseptr);
printf("Success query\n");
}
for (int i=0;i<nproduct;i++){
tmp[i].nint = n1;
tmp[i].ndble = n2;
tmp[i].ptr_int = (int*) baseptr;
tmp[i].ptr_dble = (double *) (baseptr + sizeof(int)*n1);
}
if (world_rank==0){
//MPI_Win_lock(MPI_LOCK_EXCLUSIVE,0,MPI_MODE_NOCHECK,win);
for (int i =0;i<nproduct;i++){
// initialize data stored in win via tmp
for (int j =0;j<n1;j++){
tmp[i].ptr_int[j] = j;
}
for (int j=0;j<n2;j++){
tmp[i].ptr_dble[j] = 2*(j-3);
}
}
//MPI_Win_unlock(0,win);
}
MPI_Barrier(MPI_COMM_WORLD);
// test
if (world_rank==1){
for (int j =0;j<n1;j++){
printf("%d ",tmp[1].ptr_int[j]);
}
printf("\n");
for (int j=0;j<n2;j++){
printf("%f ",tmp[1].ptr_dble[j]);
}
printf("\n");
}
MPI_Win_free(&win);
MPI_Finalize();
}