Want to gather a large 2D array in MPI on the root process
There is no problem on this code when I used small arrays but when I use number of rows 360*75.It crashes. Use 6 processes. So every slave has a chunk 60*75
program test
implicit none
include 'mpif.h'
INTEGER :: ierr, size, rank, i, j, k, l, num_angles,slice
DOUBLE PRECISION :: theta1, theta2, PI
DOUBLE PRECISION, ALLOCATABLE, DIMENSION(:,:) :: R
integer :: doublesize
integer (kind=MPI_Address_kind) :: start, extent
integer :: blocktype, resizedtype
CALL MPI_INIT(ierr)
CALL MPI_COMM_RANK(MPI_COMM_WORLD,rank,ierr)
CALL MPI_COMM_SIZE(MPI_COMM_WORLD,size,ierr)
if (rank == 0) then
if (allocated(R)) deallocate(R)
allocate(R(1:360*75,1:10))
else
if (allocated(R)) deallocate(R)
allocate(R(1:60*75,1:10))
end if
R = rank
call MPI_Type_create_subarray(2, [360*75,10], [60*75,10], [0,0], &
MPI_ORDER_FORTRAN, MPI_DOUBLE_PRECISION, &
blocktype, ierr)
start = 0
call MPI_Type_size(MPI_DOUBLE_PRECISION, doublesize, ierr)
extent = doublesize * 60*75
call MPI_Type_create_resized(blocktype, start, extent, resizedtype, ierr)
call MPI_Type_commit(resizedtype, ierr)
CALL MPI_GATHER(R, 60*75*10, MPI_DOUBLE_PRECISION, R, 1, resizedtype, 0, MPI_COMM_WORLD, ierr)
CALL MPI_FINALIZE(ierr)
end program
The output on small arrays look like
0 0 0 0
0 0 0 0
1 1 1 1
1 1 1 1
2 2 2 2
2 2 2 2
....
Can the problem be with memory allocation on root process?
EDIT
change following in the code based on comment. Still have an error
CALL MPI_Type_contiguous(60*75*10, MPI_DOUBLE_PRECISION, new_type,ierr)
call MPI_TYPE_COMMIT(new_type,ierr)
if (rank == 0) then
CALL MPI_GATHER(MPI_IN_PLACE, 60*75*10, new_type, R, 1, new_type, 0, MPI_COMM_WORLD, ierr)
else
CALL MPI_GATHER(R, 60*75*10, new_type, R, 1, new_type, 0, MPI_COMM_WORLD, ierr)
end if
CALL MPI_FINALIZE(ierr)
Related
Intro:
I am trying to write a large set of data to a single file using MPI IO using the code below.
The problem i encounter is, that i get an integer overflow (variable disp) and thus the MPI IO does not work properly.
The reason for this is, i think, the declaration of the variable disp (integer (kind=MPI_OFFSET_KIND) :: disp = 0) in the subroutine write_to_file(...).
Since for the process with the highest rank disp overflows.
Question:
Can I somehow define disp as kind=MPI_OFFSET_KIND but with higher range? I did not find a solution for that, except writing to multiple files, but I would prefer writing into a single file.
Some context:
The code is just a part of an code, which i use to output (and read; but I cut that part from the code example to make it easier) scalar (ivar = 1), vector(ivar=3) or tensor(ivar=3,6 or 9) values into binary files. The size of the 3D grid is defined by imax, jmax and kmax, where kmax is decomposed by Px processes into Mk. Lately the 3D grid grew to a size where i encountered the described problem.
Code Example: MPI_IO_LargeFile.f90
"""
PROGRAM MPI_IO_LargeFile
use MPI
implicit none
integer rank, ierr, Px
integer i, j, k, cnt
integer imax, jmax, kmax, Mk
integer status(MPI_STATUS_SIZE)
integer ivars;
real*4, dimension(:,:,:,:), allocatable :: outarr, dataarr
call MPI_Init(ierr)
call MPI_Comm_size(MPI_COMM_WORLD, Px, ierr)
call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierr)
imax = 256
jmax = 512
kmax = 1024
Mk = kmax / Px
if (rank < 1) print *, 'Preparing dataarr'
ivars = 6
allocate(dataarr(ivars,imax,jmax,Mk))
call RANDOM_NUMBER(dataarr)
! Output of Small File
if (rank < 1) print *, 'Output of SmallFile.txt'
ivars = 3
allocate(outarr(ivars,imax,jmax,Mk))
outarr(:,:,:,:) = dataarr(1:3,:,:,:)
call write_to_file(rank, 'SmallFile.txt', outarr)
deallocate(outarr)
! Output of Large File
if (rank < 1) print *, 'Output of LargeFile.txt'
ivars = 6
allocate(outarr(ivars,imax,jmax,Mk))
outarr(:,:,:,:) = dataarr(1:6,:,:,:)
call write_to_file(rank, 'LargeFile.txt', outarr)
deallocate(outarr)
deallocate(dataarr)
call MPI_Finalize(ierr)
CONTAINS
subroutine write_to_file(myrank, filename, arr)
implicit none
integer, intent(in) :: myrank
integer :: ierr, file, varsize
character(len=*), intent(in):: filename
real*4, dimension(:,:,:,:), allocatable, intent(inout) :: arr
**integer (kind=MPI_OFFSET_KIND) :: disp = 0**
varsize = size(arr)
disp = myrank * varsize * 4
**write(*,*) rank, varsize, disp**
call MPI_File_open(MPI_COMM_WORLD, filename, &
& MPI_MODE_WRONLY + MPI_MODE_CREATE, &
& MPI_INFO_NULL, file, ierr )
call MPI_File_set_view(file, disp, MPI_REAL4, &
& MPI_REAL4, 'native', MPI_INFO_NULL, ierr)
call MPI_File_write(file, arr, varsize, &
& MPI_REAL4, MPI_STATUS_IGNORE, ierr)
call MPI_FILE_CLOSE(file, ierr)
end subroutine write_to_file
END PROGRAM MPI_IO_LargeFile
"""
Output of Code: MPI_IO_LargeFile.f90
mpif90 MPI_IO_LargeFile.f90 -o MPI_IO_LargeFile
mpirun -np 4 MPI_IO_LargeFile
Preparing dataarr
Output of SmallFile.txt
2 100663296 805306368
1 100663296 402653184
3 100663296 1207959552
0 100663296 0
Output of LargeFile.txt
1 201326592 805306368
0 201326592 0
2 201326592 1610612736
3 201326592 -1879048192
mca_fbtl_posix_pwritev: error in writev:Invalid argument
mca_fbtl_posix_pwritev: error in writev:Invalid argument
The Problem is that the multiplication in
disp = myrank * varsize * 4
overflowed, since each variable was declared as integer.
One solution provided by #Gilles (in the comments of the question) was simply to change this line to
disp = myrank * size(arr, kind=MPI_OFFSET_KIND) * 4
Using size(arr, kind=MPI_OFFSET_KIND) converts the solution into an integer of kind=MPI_OFFSET_KIND, which solves the overflow problem.
Thank you for your help.
I have to face this situation:
given N number of MPI nodes
and
given a 2D real array of [N_ROWS,N_COLS] dimension
I have to partition it into in order to speed up calculus, giving to each node
a subsection of 2D array and taking advantage of number of nodes.
Following Fortran way to store data in memory, arrays are indexed using the most rapidly changing variable first, every [:,i]-column of the array is "logically" separated from the others.
I have looked around to very illuminating questions like this one Sending 2D arrays in Fortran with MPI_Gather
And I have reached the idea of using mpi_scatterv and mpi_gatherv, BUT I'm stuck against the fact that, since in the problem constraints, there is no possibility to guarantee that for each MPI node it is given the same amount of data, or, in pseudo code:
#Number_of_MPI_nodes != N_ROWS*N_COLS
I was looking to use vectors, since each "column" has is own "independent" series of data, when I say "independent" I mean that I have to do some manipulation on the data belonging the same column, without affecting other columns.
Obviously, since the inequality given, some MPI nodes will have a different number of "columns" to analyze.
After doing some math, I need to gather back the data, using mpi_gatherv
I will update the question with a working example in a few hours!
Thanks a lot to everybody !
CODE:
program main
use mpi
implicit none
integer:: N_COLS=100, N_ROWS=200
integer:: i, j
integer:: ID_mpi, COM_mpi, ERROR_mpi
integer:: master = 0, SIZE_mpi=0
integer:: to_each_cpu=0, to_each_cpu_oddment=0
integer:: sub_matrix_size=0
integer:: nans=0, infs=0, array_split =0, my_type=0
integer ,dimension(:), allocatable :: elem_to_each_cpu
integer ,dimension(:), allocatable :: displacements
integer,parameter:: seed = 12345
character*160:: message
real :: tot_sum = 0.0
real ,dimension(:,:), allocatable:: Data_Matrix
real ,dimension(:,:), allocatable:: sub_split_Data_Matrix
call srand(seed)
call MPI_INIT(ERROR_mpi)
COM_mpi = MPI_COMM_WORLD
call MPI_COMM_RANK(COM_mpi,ID_mpi,ERROR_mpi)
call MPI_COMM_SIZE(COM_mpi,SIZE_mpi,ERROR_mpi)
!! allocation Data_Matrix
i = 1; j = 1
if (ID_mpi .eq. master) then
i = N_ROWS; j = N_COLS
end if
allocate(Data_Matrix(i, j))
do j = 1, N_COLS
do i = 1, N_ROWS
Data_Matrix(i, j) = rand()
tot_sum = tot_sum + Data_Matrix(i, j)
enddo
enddo
write(message,*) "N_COLS:",N_COLS, "N_ROWS:", N_ROWS, " TOTAL_SUM:", tot_sum
write(*,*) message
!! SINCE THERE ARE NO RESTRICTIONS ON MPI NUMBER OR CPUS OR
!! SIZE OR Data_Matrix I NEED TO DO THIS
to_each_cpu =N_COLS / SIZE_mpi
to_each_cpu_oddment = N_COLS -( to_each_cpu * SIZE_mpi )
allocate(elem_to_each_cpu(SIZE_mpi))
elem_to_each_cpu = to_each_cpu
allocate(displacements(SIZE_mpi))
displacements = 0
!! I CHOOSE TO SPLIT THE DATA IN THIS WAY
if (ID_mpi .eq. master) then
write(message,*) "N_COLS:",N_COLS, "mpisize:", SIZE_mpi, "to_each_cpu\oddment:", to_each_cpu, " \ ", to_each_cpu_oddment
write(*,*) message
j=1
do i = 1 , to_each_cpu_oddment
elem_to_each_cpu(j) = elem_to_each_cpu(j) + 1
j = j + 1
if(j .gt. SIZE_mpi) j = 1
enddo
do j = 2, SIZE_mpi
displacements(j) = elem_to_each_cpu(j-1) + displacements(j-1)
enddo
do i = 1 , SIZE_mpi
write(message,*)i, " to_each_cpu:", &
elem_to_each_cpu(i), " sub_split_buff_displ:",displacements(i), "=",elem_to_each_cpu(i)+displacements(i)
write(*,*) message
enddo
end if
call MPI_BCAST(elem_to_each_cpu, SIZE_mpi, MPI_INT, 0, COM_mpi, ERROR_mpi)
call MPI_BCAST(displacements, SIZE_mpi, MPI_INT, 0, COM_mpi, ERROR_mpi)
allocate( sub_split_Data_Matrix(N_ROWS,elem_to_each_cpu(ID_mpi+1)) )
call MPI_TYPE_VECTOR(N_COLS,N_ROWS,N_ROWS,MPI_FLOAT,my_type,ERROR_mpi)
call MPI_TYPE_COMMIT(my_type, ERROR_mpi)
sub_split_Data_Matrix=0
sub_matrix_size = N_ROWS*elem_to_each_cpu(ID_mpi+1)
call MPI_scatterv( Data_Matrix,elem_to_each_cpu,displacements,&
MPI_FLOAT, sub_split_Data_Matrix, sub_matrix_size ,MPI_FLOAT, &
0, COM_mpi, ERROR_mpi)
!!! DOING SOME MATH ON SCATTERED MATRIX
call MPI_gatherv(&
sub_split_Data_Matrix, sub_matrix_size,MPI_FLOAT ,&
Data_Matrix, elem_to_each_cpu, displacements, &
MPI_FLOAT, 0, COM_mpi, ERROR_mpi)
!!! DOING SOME MATH ON GATHERED MATRIX
tot_sum = 0.0
do j = 1, N_COLS
do i = 1, N_ROWS
tot_sum = tot_sum + Data_Matrix(i, j)
enddo
enddo
write(message,*) "N_COLS:",N_COLS, "N_ROWS:", N_ROWS, " TOTAL_SUM:", tot_sum
write(*,*) message
deallocate(Data_Matrix)
if (ID_mpi .eq. master) then
deallocate(elem_to_each_cpu )
deallocate(displacements )
endif
deallocate(sub_split_Data_Matrix)
end
RESULT:
Error occurred in MPI_Gahterv
on communicator MPI_COMM_WORLD
Invalid memory reference
QUESTION:
Can you help me find the error ?
Or better, can you help me in showing if the approach
that I used was appropriate ?
Thanks a lot!
I had a look at your code and did some changes to fix it:
Unimportant: a few stylistic / cosmetic elements here and there to (from my standpoint and that is arguable) improve readability. Sorry if you don't like it.
There is no need for the process 0 to be the only one computing the lengths and displacements for the MPI_Scatterv()/MPI_Gatherv() calls. All processes should compute them since they all have the necessary data to do so. Moreover, it spares you two MPI_Bcast() which is good.
The lengths were strangely computed. I suspect it was wrong but I'm not sure since it was so convoluted I just rewrote it.
The main issue was a mix-up between the vector type and the scalar type: your lengths and displacements were computed for your vector type, but you were calling MPI_Scatterv()/MPI_Gatherv() with the scalar type. Moreover, for Fortran, this scalar type is MPI_REAL, not MPI_FLOAT. In the code I posted here-below, I computed lengths and displacements for MPI_REAL, but if you prefer, you can divide them all by N_ROWS and use the result of MPI_Type_contiguous( N_ROWS, MPI_REAL, my_type ) instead of MPI_REAL in the scatter/gather, and get the same result.
Here is the modified code:
program main
use mpi
implicit none
integer, parameter :: N_COLS=100, N_ROWS=200, master=0
integer :: i, j
integer :: ID_mpi,SIZE_mpi, COM_mpi, ERROR_mpi, my_type
integer :: to_each_cpu, to_each_cpu_oddment, sub_matrix_size
integer, allocatable :: elem_to_each_cpu(:), displacements(:)
real :: tot_sum = 0.0
real, allocatable :: Data_Matrix(:,:), sub_split_Data_Matrix(:,:)
call MPI_Init( ERROR_mpi )
COM_mpi = MPI_COMM_WORLD
call MPI_Comm_rank( COM_mpi, ID_mpi, ERROR_mpi )
call MPI_Comm_size( COM_mpi, SIZE_mpi, ERROR_mpi )
!! allocation Data_Matrix
if ( ID_mpi == master ) then
allocate( Data_Matrix( N_ROWS, N_COLS ) )
call random_number( Data_Matrix )
do j = 1, N_COLS
do i = 1, N_ROWS
tot_sum = tot_sum + Data_Matrix(i, j)
enddo
enddo
print *, "N_COLS:", N_COLS, "N_ROWS:", N_ROWS, " TOTAL_SUM:", tot_sum
end if
!! SINCE THERE ARE NO RESTRICTIONS ON MPI NUMBER OR CPUS OR
!! SIZE OR Data_Matrix I NEED TO DO THIS
to_each_cpu = N_COLS / SIZE_mpi
to_each_cpu_oddment = N_COLS - ( to_each_cpu * SIZE_mpi )
allocate( elem_to_each_cpu(SIZE_mpi) )
elem_to_each_cpu = to_each_cpu * N_ROWS
allocate( displacements(SIZE_mpi) )
displacements = 0
!! I CHOOSE TO SPLIT THE DATA IN THIS WAY
if ( ID_mpi == master ) then
print *, "N_COLS:", N_COLS, "mpisize:", SIZE_mpi, "to_each_cpu\oddment:", to_each_cpu, " \ ", to_each_cpu_oddment
end if
do i = 1, to_each_cpu_oddment
elem_to_each_cpu(i) = elem_to_each_cpu(i) + N_ROWS
enddo
do i = 1, SIZE_mpi-1
displacements(i+1) = displacements(i) + elem_to_each_cpu(i)
enddo
if ( ID_mpi == master ) then
do i = 1, SIZE_mpi
print *, i, " to_each_cpu:", &
elem_to_each_cpu(i), " sub_split_buff_displ:", displacements(i), &
"=", elem_to_each_cpu(i) + displacements(i)
enddo
end if
allocate( sub_split_Data_Matrix(N_ROWS, elem_to_each_cpu(ID_mpi+1)/N_ROWS) )
sub_split_Data_Matrix = 0
sub_matrix_size = elem_to_each_cpu(ID_mpi+1)
call MPI_scatterv( Data_Matrix, elem_to_each_cpu ,displacements, MPI_REAL, &
sub_split_Data_Matrix, sub_matrix_size, MPI_REAL, &
master, COM_mpi, ERROR_mpi )
!!! DOING SOME MATH ON SCATTERED MATRIX
call MPI_gatherv( sub_split_Data_Matrix, sub_matrix_size, MPI_REAL, &
Data_Matrix, elem_to_each_cpu, displacements, MPI_REAL, &
master, COM_mpi, ERROR_mpi )
!!! DOING SOME MATH ON GATHERED MATRIX
if ( ID_mpi == master ) then
tot_sum = 0.0
do j = 1, N_COLS
do i = 1, N_ROWS
tot_sum = tot_sum + Data_Matrix(i, j)
enddo
enddo
print *, "N_COLS:", N_COLS, "N_ROWS:", N_ROWS, " TOTAL_SUM:", tot_sum
deallocate( Data_Matrix )
endif
deallocate( elem_to_each_cpu )
deallocate( displacements )
deallocate( sub_split_Data_Matrix )
end program main
With these modifications, the code works as expected:
$ mpif90 scat_gath2.f90
$ mpirun -n 3 ./a.out
N_COLS: 100 N_ROWS: 200 TOTAL_SUM: 10004.4443
N_COLS: 100 mpisize: 3 to_each_cpu\oddment: 33 \ 1
1 to_each_cpu: 6800 sub_split_buff_displ: 0 = 6800
2 to_each_cpu: 6600 sub_split_buff_displ: 6800 = 13400
3 to_each_cpu: 6600 sub_split_buff_displ: 13400 = 20000
N_COLS: 100 N_ROWS: 200 TOTAL_SUM: 10004.4443
I am parallelising a fortran code which works with no problem in a no-MPI version. Below is an excerpt of the code.
Every processor does the following:
For a certain number of particles it evolves certain quantities in the loop "do 203"; in a given interval which is divided in Nint subintervals (j=1,Nint), every processor produces an element of the vectors Nx1(j), Nx2(j).
Then, the vectors Nx1(j), Nx2(j) are sent to the root (mype =0) which in every subinterval (j=1,Nint) sums all the contributions for every processor: Nx1(j) from processor 1 + Nx1(j) from processor 2.... The root sums for every value of j (every subinterval), and produces Nx5(j), Nx6(j).
Another issue is that if I deallocate the variables the code remains in standby after the end of the calculation without completing the execution; but I don't know if this is related to the MPI_Allreduce issue.
include "mpif.h"
...
integer*4 ....
...
real*8
...
call MPI_INIT(mpierr)
call MPI_COMM_SIZE(MPI_COMM_WORLD, npe, mpierr)
call MPI_COMM_RANK(MPI_COMM_WORLD, mype, mpierr)
! Allocate variables
allocate(Nx1(Nint),Nx5(Nint))
...
! Parameters
...
call MPI_Barrier (MPI_COMM_WORLD, mpierr)
! Loop on particles
do 100 npartj=1,npart_local
call init_random_seed()
call random_number (rand)
...
Initial condition
...
do 203 i=1,1000000 ! loop for time evolution of single particle
if(ufinp.gt.p1.and.ufinp.le.p2)then
do j=1,Nint ! spatial position at any momentum
ls(j) = lb+(j-1)*Delta/Nint !Left side of sub-interval across shock
rs(j) = ls(j)+Delta/Nint
if(y(1).gt.ls(j).and.y(1).lt.rs(j))then !position-ordered
Nx1(j)=Nx1(j)+1
endif
enddo
endif
if(ufinp.gt.p2.and.ufinp.le.p3)then
do j=1,Nint ! spatial position at any momentum
ls(j) = lb+(j-1)*Delta/Nint !Left side of sub-interval across shock
rs(j) = ls(j)+Delta/Nint
if(y(1).gt.ls(j).and.y(1).lt.rs(j))then !position-ordered
Nx2(j)=Nx2(j)+1
endif
enddo
endif
203 continue
100 continue
call MPI_Barrier (MPI_COMM_WORLD, mpierr)
print*,"To be summed"
do j=1,Nint
call MPI_ALLREDUCE (Nx1(j),Nx5(j),npe,mpi_integer,mpi_sum,
& MPI_COMM_WORLD, mpierr)
call MPI_ALLREDUCE (Nx2(j),Nx6(j),npe,mpi_integer,mpi_sum,
& MPI_COMM_WORLD, mpierr)
enddo
if(mype.eq.0)then
do j=1,Nint
write(1,107)ls(j),Nx5(j),Nx6(j)
enddo
107 format(3(F13.2,2X,i6,2X,i6))
endif
call MPI_Barrier (MPI_COMM_WORLD, mpierr)
print*,"Now deallocate"
! deallocate(Nx1) !inserting the de-allocate
! deallocate(Nx2)
close(1)
call MPI_Finalize(mpierr)
end
! Subroutines
...
Then, the vectors Nx1(j), Nx2(j) are sent to the root (mype =0) which in every subinterval (j=1,Nint) sums all the contributions for every processor: Nx1(j) from processor 1 + Nx1(j) from processor 2.... The root sums for every value of j (every subinterval), and produces Nx5(j), Nx6(j).
This is not what an allreduce does. Reduction means the summation is done in parallel across all processes. allreduce means all processes will get the result of the summing.
Your MPI_Allreduces:
call MPI_ALLREDUCE (Nx1(j),Nx5(j),npe,mpi_integer,mpi_sum, &
& MPI_COMM_WORLD, mpierr)
call MPI_ALLREDUCE (Nx2(j),Nx6(j),npe,mpi_integer,mpi_sum, &
& MPI_COMM_WORLD, mpierr)
Actually look like the count should be 1 here. This is because count just states how many elements you are to receive from each process, not how many there will be in total.
However, you actually do not need that loop, because the allreduce luckily is capable of handling multiple elements all at once. Thus, I believe instead of the loop with your allreduces, you actually want something like:
integer :: Nx1(nint)
integer :: Nx2(nint)
integer :: Nx5(nint)
integer :: Nx6(nint)
call MPI_ALLREDUCE (Nx1, Nx5, nint, mpi_integer, mpi_sum, &
& MPI_COMM_WORLD, mpierr)
call MPI_ALLREDUCE (Nx2, Nx6, nint, mpi_integer, mpi_sum, &
& MPI_COMM_WORLD, mpierr)
Nx5 will contain the sum of Nx1 across all partitions, and Nx6 the sum across Nx2.
The information in your question is a little bit lacking, so I am not quite sure, if this is what you are looking for.
I have program that i working on parallelize using MPI. I have attached the test code, which breaks down the array into multiple segement along a given number of processor and then i am using broadcast to combine the results from all of the processes and give it out to all of the processes. The problem occurs when the distribution is not same along all of the processors and causes broadcast to fail. I have attached the following code that works with 6 processes, but not 8 processes.
program main
include 'mpif.h'
integer i, rank, nprocs, count, start, stop, nloops
integer err, n1
REAL srt, sp
REAL b(1,1:10242)
REAL c(1,1:10242)
integer, Allocatable :: jjsta(:), jjlen(:)
allocate (jjsta(0:nprocs-1),jjlen(0:nprocs-1))
call MPI_Init(ierr)
call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierr)
call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr)
n1 = 10242
call para_range(1,n1,nprocs,rank,start,stop)
jjsta(rank) = start
jjlen(rank) = stop -start +1
srt = MPI_WTIME()
do i=start,stop
c(1,i) = i
enddo
sp =MPI_WTIME()
print *,"Process ",rank," performed ", jjlen(rank), &
" iterations of the loop."
! CALL MPI_ALLGATHERV(c(1,jjsta(rank)), jjlen(rank), MPI_REAL,&
! b,jjlen(rank),jjsta(rank)+jjlen(rank)+1,MPI_REAL,MPI_COMM_WORLD,ierr)
CALL MPI_BCAST(c(1,jjsta(rank)), jjlen(rank), MPI_REAL,&
rank, MPI_COMM_WORLD, ierr)
call MPI_Finalize(ierr)
deallocate(jjsta,jjlen)
end program main
subroutine para_range(n1, n2, nprocs, irank, ista, iend)
integer(4) :: n1 ! Lowest value of iteration variable
integer(4) :: n2 ! Highest value of iteration variable
integer(4) :: nprocs ! # cores
integer(4) :: irank ! Iproc (rank)
integer(4) :: ista ! Start of iterations for rank iproc
integer(4) :: iend ! End of iterations for rank iproc
iwork1 = ( n2 -n1 + 1 ) / nprocs
! print *, iwork1
iwork2 = MOD(n2 -n1 + 1, nprocs)
! print *, iwork2
ista = irank* iwork1 + n1 + MIN(irank, iwork2)
iend = ista + iwork1 -1
if (iwork2 > irank) iend = iend + 1
return
end subroutine para_range
and when i run it with 8 processors with MPI_Bcast commented out and this is what it prints out this and it works
Process 2 performed 1280 iterations of the loop.
Process 1 performed 1281 iterations of the loop.
Process 4 performed 1280 iterations of the loop.
Process 5 performed 1280 iterations of the loop.
Process 3 performed 1280 iterations of the loop.
Process 6 performed 1280 iterations of the loop.
Process 0 performed 1281 iterations of the loop.
Process 7 performed 1280 iterations of the loop.
But when i try to combine all of the results and broadcast that this is the error i get
Fatal error in MPI_Bcast:
Message truncated, error stack:
MPI_Bcast(1128)...................: MPI_Bcast(buf=0x2af182d8d028, count=1280, MPI_REAL, root=4, MPI_COMM_WORLD) failed
knomial_2level_Bcast(1268)........:
MPIDI_CH3U_Receive_data_found(259): Message from rank 0 and tag 2 truncated; 5124 bytes received but buffer size is 5120
Process 4 performed 1280 iterations of the loop.
I also tried the MPI_Allgatherv to combine the result in array b and i get the following error:
Fatal error in MPI_Allgatherv:
Message truncated, error stack:
MPI_Allgatherv(1050)................: MPI_Allgatherv(sbuf=0x635df8, scount=1280, MPI_REAL, rbuf=0x62bde8, rcounts=0x2b3ff297b244, displs=0x7fff3ebc1fb0, MPI_REAL, MPI_COMM_WORLD) failed
MPIR_Allgatherv(243)................:
MPIC_Sendrecv(117)..................:
MPIDI_CH3U_Request_unpack_uebuf(631): Message truncated; 5124 bytes received but buffer size is 5120
The problem when combining the arrays to me seems like that it is not updating the length of the array being sent. I can't seem to figure out how to solve the problem any help would be appreciated. Thank you.
I'm writing to a file as follows. The order does not necessarily matter (though it would be nice if I could get it ordered by K, as would be inherently in serial code)
CALL MPI_BARRIER(MPI_COMM_WORLD, IERR)
OPEN(EIGENVALUES_UP_IO, FILE=EIGENVALUES_UP_PATH, ACCESS='APPEND')
WRITE(EIGENVALUES_UP_IO, *) K * 0.0001_DP * PI, (EIGENVALUES(J), J = 1, ATOM_COUNT)
CLOSE(EIGENVALUES_UP_IO)
I'm aware this is likely to be the worst option.
I've taken a look at MPI_FILE_WRITE_AT etc. but I'm not sure they (directly) take data in the form that I have?
The file must be in the same format as this, which comes out as a line per K, with ATOM_COUNT + 1 columns. The values are REAL(8)
I've hunted over and over, and can't find any simple references on achieving this. Any help? :)
Similar code in C (assuming it's basically the same as FORTRAN) is just as useful
Thanks!
So determining the right IO strategy depends on a lot of factors. If you are just sending back a handful of eigenvalues, and you're stuck writing out ASCII, you might be best off just sending all the data back to process 0 to write. This is not normally a winning strategy, as it obviously doesn't scale; but if the amount of data is very small, it could well be better than the contention involved in trying to write out to a shared file (which is, again, harder with ASCII).
Some code is below which will schlep the amount of data back to proc 0, assuming everyone has the same amount of data.
Another approach would just be to have everyone write out their own ks and eigenvalues, and then as a postprocessing step once the program is finished, cat them all together. That avoids the MPI step, and (with the right filesystem) can scale up quite a ways, and is easy; whether that's better is fairly easily testable, and will depend on the amount of data, number of processors, and underlying file system.
program testio
use mpi
implicit none
integer, parameter :: atom_count = 5
integer, parameter :: kpertask = 2
integer, parameter :: fileunit = 7
integer, parameter :: io_master = 0
double precision, parameter :: pi = 3.14159
integer :: totalk
integer :: ierr
integer :: rank, nprocs
integer :: handle
integer(kind=MPI_OFFSET_KIND) :: offset
integer :: filetype
integer :: j,k
double precision, dimension(atom_count, kpertask) :: eigenvalues
double precision, dimension(kpertask) :: ks
double precision, allocatable, dimension(:,:):: alleigenvals
double precision, allocatable, dimension(:) :: allks
call MPI_INIT(ierr)
call MPI_COMM_SIZE(MPI_COMM_WORLD, nprocs, ierr)
call MPI_COMM_RANK(MPI_COMM_WORLD, rank, ierr)
totalk = nprocs*kpertask
!! setup test data
do k=1,kpertask
ks(k) = (rank*kpertask+k)*1.d-4*PI
do j=1,atom_count
eigenvalues(j,k) = rank*100+j
enddo
enddo
!! Everyone sends proc 0 their data
if (rank == 0) then
allocate(allks(totalk))
allocate(alleigenvals(atom_count, totalk))
endif
call MPI_GATHER(ks, kpertask, MPI_DOUBLE_PRECISION, &
allks, kpertask, MPI_DOUBLE_PRECISION, &
io_master, MPI_COMM_WORLD, ierr)
call MPI_GATHER(eigenvalues, kpertask*atom_count, MPI_DOUBLE_PRECISION, &
alleigenvals, kpertask*atom_count, MPI_DOUBLE_PRECISION, &
io_master, MPI_COMM_WORLD, ierr)
if (rank == 0) then
open(unit=fileunit, file='output.txt')
do k=1,totalk
WRITE(fileunit, *) allks(k), (alleigenvals(j,k), j = 1, atom_count)
enddo
close(unit=fileunit)
deallocate(allks)
deallocate(alleigenvals)
endif
call MPI_FINALIZE(ierr)
end program testio
If you can determine how long each rank's write will be, you can call MPI_SCAN(size, offset, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD) to compute the offset that each rank should start at, and then they can all call MPI_FILE_WRITE_AT. This is probably more suitable if you have a lot of data, and you are confident that your MPI implementation does the write efficiently (doesn't serialize internally, or the like).