MPI Radix-Sort implementation low speed-up - c

Question

I tried to implement a solution for the Radix-Sort algorithm using MPI to parallelize execution. My idea was to have each process compute the vector of the counts locally and then aggregate the results on the root process that does the sorting. With this solution I noticed a maximum speed up of just 1.5 compared to the sequential version.
I was wondering if a similar speed up was normal or if a different MPI implementation could be adopted to improve this result. I've seen around that you might think about sorting subarrays on processes but I don't quite understand how to put these sorted blocks together to get the final sorted array efficiently.
Sequential:
/**
* #brief This function allows to find the maximum in an array.
* #param arr array.
* #param n array size.
*/
int getMax(int* array, int n) {
int max = array[0];
for (int i = 1; i < n; i++)
if (array[i] > max)
max = array[i];
return max;
}
/**
* #brief The main function that sorts the array of size 'size'.
* #param array array.
* #param size array size.
* #param digit number which represent ciphers.
*/
void countingSort(int* array, int size, int digit) {
int* output= (int*) malloc(sizeof(int)*(size + 1));
int count[10]={0};
for (int i = 0; i < size; i++)
count[(array[i] / digit) % 10]++;
for (int i = 1; i < 10; i++)
count[i] += count[i - 1];
for (int i = size - 1; i >= 0; i--) {
output[count[(array[i] / digit) % 10] - 1] = array[i];
count[(array[i] / digit) % 10]--;
}
for (int i = 0; i < size; i++)
array[i] = output[i];
free(output);
}
/**
* #brief The function that takes the max and starts the sorting process.
* #param array array.
* #param size array size.
*/
void radixsort(int* array, int size) {
int max = getMax(array, size);
for (int digit = 1; max / digit > 0; digit *= 10)
countingSort(array, size, digit);
}
MPI version:
/**
* #brief This function allows to find the maximum in an array.
* #param arr array.
* #param n array size.
*/
int getMax(int* arr, int n) {
int max = arr[0];
for (int i = 1; i < n; i++)
if (arr[i] > max)
max = arr[i];
return max;
}
/**
* #brief The main function that sorts the array of size n.
* #param array array.
* #param rec_buf sub-array of each process.
* #param n array size.
* #param digit number which represent ciphers.
* #param num_process number of processes.
* #param rank rank of the current process.
* #param dim dimension of the rec_buf.
*/
void countingSort(int* array, int* rec_buf, int n, int digit, int num_process, int rank, int dim) {
// Compute local count for each processes
int i, local_count[10] = {0};
for (i = 0; i < dim; i++) {
local_count[(rec_buf[i] / digit) % 10]++;
}
// Reduce all the sub counts to root process
if (rank == 0) {
int count[10] = {0};
MPI_Reduce(local_count, count, 10, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
for (i = 1; i < 10; i++) {
count[i] += count[i - 1];
}
int* temp_array = (int*) malloc(sizeof(int) * n);
for (i = n - 1; i >= 0; i--) {
temp_array[count[(array[i] / digit) % 10] - 1] = array[i];
count[(array[i] / digit) % 10]--;
}
memcpy(array, temp_array, sizeof(int) * n);
free(temp_array);
} else {
MPI_Reduce(local_count, 0, 10, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
}
}
/**
* #brief The function that separates the array in subarray and starts the sorting process.
* #param array array.
* #param n array size.
* #param num_process number of processes.
* #param rank rank of the current process.
*/
void radix_sort(int* array, int n, int num_process, int rank) {
int rem = n%num_process; // elements remaining after division among processes
int dim, displacement;
if ( rank < rem) {
dim = n/num_process+1;
displacement = rank * dim;
}
else {
dim = n/num_process;
displacement = rank * dim + rem;
}
int* rec_buf= (int*) malloc(sizeof(int)*dim) ;
int* sendcounts = NULL;
int* displs = NULL;
if (rank == 0) {
sendcounts = malloc(sizeof(int)*num_process);
displs = malloc(sizeof(int)*num_process);
}
MPI_Gather(&dim,1,MPI_INT, sendcounts, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Gather(&displacement, 1, MPI_INT, displs, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Scatterv(array, sendcounts, displs, MPI_INT, rec_buf, dim, MPI_INT, 0, MPI_COMM_WORLD);
if (rank==0) {
free(sendcounts);
free(displs);
}
int local_max = getMax(rec_buf,dim);
int global_max;
MPI_Allreduce(&local_max,&global_max, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
for (int digit = 1; global_max / digit > 0; digit *= 10) {
countingSort(array, rec_buf, n, digit, num_process, rank, dim);
}
free(rec_buf);
}
Also I noticed that using the same kind of approach of calculating the vector of counts locally, but using OpenMP, the speed up reaches maximum value equal to 4.5. Could this result be due to the different architecture used in the two cases? (shared memory for OpenMP and distributed memory for MPI).
For completeness I also report the solution with OpenMP:
/**
* #brief This function allows to find the maximum in an array.
* #param n array size.
* #param arr[n] array.
*/
unsigned getMax(int n, unsigned arr[n]) {
unsigned mx = arr[0];
#pragma omp parallel for reduction(max:mx)
for (int i = 1; i < n; i++)
if (arr[i] > mx)
mx = arr[i];
return mx;
}
/* Source: https://gist.github.com/wanghc78/2c2b403299cab172e74c62f4397a6997
Copyright (c) 2014, Haichuan Wang All rights reserved. */
/**
* #brief The main function that sorts arr[] of size n using Radix Sort
* #param n array size.
* #param arr[n] array.
* #param threads number of threads.
*/
unsigned * radixsort(int n, unsigned arr[n], int threads) {
if (threads == 0)
threads+=1;
unsigned m = getMax(n, arr);
unsigned exp;
unsigned *output = malloc(n*sizeof(unsigned));
for (exp = 1; m / exp > 0; exp *= 10) {
int count[10] = {0}, local_count[10] = { 0 };
#pragma omp parallel firstprivate(local_count) num_threads(threads)
{
#pragma omp for schedule(static) nowait
for (int i = 0; i < n; i++)
local_count[(arr[i] / exp) % 10]++;
#pragma omp critical
for(int i = 0; i < 10; i++)
count[i] += local_count[i];
#pragma omp barrier
#pragma omp single
for (int i = 1; i < 10; i++)
count[i] += count[i - 1];
int tid = omp_get_thread_num();
for(int cur_t = threads - 1; cur_t >= 0; cur_t--) {
if(cur_t == tid) {
for(int i = 0; i < 10; i++) {
count[i] -= local_count[i];
local_count[i] = count[i];
}
}
else {
#pragma omp barrier
}
}
#pragma omp for schedule(static)
for(int i = 0; i < n; i++)
output[local_count[(arr[i] / exp) % 10]++] = arr[i];
}
unsigned* tmp = arr;
arr = output;
output = tmp;
}
free(output);
return arr;
}
/**
* #brief This function initializes all the data structures needed in the program.
* #param N array size.
* #param threads number of threads.
*/
unsigned* init_structure(int N, int threads) {
unsigned * data_vector = malloc(N*sizeof(unsigned));
#pragma omp parallel for shared(data_vector) num_threads(threads)
for (int i=0; i<N; i++)
data_vector[i]=N-i;
return data_vector;
}
Thanks everyone for the answers.