OpenCl: addition of large amount of arrays - arrays

Question

In the code bellow, each work items generate a array sum_qcos_i.
In order to add them I first make local addition using the local array sum_qcos_tmp.
Then I copy every local array in a global one dimensional matrix sum_qcos_part.
I need to add that matrix columns for my purpose, that's what does each work item before using the result.
Here is the code
__kernel __attribute__((vec_type_hint(double4))) void energy_forces( const int atom_number,
const int nvect,
__global double4 *kvect,__global double *qcos,__global double *qsin,
__global double *cst_ewald ,
__global double4 *positions,
__global double4 *forces_r,
__global double *sum_qcos_part,__global double *sum_qsin_part)
{
int i = 0 ;
int gti = 0 , ggi = 0 , lti = 0;
double kr = (double)0.0 ;
double ss = (double)0.0 , cc = (double)0.0 ;
double prod = (double)0.0 ;
double valqcos = 0. , valqsin = 0. ;
double4 zeroes_4 = (double4){ 0.0,0.0,0.0,0.0 };
double sum_qcos_i[NVECTOR_MAX] ;
double sum_qsin_i[NVECTOR_MAX] ;
#if defined NVECTOR_MAX
__local double sum_qcos_tmp[NVECTOR_MAX] ;
__local double sum_qsin_tmp[NVECTOR_MAX] ;
#endif
lti = get_local_id(0);
ggi = get_group_id(0);
for (k=0;k<nvect;k++) { /*k-vectors*/
sum_qcos_tmp[k] = .0 ;
sum_qsin_tmp[k] = .0 ;
sum_qcos_i[k] = .0 ;
sum_qsin_i[k] = .0 ;
}
double fk = (double)0.0 ;
double4 fr_i = zeroes_4 ;
double4 kvec_i = zeroes_4;
for (gti = get_global_id(0); gti < atom_number; gti += get_global_size(0))
{
pos_i = positions[gti];
for (k=0;k<nvect;k++) { /* sum over k-vectors to compute QCOS and QSIN for Ewald sum*/
prod = dot((double4)pos_i,(double4)kvect[k]);
ss = (double)sincos(-prod,&cc);
valqcos = cc ;
valqsin = ss ;
// valqcos = 1. ;
// valqsin = 1. ;
qcos[gti*NVECTOR_MAX+k] = valqcos ;
qsin[gti*NVECTOR_MAX+k] = valqsin ;
sum_qcos_i[k] = valqcos ; /* private variable */
sum_qsin_i[k] = valqsin ;
} /* end sum over k-vectors to compute QCOS and QSIN for Ewald sum*/
} // end for gti
int ii = 0 ;
for ( ii = 0;ii<get_local_size(0);ii++ )
{
if (lti == ii)
{
for (k=0;k<nvect;k++)
{ /* k-vectors */
sum_qcos_tmp[k] += sum_qcos_i[k] ; /* accumulates private data to local variable */
sum_qsin_tmp[k] += sum_qsin_i[k] ;
}
barrier(CLK_LOCAL_MEM_FENCE|CLK_GLOBAL_MEM_FENCE) ;
}
}
if (lti == 0)
{
for (k=0;k<nvect;k++) {
sum_qcos_part[ggi*NVECTOR_MAX+k] = sum_qcos_tmp[k] ; /* cp local data to global array */
sum_qsin_part[ggi*NVECTOR_MAX+k] = sum_qsin_tmp[k] ;
}
}
int iii = 0 ;
for (gti = get_global_id(0); gti < atom_number; gti += get_global_size(0))
{
fr_i = zeroes_4 ;
barrier(CLK_LOCAL_MEM_FENCE|CLK_GLOBAL_MEM_FENCE) ;
for (k=0;k<nvect;k++)
{
sum_qcos_i[k] = .0 ;
sum_qsin_i[k] = .0 ;
for (iii=0;iii<get_num_groups(0);iii++)
{
sum_qcos_i[k] += sum_qcos_part[iii*NVECTOR_MAX+k] ;
sum_qsin_i[k] += sum_qsin_part[iii*NVECTOR_MAX+k] ;
}
}
barrier(CLK_LOCAL_MEM_FENCE|CLK_GLOBAL_MEM_FENCE) ;
for (k=0;k<nvect;k++)
{
fk = ( sum_qcos_i[k]*qsin[gti*NVECTOR_MAX+k] - sum_qsin_i[k]*qcos[gti*NVECTOR_MAX+k] ) ;
fr_i += cst_ewald[k] * fk * kvect[k] ;
}
#if defined(SCALAR_KERNELS)
forces_r[gti].x = fr_i.x;
forces_r[gti].y = fr_i.y;
forces_r[gti].z = fr_i.z;
forces_r[gti].w = .0 ;
#elif defined(VECTOR_KERNELS)
forces_r[gti] = fr_i;
#endif
} // end for gti
}
This kernel doesn't work and I can't figure why.
Some hints would be very helpful here.
Thank you.

Answer 1

Adding a barrier made the trick:
for ( ii = 0;ii<get_local_size(0);ii++ )
{
if (lti == ii)
{
barrier(CLK_LOCAL_MEM_FENCE|CLK_GLOBAL_MEM_FENCE) ;
for (k=0;k<nvect;k++)
{ /* k-vectors */
sum_qcos_tmp[k] += sum_qcos_i[k] ; /* accumulates private data to local variable */
sum_qsin_tmp[k] += sum_qsin_i[k] ;
}
}
}
The final of the the vector forces_r is still wrong abut it's always the same now.

Answer 2

Actually the problem is not solved.
I define the global work size based on the number of particles so that each work item takes care of one particles.
In my calculations, I dot product each particles coordinates with a certain number of vectors.
My issue now is that the result of those dot products depends on the group size. I put the vectors in a array of double4. The global work size goes from something like 1000 to 10000, whereas the number of vectors I use for the dot product is always around 200.
I'm wondering whether there is a requirement on arrays size regarding the global work size.
for (gti = get_global_id(0); gti < atom_number; gti += get_global_size(0))
{
pos_i = positions[gti];
for (k=0;k<nvect;k++) { /* sum over k-vectors to compute QCOS and QSIN for Ewald sum*/
prod = dot((double4)pos_i,(double4)kvect[k]);
ss = (double)sincos(-prod,&cc);
valqcos = cc ;
valqsin = ss ;
qcos[gti*NVECTOR_MAX+k] = valqcos ;
qsin[gti*NVECTOR_MAX+k] = valqsin ;
sum_qcos_i[k] = valqcos ; /* private variable */
sum_qsin_i[k] = valqsin ;
} /* end sum over k-vectors to compute QCOS and QSIN for Ewald sum*/
} // end for gtiforgot
Any hints here?