Multithreading going throught an array of struct in C - c

Question

I have wrote a program that receives as input a text file and return as output another text file.
The text file is created with a script(python) inside a 3D app (Blender) , and it contains a list of vertex that are part of a square mesh. The program receives that data, stores it in a struct, and return a list of vertex that forms a smaller square. Than, the 3D app, again with a script, reads this vertices and separate them from the original mesh. Doing this several times, the original mesh will be divided in many squares of the same area.
BY NOW, IT WORKS ;)
But is terribly low.. When doing it on 200k vertices it takes a while, but running it on 1kk vertices it takes ages
Here the source:
#include <stdio.h>
#include <stdlib.h>
#include <malloc.h>
typedef struct{
int index;
float x,y,z;
} vertex;
vertex *find_vertex(vertex *list, int len)
{
int i;
vertex lower,highter;
lower=list[0];
highter=list[1];
//find the lower lefter and the upper righter vertices
for(i=0;i<len;i++)
{
if ((list[i].x<=lower.x) && (list[i].y<=lower.y))
lower=list[i];
if ((list[i].x>=highter.x) && (list[i].y>=highter.y))
highter=list[i];
}
vertex *ret;//create a pointer for returning 2 structs
ret=(vertex*)malloc(sizeof(vertex)*2);
if (ret==NULL)
{
printf("Can't allocate the memory");
return 0;
}
ret[0]=lower;
ret[1]=highter;
return ret;
}
vertex *square_list_of_vertex(vertex *list,int len,vertex start, float size)
{
int i=0,a=0;
unsigned int *num;
num=(int*)malloc(sizeof(unsigned int)*len);
if (num==NULL)
{
printf("Can't allocate the memory");
return 0;
}
//controlls if point is in the right position and adds its index in the main list in another array
for(i=0;i<len;i++)
{
if ((list[i].x-start.x)<size && (list[i].y-start.y<size))
{
if (list[i].y-start.y>-size/100)//it was adding also wrong vertices. This line is to solve a bug
{
num[a]=i;
a++;//len of the return list
}
}
}
//create the list with the right vertices
vertex *retlist;
retlist=(vertex*)malloc(sizeof(vertex)*(a+1));
if (retlist==NULL)
{
printf("Can't allocate the memory");
return 0;
}
//the first index is used only as an info container
vertex infos;
infos.index=a+1;
retlist[0]=infos;
//set the value for the return pointer
for(i=1;i<=a;i++)
{
retlist[i]=list[num[i-1]];
}
return retlist;
}
//the function that pass the data to python
void return_funct_1(vertex lower,vertex highter)
{
FILE* ret;
ret=fopen("max_min.txt","w");
if (ret==NULL)
{
printf("Error opening the file\n");
return;
}
fprintf(ret,"%i\n",lower.index);
fprintf(ret,"%i\n",highter.index);
fclose(ret);
}
//the function that pass the data to python
void return_funct_2(vertex *squarelist)
{
FILE* ret;
int i,len;
ret=fopen("square_list.txt","w");
if (ret==NULL)
{
printf("Error opening the file\n");
return;
}
len=squarelist[0].index;
for(i=1;i<len;i++)
{
//return all the informations
//fprintf(ret,"%i %f %f %f\n",squarelist[i].index,squarelist[i].x,squarelist[i].y,squarelist[i].z);
//just return the index(it's enought for the python script)
fprintf(ret,"%i\n",squarelist[i].index);
}
fclose(ret);
}
//argv:
//function[1/2] number_of_vert(int) size_of_square(int) v_index(int) v_xcoord(float) v_ycoord(float) v_zcoord(float)...
//example of file: 2 4 2 0 1 2 3 1 1 2 3 2 1 2 3 3 1 2 3 4 1 2 3 //function 1, number of ver=4, size=2 and then the 4 vertex with their coords
int main(int argc, char *argv[])
{
if(argc==1)
{
printf("%s need a path to a vectorlist file\n",argv[0]);
return 0;
}
FILE* input;
input=fopen(argv[1],"r");
if (input==NULL)
{
printf("Error opening the file\n");
return(0);
}
int func=0,i=0,a=0,u=0;
char read;
char* argument;
argument=(char*)malloc(sizeof(char)*50);//yeah, i know, i should use list instead of an array, but when i first coded this i was quite in hurry (and i'm still learning )
//get the first paramater in the file
argument[0]=fgetc(input);
argument[1]='\0';
func=atoi(argument);
//skipp the space
read=fgetc(input);
//get the number of vertices;
i=0;
do {
read=fgetc(input);
argument[i]=read;
i++;
}while(read!=' ' && !feof(input) );
//set the end of the string
argument[i]='\0';
//set the variable to the correct integer value;
int vnumber=atoi(argument);
i=0;
do {
read=fgetc(input);
argument[i]=read;
i++;
} while(read!=' ' && !feof(input));
//set the end of the string
argument[i]='\0';
float sqsize=atof(argument);
vertex *list;
//allocate memory in the array to fit the number of vertex needed
list=(vertex*)malloc(sizeof(vertex)*vnumber);
//control if the memory get allocated
if (list==NULL)
{
printf("Can't allocate the memory");
return 0;
}
//do the cycle for each vertex
for(u=0;u<vnumber;u++)
{
//read the number and assign it to the proper value of the vertex
for(a=0;a<4;a++)
{
i=0;
do
{
read=fgetc(input);
argument[i]=read;
i++;
} while(read!=' ' && !feof(input));
argument[i]='\0';
if(a==0)
list[u].index=atoi(argument);
if(a==1)
list[u].x=atof(argument);
if(a==2)
list[u].y=atof(argument);
if(a==3)
list[u].z=atof(argument);
}
}
//close the file
fclose(input);
if (func==1)
{
//find the lowest vertex and the higtest vertex
vertex lower;
vertex highter;
vertex *lohi;
lohi=(vertex*)find_vertex(list, vnumber);
lower=lohi[0];
highter=lohi[1];
free(lohi);
return_funct_1(lower,highter);//the function that return the data to python
return 1;
}
if(func==2)
{
//find the list to return
vertex *lohi;
lohi=(vertex*)find_vertex(list, vnumber);
vertex lower;
lower=lohi[0];
free(lohi);
return_funct_2(square_list_of_vertex(list,vnumber, lower, sqsize));//the function that return the data to python
return 1;
}
printf("Function argument was wrong: nothing was done\n");
}
I would really appreciate any help for making this multithreaded.. It takes ages to work on really big data(today i've tried with a 50mb text file, and after 20 mins it had run only 30 times(on the 26000 i needed)), and since quite all pc that will use this will have at least 4 cores, i would really like to get it multithreaded!
Thanks in advice! :)
Ps: if you need, i can post the python script code too, but it's quite full of calls to the internal api of the program, so i don't really know if it would be usefull.

Answer 1

I am not going to work specifically through your code but your algorithm may be able to apply Map and Reduce.
This is an article of how you can use it in C:
http://pages.cs.wisc.edu/~gibson/mapReduceTutorial.html

Answer 2

When I profile your code running over a sample dataset of 2 million random vertexes, with the source file preloaded into the page cache the bottleneck is the conversion of strings to floats (it still runs in only 5 seconds, though - so it's not that slow).
It is possible to multithread the conversion of strings to floats, and with careful coding you will get some gains this way. However, you will get much more bang for your buck if instead you change the Python code to write the floating point numbers in a binary format that can be directly loaded by the C code (with fread()). I believe you can use struct.pack on the Python side to achieve this.
The processing part of your code can certainly be improved too, but until it is the bottleneck I wouldn't worry about it.