How to stop page faults from slowing C program with mmap? - c

Question

I am developing a program that involves making reads and writes to a file on parallel pthreads in C. The program mmaps a portion of the file (which is always a multiple of the page size) into different pages and then creates assignments for each thread on the system that contain an array of pointers to the start of various pages that have been mapped. Each thread is then given an assignment of pages that must be transformed with a function, and the main thread waits until all the other threads finish their work on the pages. Then, it will mmap the next portion of the file and repeat the thread dispersal and execution process. I have noticed that as pages continue to load, their loading time increases significantly. I believe that page faults are taking longer and longer as more data is edited. I have done various testing indicating that the first hit on a page is what slows the transformation of the pages down increasingly more and more. I realize this is likely the operating system slowing things down, any ideas on what is going on and how I can increase speed?
I am working on a 16gb RAM, 4 core (1 thread per core) machine. Below, you will find a simpler version of my program where a function called simpleMod is setting the first byte of each page of the file to 0. disperseFunction splits up the file into the max amount that can be mmaped at once and runFunction does the actual thread dispersal. For a 1gb file, it take ~0.3 seconds. For and 8gb file, it takes ~35 second showing the drastic time increase. I did try madvise for sequential reads. Here is the code:
#include <pthread.h>
#include <sched.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <stdio.h>
#include <stdint.h>
#define DEFAULT_THREADS_PER_CORE 2
#define BYTE_COUNT 4096
#define LONG_COUNT 512
#define DEFAULT_MAX_LOAD 1073741824
#define PTR_ADD(ptr, off) ((void *) (((size_t) (ptr)) + ((size_t) (off))))
//the blocks that are read to be changed
union block {
uint_fast64_t longs[LONG_COUNT];
char bytes[BYTE_COUNT];
} block;
//pointers to positions in file
typedef struct job {
void *start;
} job;
//what each thread has to do
typedef struct assignment {
job *jobs;
unsigned long long jobCount;
} assignment;
unsigned int threadsPerCore;
unsigned int threadCount;
unsigned long long maxLoad;
/*
This function takes in a function and distributes it to threads
#param loadSize is how many bytes to distribute
#param offset is how many bytes into file to start at
#param fd is the file descriptor
#param assignments is the assignments array to utilize
#param threads is the threads array to dispatch to
#param function is the function to be dispatched
The load size and offset must be multiples of 4096
*/
int runFunction(unsigned long long loadSize, unsigned long long offset, int fd,
assignment assignments[], pthread_t threads[], void *(*function)(void *)) {
//get pointer to location in file
void *fileBytes = mmap(NULL, loadSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd, offset);
if(fileBytes == MAP_FAILED) { //check for error
printf("Could not open file\n");
return 1;
}
//calculate distribution data
unsigned long long chunkCount = loadSize / BYTE_COUNT;
unsigned long long chunksPerThread = chunkCount / threadCount;
unsigned long long remainingChunks = chunkCount - chunksPerThread * threadCount;
unsigned long long addPerAssignemnt = (remainingChunks + (threadCount - 1)) / threadCount;
//get jobs counts and malloc sapce in assignments
for(int i = 0; i < threadCount; i++) {
assignments[i].jobCount = chunksPerThread;
if(remainingChunks == 0) {
assignments[i].jobCount += remainingChunks;
remainingChunks = 0;
} else if(remainingChunks >= addPerAssignemnt) {
assignments[i].jobCount += addPerAssignemnt;
remainingChunks -= addPerAssignemnt;
}
assignments[i].jobs = malloc(sizeof(job) * assignments[i].jobCount);
}
//then assignments need to be filled with jobs
//sequential memory blocks deployed to threads
unsigned long long done = 0;
unsigned int t;
for(t = 0; t < threadCount; t++) { //cycle through dimensions
for(unsigned long long j = 0; j < assignments[t].jobCount; j++) {
//setup new job
job *newJob = &assignments[t].jobs[j];
newJob->start = PTR_ADD(fileBytes, done + j * BYTE_COUNT);
}
done += assignments[t].jobCount * BYTE_COUNT;
}
//setup threads for function
for(t = 0; t < threadCount; t++) {
pthread_create(&threads[t], NULL, function, &assignments[t]);
}
for(t = 0; t < threadCount; t++) {
pthread_join(threads[t], NULL);
}
//free the jobs
for(t = 0; t < threadCount; t++) {
free(assignments[t].jobs);
}
//unmap mem
munmap(fileBytes, loadSize);
}//end runFunction
/*
This function breaks up the file into chunks no greater than the max size and then disperses that chunk to the threads
#param function is the function to disperse
#param threads are the threads to disperse onto
#param assignments is the assignments array to ultize
#param fd is the file descriptor
#param fileSize is the size of the file described by fd
*/
int disperseFunction(void *(*function)(void *), pthread_t threads[],
assignment assignments[], int fd, unsigned long long fileSize) {
//only load up to certain amount of file at time
for(unsigned long long lc = 0; lc < fileSize / maxLoad; lc++) {
runFunction(maxLoad, maxLoad * lc, fd, assignments, threads, function);
}
//run remainder
unsigned long long lastSize = fileSize % maxLoad;
if(lastSize) {
runFunction(lastSize, fileSize - lastSize, fd, assignments, threads, function);
}
return 0;
}
void *simpleMod(void *data) {
//get assignment info
//assignments are a struct that hold an array of jobs (which are pointers) and the job count
assignment *assignmentPtr = data;
assignment currentAssignment = *assignmentPtr;
job *jobList = currentAssignment.jobs;
unsigned long long jobCount = currentAssignment.jobCount;
//setup tracking vars
job currentJob;
union block *chunk;
//go through jobs one by one and set first byte to 0
for(unsigned long long j = 0; j < jobCount; j++) {
currentJob = jobList[j];
chunk = currentJob.start;
chunk->bytes[0] = 0;
}//end for each job
return NULL;
}
int main() {
threadCount = 4; //just an example for my four thread system
maxLoad = DEFAULT_MAX_LOAD;
assignment assignments[threadCount];
pthread_t threads[threadCount];
struct stat statBuf;
int fd = open("8g", O_RDWR);
fstat(fd, &statBuf);
//run simpleMod over file
disperseFunction(simpleMod, threads, assignments, fd, statBuf.st_size);
}
Thank you in advance and please let me know any other information I can provide.