multiple threads reading a file: pread or multiple fds - c

Question

I have a multi-threaded program and I want to have each thread read from the same file and have the same data
I know that from pread(2) it is thread safe; however, I'm concerned about speed and not memory.
Will it be faster to have multiple threads read from the same file descriptor using pread or would it be faster for each individual thread to have its own file descriptor to the same file and read from there?
My thought is, if pread is atomic, then that prevents another thread from reading at the same time; however, if having lots of threads each with their own fd and the OS needs to service each read, then all that context switching might take up more time.

Answer 1

Having the same requirement I did a test for this. According to the test on a SSD and a HDD,
pread - improves the read speed, but if same FD is used for the writing, write becomes slow.
read (with a separate FD) - read operations are slower compared to pread. But it does NOT impact write operations (done using a separate FD).
So best option is to,
Use a separate FD for writing (Write Only mode)
Open a single FD (Read only mode) and use it across multiple threads using pread
results on a SSD (read/write count 10,000,000)
-
pread (same FD used for write and read)
read using a different FD
Read
25 sec +
27 sec +
Write
38 sec +
33 sec+
Code used for the test
#include <string.h>
#include <iostream>
#include <unistd.h>
#include <sstream>
#include <string.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <time.h>
#include <errno.h>
#include <assert.h>
#include <sys/time.h>
#include <memory>
#include <atomic>
#include <thread>
off_t tell(int fd)
{
return lseek(fd, 0, SEEK_CUR);
}
inline int64_t getMicroSecDiff(const timeval& begin, const timeval& end)
{
constexpr uint32_t MicroSecPerSec = 1000 * 1000;
return (end.tv_sec - begin.tv_sec) * MicroSecPerSec + (end.tv_usec - begin.tv_usec);
}
inline int64_t getMicroSecDiffFrom(const timeval& begin)
{
timeval end;
gettimeofday(&end, nullptr);
return getMicroSecDiff(begin, end);
}
using namespace std;
int g_fileFd_A = -1;
int g_fileFd_B = -1;
std::atomic<int64_t> g_lastFileAPos = { -1 };
std::atomic<int64_t> g_lastFileBPos = { -1 };
std::atomic<bool> g_writeCompleted = { false };
const int WriteSz = 1;
uint64_t g_writeCount = 0;
void writer(int fdA, int fdB)
{
timeval begin;
gettimeofday(&begin, nullptr);
for (size_t i = 0; i < g_writeCount; i++)
{
int64_t fileAPosition = tell(fdA);
int64_t fileBPosition = tell(fdB);
auto written = write(fdA, "A", WriteSz);
assert(written == WriteSz);
written = write(fdB, "b", WriteSz);
assert(written == WriteSz);
g_lastFileAPos.store(fileAPosition, std::memory_order_relaxed);
g_lastFileBPos.store(fileBPosition, std::memory_order_relaxed);
}
g_writeCompleted = true;
auto diff = getMicroSecDiffFrom(begin);
std::ostringstream oss;
oss << "Write time: " << diff / 1000000 << " sec " << diff % 1000000 << " us" << endl;
cout << oss.str();
}
void reader(bool duplicateFds)
{
char bufferA[8];
char bufferB[8];
int failedCount = 0;
int fdA, fdB;
if (duplicateFds)
{
fdA = open("./A", O_RDONLY);
fdB = open("./B", O_RDONLY);
}
else
{
fdA = g_fileFd_A;
fdB = g_fileFd_B;
}
timeval begin;
gettimeofday(&begin, nullptr);
size_t iteNo = 0;
//for (; g_writeCompleted.load(memory_order_relaxed) == false; iteNo++)
for (; iteNo < g_writeCount; iteNo++)
{
off_t posA = g_lastFileAPos.load(std::memory_order_relaxed);
off_t posB = g_lastFileBPos.load(std::memory_order_relaxed);
if (posA < 0 or posB < 0)
{
iteNo--;
gettimeofday(&begin, nullptr);
continue;
}
int readSzA;
int readSzB;
off_t readPosA = rand();
off_t readPosB = rand();
if (readPosA > posA)
{
readPosA = posA;
}
if (readPosB > posB)
{
readPosB = posB;
}
if (duplicateFds)
{
lseek(fdA, readPosA, SEEK_SET);
lseek(fdB, readPosB, SEEK_SET);
readSzA = read(fdA, bufferA, WriteSz);
readSzB = read(fdB, bufferB, WriteSz);
}
else
{
readSzA = pread(fdA, bufferA, WriteSz, readPosA);
readSzB = pread(fdB, bufferB, WriteSz, readPosB);
}
if (readSzA < WriteSz or readSzB < WriteSz)
{
failedCount++;
if (failedCount % 1000 == 0)
{
cout << " " << failedCount;
cout.flush();
}
}
}
auto diff = getMicroSecDiffFrom(begin);
std::ostringstream oss;
oss << "Read time: " << diff / 1000000 << " sec " << diff % 1000000 << " us" << " ReadCount=" << iteNo << endl;
oss << "failedCount=" << failedCount << endl;
cout << oss.str();
if (duplicateFds)
{
close(fdA);
close(fdB);
}
}
int main (int argc, char** argv)
{
if (argc < 3)
{
cout << "usage: " << argv[0] << " duplicateFds" << " write-count" << endl;
return 0;
}
bool duplicateFds = atoi(argv[1]);
g_writeCount = atoi(argv[2]);
cout << "duplicateFds=" << duplicateFds << " write-count=" << g_writeCount << endl;
if (duplicateFds)
{
g_fileFd_A = open("./A", O_WRONLY | O_CREAT, S_IREAD | S_IWRITE | S_IRGRP | S_IROTH);
g_fileFd_B = open("./B", O_WRONLY | O_CREAT, S_IREAD | S_IWRITE | S_IRGRP | S_IROTH);
}
else
{
g_fileFd_A = open("./A", O_RDWR | O_CREAT, S_IREAD | S_IWRITE | S_IRGRP | S_IROTH);
g_fileFd_B = open("./B", O_RDWR | O_CREAT, S_IREAD | S_IWRITE | S_IRGRP | S_IROTH);
}
std::thread readerThread1(reader, duplicateFds);
std::thread readerThread2(reader, duplicateFds);
std::thread writerThread(writer, g_fileFd_A, g_fileFd_B);
readerThread1.join();
readerThread2.join();
writerThread.join();
close(g_fileFd_A);
close(g_fileFd_B);
return 0;
}