The fastest way to save graph to file in C - c

Question

I have such a problem: I need to save a large graph with > million edges to txt file. Each edge is represented with a structure containing 3 integers: from, to, cost. My task is to write a program that will fastly save the whole graph to txt file in such format: "from to cost\n".
I am interested in the method, how to do that.
My idea is in creating a huge buffer of chars, where I add each digit to the buffer without the need to reverse then (first of all I get the number of digits of each integer, then add each digit to the buffer, then I add whitespace/new line symbol and do it until the last number is added.
Then I save the whole buffer to file by using fwrite() function.
Despite the fact that this method is relatively fast, I saw programs that do it faster. My question is: do you know more efficient way to implement this program in order to get faster results?
The program must be in C language.
typedef struct {
int edge_start;
int edge_count;
int parent;
int cost;
} node_t;
typedef struct {
graph_t *graph;
node_t *nodes;
int num_nodes;
int start_node;
} dijkstra_t;
The function to get the number of digits:
int getNumberOfDigitsBig(int x) {
if (x >= 10000) {
if (x >= 10000000) {
if (x >= 100000000) {
if (x >= 1000000000)
return 9;
return 8;
}
return 7;
}
if (x >= 100000) {
if (x >= 1000000)
return 6;
return 5;
}
return 4;
}
if (x >= 100) {
if (x >= 1000)
return 3;
return 2;
}
if (x >= 10)
return 1;
return 0;
}
Save function:
const dijkstra_t *const dij = (dijkstra_t*)dijkstra;
if (dij) {
FILE *f = fopen(filename, "w");
if (f) {
int numberOfNodes = dij->num_nodes;
long bufferLength = numberOfNodes * (9 * 3 + 3);
buffer = (char *)malloc(bufferLength + 1);
long bufferCounter = 0;
int number;
// printf("i = %d\n", number);
int counter;
int digits;
buffer[bufferCounter++] = '0';
buffer[bufferCounter++] = ' ';
buffer[bufferCounter++] = '0';
buffer[bufferCounter++] = ' ';
buffer[bufferCounter++] = '-';
buffer[bufferCounter++] = '1';
buffer[bufferCounter++] = '\n';
for(int i = 1; i < numberOfNodes; i++) {
const node_t *const node = &(dij->nodes[i]);
number = i;
digits = getNumberOfDigits(number);
counter = bufferCounter;
do {
buffer[counter + digits] = ZERO + number % 10;
--digits;
++bufferCounter;
} while(number /= 10);
buffer[bufferCounter++] = ' ';
number = node->cost;
if(number != -1) {
digits = getNumberOfDigitsBig(number);
counter = bufferCounter;
do {
buffer[counter + digits] = ZERO + number % 10;
digits = digits - 1;
bufferCounter = bufferCounter + 1;
} while(number /= 10);
} else {
buffer[bufferCounter++] = '-';
buffer[bufferCounter++] = '1';
}
buffer[bufferCounter++] = ' ';
buffer[bufferCounter++] = ' ';
number = node->parent;
if(number != -1) {
digits = getNumberOfDigitsBig(number);
counter = bufferCounter;
do {
buffer[counter + digits] = ZERO + number % 10;
--digits;
++bufferCounter;
} while(number /= 10);
} else {
buffer[bufferCounter++] = '-';
buffer[bufferCounter++] = '1';
}
buffer[bufferCounter++] = '\n';
}
fwrite(buffer, 1, bufferCounter, f);
ret = fclose(f) == 0;
free(buffer);
}
}
Thanks for attention.

Answer 1

I assume that what you need is an optimized version of printf that only processes positive integers. I did not benchmark it, but I would try to do as little comparisons and operations as possible, so I ended with that function:
int printint(FILE *fd, int n) {
char buffer[32]; // an uint64_t uses max 20 chars in base 10
int i = sizeof(buffer);
do {
buffer[--i] = '0' + n%10; // write digits from the right of buffer
n /= 10;
} while(n > 0);
return fwrite(buffer + i, 1, sizeof(buffer) - i, fd);
}
Then I would not use a huge buffer, but just rely on the default buffering of a FILE *
Saving code could then become (more or less just started with the example form the question):
const dijkstra_t *const dij = (dijkstra_t*)dijkstra;
if (dij) {
FILE *f = fopen(filename, "w");
if (f) {
int numberOfNodes = dij->num_nodes;
fputs("0 0 -1\n", f);
for(int i = 1; i < numberOfNodes; i++) {
const node_t *const node = &(dij->nodes[i]);
fputc(' ', f);
number = node->parent;
//printf("parent = %d\n", number);
if(number != -1) {
printint(number, f);
} else {
fwrite("-1", 1, 2, f);
}
fputc('\n', f);
}
ret = fclose(f) == 0;
free(buffer);
}
}

Answer 2

You can improve a little by using this "itoa":
void gwf_i2a(char *d, int i, int l) {
char *e = d + l;
while (l > 0) {
e--;
l--;
e[0] = '0' + (i % 10);
i /= 10;
}
}
ORIGINAL TIME: 76 clicks (7.6e-05 seconds).
NEW TIME: 39 clicks (3.9e-05 seconds).
source:
#include <ctime>
#include <iostream>
#include <random>
#include <vector>
#define ZERO '0'
void gwf_i2a(char *d, int i, int l) {
char *e = d + l;
while (l > 0) {
e--;
l--;
e[0] = '0' + (i % 10);
i /= 10;
}
}
typedef struct {
int x, y, z;
} graph_t;
typedef struct {
int edge_start;
int edge_count;
int parent;
int cost;
} node_t;
typedef struct {
graph_t *graph;
node_t *nodes;
int num_nodes;
int start_node;
} dijkstra_t;
graph_t graph = {111, 222, 3456789};
node_t nodes[] = {{1, 1, 1, 9999}, {2, 2, 2, 8999}, {2, 2, 2, 1234567890}};
dijkstra_t data[] = {&graph, (node_t *)&nodes, 4, 0};
int getNumberOfDigits(int x) {
if (x >= 100) {
if (x >= 1000) return 3;
return 2;
}
if (x >= 10) return 1;
return 0;
}
int getNumberOfDigitsBig(int x) {
if (x >= 10000) {
if (x >= 10000000) {
if (x >= 100000000) {
if (x >= 1000000000) return 9;
return 8;
}
return 7;
}
if (x >= 100000) {
if (x >= 1000000) return 6;
return 5;
}
return 4;
}
if (x >= 100) {
if (x >= 1000) return 3;
return 2;
}
if (x >= 10) return 1;
return 0;
}
void save(const char *filename, const dijkstra_t *dijkstra) {
int ret;
const dijkstra_t *const dij = (dijkstra_t *)dijkstra;
char *buffer;
if (dij) {
FILE *f = fopen(filename, "w");
if (f) {
int numberOfNodes = dij->num_nodes;
long bufferLength = numberOfNodes * (9 * 3 + 3);
buffer = (char *)malloc(bufferLength + 1);
long bufferCounter = 0;
int number;
// printf("i = %d\n", number);
int counter;
int digits;
buffer[bufferCounter++] = '0';
buffer[bufferCounter++] = ' ';
buffer[bufferCounter++] = '0';
buffer[bufferCounter++] = ' ';
buffer[bufferCounter++] = '-';
buffer[bufferCounter++] = '1';
buffer[bufferCounter++] = '\n';
for (int i = 1; i < numberOfNodes; i++) {
const node_t *const node = &(dij->nodes[i]);
number = i;
digits = getNumberOfDigits(number);
counter = bufferCounter;
do {
buffer[counter + digits] = ZERO + number % 10;
--digits;
++bufferCounter;
} while (number /= 10);
buffer[bufferCounter++] = ' ';
number = node->cost;
if (number != -1) {
digits = getNumberOfDigitsBig(number);
counter = bufferCounter;
do {
buffer[counter + digits] = ZERO + number % 10;
digits = digits - 1;
bufferCounter = bufferCounter + 1;
} while (number /= 10);
} else {
buffer[bufferCounter++] = '-';
buffer[bufferCounter++] = '1';
}
buffer[bufferCounter++] = ' ';
buffer[bufferCounter++] = ' ';
number = node->parent;
if (number != -1) {
digits = getNumberOfDigitsBig(number);
counter = bufferCounter;
do {
buffer[counter + digits] = ZERO + number % 10;
--digits;
++bufferCounter;
} while (number /= 10);
} else {
buffer[bufferCounter++] = '-';
buffer[bufferCounter++] = '1';
}
buffer[bufferCounter++] = '\n';
}
fwrite(buffer, 1, bufferCounter, f);
ret = fclose(f) == 0;
free(buffer);
}
}
}
void new_save(const char *filename, const dijkstra_t *dijkstra) {
int ret;
const dijkstra_t *const dij = (dijkstra_t *)dijkstra;
char *buffer;
if (dij) {
FILE *f = fopen(filename, "w");
if (f) {
int numberOfNodes = dij->num_nodes;
long bufferLength = numberOfNodes * (9 * 3 + 3);
buffer = (char *)malloc(bufferLength + 1);
long bufferCounter = 0;
int number;
int counter;
int digits;
buffer[bufferCounter++] = '0';
buffer[bufferCounter++] = ' ';
buffer[bufferCounter++] = '0';
buffer[bufferCounter++] = ' ';
buffer[bufferCounter++] = '-';
buffer[bufferCounter++] = '1';
buffer[bufferCounter++] = '\n';
for (int i = 1; i < numberOfNodes; i++) {
const node_t *const node = &(dij->nodes[i]);
int len = getNumberOfDigits(i) + 1;
gwf_i2a((char *)&buffer[bufferCounter], i, len);
bufferCounter += len;
buffer[bufferCounter++] = ' ';
number = node->cost;
if (number != -1) {
len = getNumberOfDigitsBig(number) + 1;
gwf_i2a((char *)&buffer[bufferCounter], number, len);
bufferCounter += len;
} else {
buffer[bufferCounter++] = '-';
buffer[bufferCounter++] = '1';
}
buffer[bufferCounter++] = ' ';
buffer[bufferCounter++] = ' ';
number = node->parent;
if (number != -1) {
digits = getNumberOfDigitsBig(number);
counter = bufferCounter;
do {
buffer[counter + digits] = ZERO + number % 10;
--digits;
++bufferCounter;
} while (number /= 10);
} else {
buffer[bufferCounter++] = '-';
buffer[bufferCounter++] = '1';
}
buffer[bufferCounter++] = '\n';
}
fwrite(buffer, 1, bufferCounter, f);
ret = fclose(f) == 0;
free(buffer);
}
}
}
void original() {
clock_t t;
t = clock();
save("bogus.txt", data);
t = clock() - t;
std::cout << "original: " << t << " clicks (" << ((float)t) / CLOCKS_PER_SEC
<< " seconds)." << std::endl;
}
void new_test() {
clock_t t;
t = clock();
new_save("new_bogus.txt", data);
t = clock() - t;
std::cout << "NEW: " << t << " clicks (" << ((float)t) / CLOCKS_PER_SEC
<< " seconds)." << std::endl;
}
int main(int argc, char **argv) {
original();
new_test();
return 0;
}

Answer 3

[Rewritten on 2018-01-13.]
Standard I/O (printf() etc.) is indeed comparatively slow in converting numerical data to text form. Here, the problem is to output lines of form
<node> <cost> <parent>
where all three are either unsigned (32-bit) integers in decimal notation, or -1. For simplicity, let's reserve the value UINT32_MAX (4294967295) for -1.
I suggest a two-fold approach:
Construct each record from right to left. This avoids the need of checking how many digits there are in a number.
Buffer a number of records at once. This reduces the number of fwrite() calls, at the cost of a modest dynamically allocated buffer.
Note that this means that the records in each chunk must be processed last-to-first, in order to retain the correct order.
Consider the following code. Note that I've reduced the definitions of node_t and dijkstra_t to the fields that area actually used, so that the following example can be compiled as-is. Also note that instead of -1 for parent or cost, one must use UINT32_MAX, as their types are now uint32_t.
#include <stdlib.h>
#include <stdint.h>
#include <limits.h>
#include <stdio.h>
typedef struct {
uint32_t parent; /* Use UINT32_MAX for -1 */
uint32_t cost; /* Use UINT32_MAX for -1 */
} node_t;
typedef struct {
node_t *nodes;
uint32_t num_nodes;
} dijkstra_t;
/* This function will store an unsigned 32-bit value
in decimal form, ending at 'end'.
UINT32_MAX will be written as "-1", however.
Returns a pointer to the start of the value.
*/
static inline char *prepend_value(char *end, uint32_t value)
{
if (value == UINT32_MAX) {
*(--end) = '1';
*(--end) = '-';
} else {
do {
*(--end) = '0' + (value % 10u);
value /= 10u;
} while (value);
}
return end;
}
/* Each record consists of three unsigned 32-bit integers,
each at most 10 characters, with spaces in between
and a newline at end. Thus, at most 33 characters. */
#define RECORD_MAXLEN 33
/* We process records in chunks of 16384.
Maximum number of records (nodes) is 2**32 - 2 - RECORD_CHUNK,
or 4,294,950,910 in this case. */
#define RECORD_CHUNK 16384
/* Each chunk of record is up to CHUNK_CHARS long.
(Roughly half a megabyte in this case.) */
#define CHUNK_CHARS (RECORD_MAXLEN * RECORD_CHUNK)
/* Save the edges in a graph to a stream.
Returns 0 if success, -1 if an error occurs.
*/
int save_edges(dijkstra_t *dij, FILE *out)
{
if (dij && out && !ferror(out)) {
const int nodes = dij->num_nodes;
const node_t *node = dij->nodes;
const uint32_t root_parent = dij->nodes->parent;
const uint32_t root_cost = dij->nodes->cost;
char *buf, *end, *ptr;
uint32_t o;
/* Allocate memory for the chunk buffer. */
buf = malloc(CHUNK_CHARS);
if (!buf)
return -1;
end = buf + CHUNK_CHARS;
/* Temporarily, we reset the root node parent
to UINT32_MAX and cost to 0, so that the
very first record in the output is "0 0 -1". */
dij->nodes->cost = 0;
dij->nodes->parent = UINT32_MAX;
for (o = 0; o < nodes; o += RECORD_CHUNK) {
uint32_t i = (o + RECORD_CHUNK < nodes) ? o + RECORD_CHUNK : nodes;
/* Fill buffer back-to-front. */
ptr = end;
while (i-->o) {
const node_t *curr = node + i;
/* Format: <i> ' ' <cost> ' ' <parent> '\n' */
/* We construct the record from right to left. */
*(--ptr) = '\n';
ptr = prepend_value(ptr, curr->parent);
*(--ptr) = ' ';
ptr = prepend_value(ptr, curr->cost);
*(--ptr) = ' ';
ptr = prepend_value(ptr, i);
}
/* Write the chunk buffer out. */
if (fwrite(ptr, 1, (size_t)(end - ptr), out) != (size_t)(end - ptr)) {
dij->nodes->cost = root_cost;
dij->nodes->parent = root_parent;
free(buf);
return -1;
}
}
/* Reset root node, and free the buffer. */
dij->nodes->cost = root_cost;
dij->nodes->parent = root_parent;
free(buf);
/* Check for write errors. */
if (fflush(out))
return -1;
if (ferror(out))
return -1;
/* Success. */
return 0;
}
return -1;
}
Additional speedup is possible, if we can use POSIX low-level I/O (open(), close(), write(), and fstat() from <unistd.h>). When the destination is a pipe or device, we can just directly write the data; when the destination is a file, we should write in chunks of multiples of st_blksize, to avoid read-modify-write cycles. Unlike standard I/O, with low-level I/O we can do that with just one "overflow" buffer of st_blksize, without having to copy the entire chunk buffer around in memory. However, since the question is not tagged posix, I shall refrain from further discussion along those edges.
OP stated their own version is still faster. I found that difficult to believe, because it does much more work than my version above. When I checked, on my machine a large dataset (of say 100,000,000) cannot be written in a single fwrite() call, as it only does a partial write; a loop is required to actually write the entire dataset. Therefore, in my opinion, the benchmark OP uses to compare the different version is very suspect.
Consider the following microbenchmark instead. It generates a singly linked list, and uses an externally compiled save_graph() function to output it (to standard output). There are three versions implemented: null, which does not save anything at all; antonkretov, for OP's implementation (adapted to work here); and nominalanimal, for mine.
Makefile:
CC := gcc
CFLAGS := -std=c99 -O2 -Wall
LDFLAGS :=
BINS := test-null test-antonkretov test-nominalanimal
NODES := 100000000
.PHONY: all clean run
all: clean $(BINS)
clean:
rm -f $(BINS) *.o
%.o: %.c
$(CC) $(CFLAGS) -c $^
test-null: main.o data-null.o
$(CC) $(CFLAGS) $^ -o $#
test-antonkretov: main.o data-antonkretov.o
$(CC) $(CFLAGS) $^ -o $#
test-nominalanimal: main.o data-nominalanimal.o
$(CC) $(CFLAGS) $^ -o $#
run: $(BINS)
#echo "Testing $(NODES) nodes."
#./test-null $(NODES) > /dev/null
#echo "Overhead (nothing saved):"
#bash -c 'time ./test-null $(NODES) > /dev/null'
#echo ""
#echo "Anton Kretov:"
#bash -c 'time ./test-antonkretov $(NODES) > /dev/null'
#echo ""
#echo "Nominal Animal:"
#bash -c 'time ./test-nominalanimal $(NODES) > /dev/null'
#echo ""
Note that this forum converts Tabs to spaces, and Makefile format requires the indentation to use spaces, so if you copy and paste the above to file, you need to run e.g. sed -e 's|^ *|\t|' -i Makefile to fix it.
data.h:
#ifndef DATA_H
#define DATA_H
#include <stdint.h>
#include <limits.h>
#include <stdio.h>
#define INVALID_COST UINT32_MAX
#define INVALID_PARENT UINT32_MAX
typedef struct {
uint32_t parent; /* Use INVALID_PARENT for -1 */
uint32_t cost; /* Use INVALID_COST for -1 */
} node_t;
typedef struct {
node_t *nodes;
uint32_t num_nodes;
} dijkstra_t;
int save_graph(dijkstra_t *, FILE *);
#endif /* DATA_H */
data-null.c, for measuring runtime overhead:
#include "data.h"
int save_graph(dijkstra_t *dij, FILE *out)
{
/* Does not do anything */
return 0;
}
data-antonkretov.c, version of OP's save routine, for comparison:
#include <stdlib.h>
#include "data.h"
int getNumberOfDigits(uint32_t x)
{
if (x >= 10000) {
if (x >= 10000000) {
if (x >= 100000000) {
if (x >= 1000000000)
return 9;
return 8;
}
return 7;
}
if (x >= 100000) {
if (x >= 1000000)
return 6;
return 5;
}
return 4;
}
if (x >= 100) {
if (x >= 1000)
return 3;
return 2;
}
if (x >= 10)
return 1;
return 0;
}
int save_graph(dijkstra_t *dij, FILE *out)
{
uint32_t numberOfNodes = dij->num_nodes;
size_t bufferLength = numberOfNodes * (size_t)33;
size_t bufferCounter = 0, counter;
size_t bytes;
uint32_t number, digits, i;
char *buffer;
if ((size_t)(bufferLength / 33) != numberOfNodes)
return -1;
buffer = malloc(bufferLength);
if (!buffer)
return -1;
buffer[bufferCounter++] = '0';
buffer[bufferCounter++] = ' ';
buffer[bufferCounter++] = '0';
buffer[bufferCounter++] = ' ';
buffer[bufferCounter++] = '-';
buffer[bufferCounter++] = '1';
buffer[bufferCounter++] = '\n';
for (i = 1; i < numberOfNodes; i++) {
const node_t *const node = dij->nodes + i;
number = i;
digits = getNumberOfDigits(number);
counter = bufferCounter;
do {
buffer[counter + digits] = '0' + (number % 10u);
--digits;
++bufferCounter;
} while (number /= 10u);
buffer[bufferCounter++] = ' ';
number = node->cost;
if (number != UINT32_MAX) {
digits = getNumberOfDigits(number);
counter = bufferCounter;
do {
buffer[counter + digits] = '0' + (number % 10u);
--digits;
++bufferCounter;
} while (number /= 10u);
} else {
buffer[bufferCounter++] = '-';
buffer[bufferCounter++] = '1';
}
buffer[bufferCounter++] = ' ';
number = node->parent;
if (number != UINT32_MAX) {
digits = getNumberOfDigits(number);
counter = bufferCounter;
do {
buffer[counter + digits] = '0' + (number % 10u);
--digits;
++bufferCounter;
} while (number /= 10u);
} else {
buffer[bufferCounter++] = '-';
buffer[bufferCounter++] = '1';
}
buffer[bufferCounter++] = '\n';
}
counter = 0;
while (counter < bufferCounter) {
bytes = fwrite(buffer + counter, 1, bufferCounter - counter, out);
if (!bytes) {
free(buffer);
return -1;
}
counter += bytes;
}
free(buffer);
return 0;
}
data-nominalanimal.c, my chunked back-to-front version of the save routine:
#include <stdlib.h>
#include "data.h"
/* This function will store an unsigned 32-bit value
in decimal form, ending at 'end'.
UINT32_MAX will be written as "-1", however.
Returns a pointer to the start of the value.
*/
static inline char *prepend_value(char *end, uint32_t value)
{
if (value == UINT32_MAX) {
*(--end) = '1';
*(--end) = '-';
} else {
do {
*(--end) = '0' + (value % 10u);
value /= 10u;
} while (value);
}
return end;
}
/* Each record consists of three unsigned 32-bit integers,
each at most 10 characters, with spaces in between
and a newline at end. Thus, at most 33 characters. */
#define RECORD_MAXLEN 33
/* We process records in chunks of 16384.
Maximum number of records (nodes) is 2**32 - 2 - RECORD_CHUNK,
or 4,294,950,910 in this case. */
#define RECORD_CHUNK 16384
/* Each chunk of record is up to CHUNK_CHARS long.
(Roughly half a megabyte in this case.) */
#define CHUNK_CHARS (RECORD_MAXLEN * RECORD_CHUNK)
/* Save the edges in a graph to a stream.
Returns 0 if success, -1 if an error occurs.
*/
int save_graph(dijkstra_t *dij, FILE *out)
{
if (dij && out && !ferror(out)) {
const int nodes = dij->num_nodes;
const node_t *node = dij->nodes;
const uint32_t root_parent = dij->nodes->parent;
const uint32_t root_cost = dij->nodes->cost;
char *buf, *end, *ptr;
uint32_t o;
/* Allocate memory for the chunk buffer. */
buf = malloc(CHUNK_CHARS);
if (!buf)
return -1;
end = buf + CHUNK_CHARS;
/* Temporarily, we reset the root node parent
to UINT32_MAX and cost to 0, so that the
very first record in the output is "0 0 -1". */
dij->nodes->cost = 0;
dij->nodes->parent = UINT32_MAX;
for (o = 0; o < nodes; o += RECORD_CHUNK) {
uint32_t i = (o + RECORD_CHUNK < nodes) ? o + RECORD_CHUNK : nodes;
/* Fill buffer back-to-front. */
ptr = end;
while (i-->o) {
const node_t *curr = node + i;
/* Format: <i> ' ' <cost> ' ' <parent> '\n' */
/* We construct the record from right to left. */
*(--ptr) = '\n';
ptr = prepend_value(ptr, curr->parent);
*(--ptr) = ' ';
ptr = prepend_value(ptr, curr->cost);
*(--ptr) = ' ';
ptr = prepend_value(ptr, i);
}
/* Write buffer. */
if (fwrite(ptr, 1, (size_t)(end - ptr), out) != (size_t)(end - ptr)) {
dij->nodes->cost = root_cost;
dij->nodes->parent = root_parent;
free(buf);
return -1;
}
}
/* Reset root node, and free the buffer. */
dij->nodes->cost = root_cost;
dij->nodes->parent = root_parent;
free(buf);
if (fflush(out))
return -1;
if (ferror(out))
return -1;
return 0;
}
return -1;
}
and finally the main program itself, main.c, that generates the data and calls the save_graph() functions:
#include <stdlib.h>
#include <inttypes.h>
#include <limits.h>
#include <string.h>
#include "data.h"
#define EDGES_MAX 4294901759
int main(int argc, char *argv[])
{
dijkstra_t graph;
size_t bytes;
uint32_t edges, i;
char dummy;
if (argc != 2 || !strcmp(argv[1], "-h") || !strcmp(argv[1], "--help")) {
fprintf(stderr, "\nUsage: %s EDGES\n\n", argv[0]);
return EXIT_SUCCESS;
}
if (sscanf(argv[1], " %" SCNu32 " %c", &edges, &dummy) != 1 || edges < 1 || edges > EDGES_MAX) {
fprintf(stderr, "%s: Invalid number of edges.\n", argv[1]);
return EXIT_FAILURE;
}
bytes = (1 + (size_t)edges) * sizeof graph.nodes[0];
if ((size_t)(bytes / (1 + (size_t)edges)) != sizeof graph.nodes[0]) {
fprintf(stderr, "%s: Too many edges.\n", argv[1]);
return EXIT_FAILURE;
}
graph.num_nodes = edges + 1;
graph.nodes = malloc(bytes);
if (!graph.nodes) {
fprintf(stderr, "%s: Too many edges: out of memory.\n", argv[1]);
return EXIT_FAILURE;
}
/* Generate a graph; no randomness, to keep timing steady. */
graph.nodes[0].parent = INVALID_COST;
graph.nodes[0].cost = 0;
for (i = 1; i <= edges; i++) {
graph.nodes[i].parent = i - 1;
graph.nodes[i].cost = 1 + (i % 10);
}
/* Print graph. */
if (save_graph(&graph, stdout)) {
fprintf(stderr, "Write error!\n");
return EXIT_FAILURE;
}
/* Done. */
return EXIT_SUCCESS;
}
Running make clean run (or make NODES=100000000 clean run) recompiles the benchmarks, and measures their run time, for a graph with 100,000,000 nodes. On my machine, the output is
Testing 100000000 nodes.
Overhead (nothing saved):
real0m0.514s
user0m0.297s
sys0m0.217s
Anton Kretov:
real0m4.059s
user0m3.379s
sys0m0.680s
Nominal Animal:
real0m3.336s
user0m3.151s
sys0m0.184s
which shows that mine is significantly faster. If we ignore the overhead (of generating the graph), mine took about 2.8 seconds of real time to save the data to /dev/null, whereas OP's took about 3.5 seconds. In other words, mine shows a 20% speed improvement.
It is important to note that both tests do produce the exact same output. For example, both ./test-nominalanimal 100000000 | sha256sum - and ./test-antonkretov 100000000 | sha256sum - show the exact same SHA256 checksums, 7504a1c97167701297c03c4aab8b0f20c5cac82a50128074d6e09c474353d0f8.
(You can also save the output to a file, and compare them; both are exactly 1,987,777,795 bytes long, and contain the exact same data. I did check.)
If you want to run a benchmark that stores the data to storage, for the comparisons to be fair, you need to start with cold caches. Otherwise the order in which you run the benchmarks will heavily impact their timings.