I am using Berkeley DB to store the data persistently in my program. I tested it on my SSD, my SSD speed is 1.4Gb/s for writing. My progrma for testing the storing speed of DB is shown below (error checking is omitted).
const char* db_dir="./.db";
const char* db_name = "node_test_0";
void mk_path(char* dest,const char* prefix,const char* db_name){
memcpy(dest,prefix,strlen(prefix));
dest[strlen(prefix)] = '/';
memcpy(dest+strlen(prefix)+1,db_name,strlen(db_name));
dest[strlen(prefix)+strlen(db_name)+1] = '\0';
return;
}
int main() {
DB* b_db;
DB_ENV* dbenv;
int ret;
int flag = 0;
DBT key, data;
char* full_path = NULL;
if((ret = mkdir(db_dir,S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)) != 0){
if(errno != EEXIST){
}
}
full_path = (char*)malloc(strlen(db_dir) + strlen(db_name) + 2);
mk_path(full_path, db_dir, db_name);
if ((ret = db_env_create(&dbenv, 0)) != 0) {
dbenv->err(dbenv, ret, "Environment Created: %s", db_dir);
}
if ((ret = dbenv->open(dbenv, db_dir, DB_CREATE|DB_INIT_CDB|DB_INIT_MPOOL|DB_THREAD, 0)) != 0) {
}
if((ret = db_create(&b_db,dbenv,flag)) != 0){
}
if((ret = b_db->open(b_db, NULL, db_name, NULL, DB_BTREE, DB_THREAD|DB_CREATE,0)) != 0){
}
struct timeval start_time;
struct timeval end_time;
unsigned long e_usec;
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
key.data = "fruit";
key.size = sizeof("fruit");
data.data = "apple";
data.size = sizeof("apple");
gettimeofday(&start_time, NULL);
ret = b_db->put(b_db, NULL, &key, &data, 0);
gettimeofday(&end_time, NULL);
e_usec = ((end_time.tv_sec * 1000000) + end_time.tv_usec) - ((start_time.tv_sec * 1000000) + start_time.tv_usec);
printf("%lu\n", e_usec);
if (ret == 0)
printf("db: %s: key stored.\n", (char *)key.data);
else {
b_db->err(b_db, ret, "DB->put");
}
return 0;
}
The result is about 23 microseconds. It's much slower than I expected. Does anyone have idea on this? What can I do to make my persistent storage as fast as writing to SSD.
BerkeleyDB does a fsync after write's to ensure that the data on disk is saved. You can stub out the fsync(2) vector (just return 0) if you are willing to live with the consequence of possibly catastrophic data loss (which is acceptable for a temporary, non-persistent, data store using BDB)
Related
I got some problems with AF_UNIX socket communication because after writing a data buffer there seem to remain some hanging bytes to read that I do not know where they come from.
I am writing a multithreaded server program in C that communicates with clients through AF_UNIX sockets, it must implement a simple chatroom. Among other things, the server must implement file transfers between clients and servers and i encountered problems when i try to send a quite large file (269K) from Server to client. (With smaller files i do not have any problems)
For file transfer i use mmap() function which return a pointer to the map of the file I want to send, then i use write() for write that data on socket linked with the client that must recieve the file.
After write() call i check the returned value to be equal than the file size. (always verified)
The client, after receiving the file, check the size of read data (always verified) and start waiting for other messages so it call a blocking read(). This is the point where I found the error because the client reads something that should not be there, as if there was something left to read on the socket.
I've been debugging this part (both server and client) for two days and I have not yet been able to understand the origin of the problem.
I am sure that no other thread write on the same socket at the same time
Does any of you have an idea of what the cause of this error is?
I try to post some useful code thinking at a normal operation sequence:
First of all message structure:
struct message_hdr
{
op_t op;
char sender[MAX_NAME_LENGTH+1];
};
struct message_data_hdr{
char receiver[MAX_NAME_LENGTH+1];
unsigned int len;
};
struct message_data
{
message_data_hdr_t hdr;
char *buf;
};
struct message
{
message_hdr_t hdr;
message_data_t data;
};
A server->client file transfer starts with server that send a message_hdr_t to a client which is waiting on a read() (the client expects to receive only a message_hdr_t).
int sendHeader(long fd, message_hdr_t* hdr)
{
if(hdr == NULL || fd < 0) {errno = EINVAL; return -1;}
int test;
struct iovec iov;
iov.iov_base = hdr;
iov.iov_len = sizeof(message_hdr_t);
test = writev(fd, &iov, 1);
return test;
}
The client understands from the operation code (message.hdr.op) that it is a file type message and it begins to wait for file,
So server send it:
int sendData(long fd, message_data_t *msg)
{
if(msg == NULL || fd < 0) {errno = EINVAL; return -1;}
int test;
struct iovec iov;
iov.iov_base = &(msg->hdr);
iov.iov_len = sizeof(message_data_hdr_t);
test = writev(fd, &(iov), 1);
if(test == -1){return -1;}
if (msg->hdr.len != 0)
{
test = write(fd, msg->buf, msg->hdr.len);
if(test <= 0)
return -1;
}
return test;
}
And client read it:
int readData(long fd, message_data_t *data)
{
if(data == NULL || fd < 0) {errno = EINVAL; return -1;}
int test;
struct iovec iov;
iov.iov_base = &(data->hdr);
iov.iov_len = sizeof(message_data_hdr_t);
test = readv(fd, &iov, 1);
if(test <= 0){return -1;}
if(data->hdr.len != 0)
{
data->buf = malloc(data->hdr.len);
if(data->buf == NULL){return -1;}
test = read(fd, data->buf, data->hdr.len);
if((unsigned int)test != data->hdr.len)
return -1;
}
return test;
}
At this point the client recived file, and it restart waiting for new messages:
int readMsg(long fd, message_t *msg)
{
if(msg == NULL || fd < 0) {errno = EINVAL; return -1;}
int test;
test = readHeader(fd, &(msg->hdr));
if(test == -1 || test == 0){return -1;}
test += readData(fd, &(msg->data));
return test;
}
This is the point where the client should simply wait because there is no income messages, insted in this case it read something that I do not know where it comes from.
When i try to print this unwanted message with GDB it prints:
{hdr = {op = 512,
sender = "\000\000\020G\032\324\t\000\000\n\000\000\000\000\030\021B\bC\n\000\000\v\000\000\000\000\021D\v\222\000"},
data = {hdr = {receiver = "\000\000\000\000\021E\022C\n\000\000\b\v\000\000\000\000\021F\020I\n\000\000\020\000\006\b\002\n\000\000\006",
len = 131072},
buf = 0x7ffff7f2f010 ""}`
Of course this is meaningless.
I hope that this description will be useful
Thank you all in advance.
Ok, I solved my issue.
As written in the comment, this problem was due to the lack of a check on partial writing.
Now the function readData() looks like this:
int readData(long fd, message_data_t *data)
{
if(data == NULL || fd < 0) {errno = EINVAL; return -1;}
int test;
char* ph;
unsigned int rd = 0;
struct iovec iov;
iov.iov_base = &(data->hdr);
iov.iov_len = sizeof(message_data_hdr_t);
test = readv(fd, &iov, 1);
if(test <= 0){return -1;}
if(data->hdr.len != 0)
{
data->buf = malloc(data->hdr.len);
if(data->buf == NULL){return -1;}
ph = data->buf;
while (rd < data->hdr.len)
{
test = read(fd, ph, data->hdr.len - rd);
if(test == -1)
return -1;
else if(test == 0)
{
errno = ENOENT;
return -1;
}
rd += test;
ph += test;
}
}
return rd;
}
and sendData():
int sendData(long fd, message_data_t *msg)
{
if(msg == NULL || fd < 0) {errno = EINVAL; return -1;}
int test;
char* ph;
unsigned int wr = 0;
struct iovec iov;
iov.iov_base = &(msg->hdr);
iov.iov_len = sizeof(message_data_hdr_t);
test = writev(fd, &(iov), 1);
if(test == -1){return -1;}
if(msg->hdr.len != 0)
{
ph = msg->buf;
while (wr < msg->hdr.len)
{
test = write(fd, ph, msg->hdr.len - wr);
if(test == -1)
return -1;
else if(test == 0)
{
errno = ENOENT;
return -1;
}
wr += test;
ph += test;
}
}
return test;
}
In this way I no longer found the error.
Thanks for the help!
I'm trying to get the source code of my website using c, I'm able to connect and everything but when I implement the recv() code, it only receives the last few bytes of the source code. I'd like to dynamically allocate space for the buffer to receive more using the C functions malloc and realloc.
This is the code I have so far:
char *buffer = NULL;
unsigned int i = 0;
unsigned long LEN = 200;
unsigned long cur_size = 0;
buffer = (char*)malloc(sizeof(char)*LEN);
do
{
if( status >= LEN )
{
cur_size += status;
buffer = (char*)realloc(buffer, cur_size);
}
status = recv(cSocket, buffer, LEN, 0);
if( status == 0 )
{
printf("Bye\n");
}
else if( status > 0 )
{
printf("%d\n", status);
}
else
{
printf("socket error=%d\n", WSAGetLastError());
break;
}
}while( status > 0 );
printf("%s\n", buffer);
It still doesn't print the whole source code. How should I go about this?
Pseudocode:
buffer = 'len chars';
loop:
if( status >= buffer ) buffer = 'resize to status chars';
status = recv(sock, buffer, len, 0);
end loop
As you resize the buffer in advance this needs to be reflected by its size. Which currently is not the case.
To fix this you could, for example, initialise cur_size with LEN by changing
unsigned long cur_size = 0;
to
unsigned long cur_size = LEN;
Assuming the fix above, you want to append to the buffer and not overwrite it with every call to recv().
To do so change this line
status = recv(cSocket, buffer, LEN, 0);
to be
status = recv(cSocket, buffer + cur_size - LEN, LEN, 0);
A more straight forward approach would be to not track the size of the buffer, but the number of bytes received and just always increase the buffer by a constant size.
Also the two calls to allocate memory can be replaced by one:
char *buffer = NULL;
unsigned long LEN = 200;
unsigned long bytes_received = 0;
unsigned long cur_size = 0;
int status = 0;
do
{
if (bytes_received >= cur_size)
{
char * tmp;
cur_size += LEN;
tmp = realloc(buffer, cur_size);
if (NULL == tmp)
{
fprintf(stderr, "realloc error=%d\n", WSAGetLastError());
break;
}
buffer = tmp;
}
status = recv(cSocket, buffer + bytes_received, LEN, 0);
if (status == 0)
{
printf("Bye\n");
}
else if (status > 0)
{
bytes_received += status;
printf("%d\n", status);
}
else /* < 0 */
{
fprintf(stderr, "socket error=%d\n", WSAGetLastError());
}
} while (status > 0);
printf("%s\n", buffer);
Well, after a bit of research, I came across this website and finally found what I was looking for.
Binary tides
Although it uses linux's fcntl, the windows equivalent is ioctlsocket which is used to set the socket's non-blocking mode.
To see the exact function, visit the website. I modified the version and set my socket to blocking mode.
int total_recv(SOCKET s)
{
int size_recv = 0, total_size = 0, block = 00;
char chunk[BUFLEN];
ioctlsocket(s, FIONBIO, (unsigned long*)&block); // set mode to block
// not necessary but clarification of function, mode is block by
// default
while( 1 )
{
memset(chunk, 0, BUFLEN);
if( ( size_recv = recv(s, chunk, BUFLEN, 0) ) == SOCKET_ERROR )
{
printf("Error receiving\n");
}
else if( size_recv == 0 )
{
break;
}
else
{
total_size += size_recv;
// i used file since console wouldn't show full source code
FILE *fp = NULL;
fp = fopen("source.txt", "a");
fprintf(fp, chunk);
fclose(fp);
}
}
return total_size;
}
Here is my code:
// init
int total;
struct aiocb aio;
bzero((char *)&aio, sizeof(struct aiocb));
aio.aio_buf = malloc(BUF_MAX1+1);
aio.aio_nbytes = BUF_MAX1;
aio.aio_fildes = write_file;
aio.aio_offset = 0;
int ret;
while (total <= 900008200){ // 858MB
ret = aio_write(&aio);
assert(ret >= 0);
while ((ret = aio_error(&aio)) == EINPROGRESS);
assert(ret == 0);
int n;
if ((n = aio_return(&aio)) > 0) {
total += n;
aio.aio_offset += n;
} else {
assert(0);
}
}
When i run in ext4 filesystem, i got this results "total(900008200) time(12.63s)"
In btr filesystem the result is "total(900008200) time(27.42s)"
Below is my code:
#include <sys/types.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <db.h>
#define DATABASE "access.db"
typedef struct {
char data1[20];
char src[20];
} pearson_record;
I am geting the error:
illegal flag specified to DB->get
DB->get: Invalid argument
Any idea where I am going wrong.
int
main()
{
pearson_record s;
char *papa="1.1.1.1";
char *source="papa";
DB *dbp;
DBT key, data;
int ret, t_ret;
db_recno_t recno;
if ((ret = db_create(&dbp, NULL, 0)) != 0) {
fprintf(stderr, "db_create: %s\n", db_strerror(ret));
exit (1);
}
if ((ret = dbp->open(dbp,
NULL, DATABASE, NULL, DB_BTREE, DB_CREATE, 0664)) != 0) {
dbp->err(dbp, ret, "%s", DATABASE);
goto err;
}
recno = 10;
#define BUFFER_LENGTH (5 * 1024 * 1024)
data.ulen = BUFFER_LENGTH;
data.flags = DB_DBT_USERMEM;
strncpy(s.data1, papa, strlen(papa)+1);
strncpy(s.src, source, strlen(source)+1);
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
//memset(&s, 0, sizeof(struct pearson_record));
key.data = &recno;
key.size = sizeof(recno);
data.data = &s;
data.size = sizeof(s);
papa="1.1.1.2";
source="papaa";
strncpy(s.data1, papa, strlen(papa)+1);
strncpy(s.src, source, strlen(source)+1);
if ((ret = dbp->put(dbp, NULL, &key,&data,0)) == 0)
printf("db: %d: key stored.\n", *(int *)key.data);
else
{
dbp->err(dbp, ret, "DB->put");
goto err;
}
recno = 11;
strncpy(s.data1, papa, strlen(papa)+1);
strncpy(s.src, source, strlen(source)+1);
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
//memset(&s, 0, sizeof(struct pearson_record));
key.data = &recno;
key.size = sizeof(recno);
data.data = &s;
data.size = sizeof(s);
papa="1.1.1.2";
source="papaa";
strncpy(s.data1, papa, strlen(papa)+1);
strncpy(s.src, source, strlen(source)+1);
if ((ret = dbp->put(dbp, NULL, &key,&data,0)) == 0)
printf("db: %d: key stored.\n", *(int *)key.data);
else
{
dbp->err(dbp, ret, "DB->put");
goto err;
}
pearson_record *ppr;
if ((ret = dbp->get(dbp, NULL, &key, &data, DB_SET_RECNO)) == 0) {
ppr = (pearson_record *) data.data;
printf("db: %d: key retrieved: data was %s,%s. %d\n",
*(int *)key.data, ppr->data1,ppr->src, data.size);
}
else {
dbp->err(dbp, ret, "DB->get");
goto err;
}
if ((ret = dbp->get(dbp, NULL, &key, &data, DB_SET_RECNO)) == 0) {
ppr = (pearson_record *) data.data;
printf("db: %d: key retrieved: data was %s,%s. %d\n",
*(int *)key.data, ppr->data1,ppr->src, data.size);
}
else {
dbp->err(dbp, ret, "DB->get");
goto err;
}
err: if ((t_ret = dbp->close(dbp, 0)) != 0 && ret == 0)
ret = t_ret;
exit(ret);
}
A quick glance at the documentation for DB->get says: "For DB_SET_RECNO to be specified, the underlying database must be of type Btree, and it must have been created with the DB_RECNUM flag."
It doesn't look like you created the database with that flag. I haven't looked through the rest of your code, but that's one obvious place that needs to be fixed.
You can also ask these questions on the online Berkeley DB forum.
Regards,
Dave
I am trying to develop a database using Berkeley Db in C. I want to have multiple data inside the database and then access them. my code is below:
#include <sys/types.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <db.h>
#define DATABASE "access.db"
typedef struct {
int id;
char data1[20];
char src[20];
} pearson_record;
int
main()
{
pearson_record s;
char *papa="1.1.1.1";
char *source="papa";
DB *dbp;
DBT key, data;
int ret, t_ret;
if ((ret = db_create(&dbp, NULL, 0)) != 0) {
fprintf(stderr, "db_create: %s\n", db_strerror(ret));
exit (1);
}
if ((ret = dbp->open(dbp, NULL, DATABASE, NULL, DB_BTREE, DB_CREATE, 0664)) != 0) {
dbp->err(dbp, ret, "%s", DATABASE);
goto err;
}
s.id = 10;
strncpy(s.data1, papa, strlen(papa)+1);
strncpy(s.src, source, strlen(source)+1);
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
//memset(&s, 0, sizeof(struct pearson_record));
key.data = &(s.id);
key.size = sizeof(int);
data.data = &s;
data.size = sizeof(s);
papa="1.1.1.2";
source="papaa";
strncpy(s.data1, papa, strlen(papa)+1);
strncpy(s.src, source, strlen(source)+1);
if ((ret = dbp->put(dbp, NULL, &key,&data,DB_NOOVERWRITE)) == 0)
printf("db: %d: key stored.\n", *(int *)key.data);
else
{
dbp->err(dbp, ret, "DB->put");
goto err;
}
s.id = 11;
strncpy(s.data1, papa, strlen(papa)+1);
strncpy(s.src, source, strlen(source)+1);
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
//memset(&s, 0, sizeof(struct pearson_record));
key.data = &(s.id);
key.size = sizeof(int);
data.data = &s;
data.size = sizeof(s);
papa="1.1.1.2";
source="papaa";
strncpy(s.data1, papa, strlen(papa)+1);
strncpy(s.src, source, strlen(source)+1);
if ((ret = dbp->put(dbp, NULL, &key,&data,DB_NOOVERWRITE)) == 0)
printf("db: %d: key stored.\n", *(int *)key.data);
else
{
dbp->err(dbp, ret, "DB->put");
goto err;
}
pearson_record *ppr;
if ((ret = dbp->get(dbp, NULL, &key, &data, DB_MULTIPLE)) == 0) {
ppr = (pearson_record *) data.data;
printf("db: %d: key retrieved: data was %s,%s. %d\n",
*(int *)key.data, ppr->data1,ppr->src, data.size);
} else {
dbp->err(dbp, ret, "DB->get");
goto err;
}
if ((ret = dbp->get(dbp, NULL, &key, &data, DB_MULTIPLE)) == 0) {
ppr = (pearson_record *) data.data;
printf("db: %d: key retrieved: data was %s,%s. %d\n",
*(int *)key.data, ppr->data1,ppr->src, data.size);
} else {
dbp->err(dbp, ret, "DB->get");
goto err;
}
err:
if ((t_ret = dbp->close(dbp, 0)) != 0 && ret == 0)
ret = t_ret;
exit(ret);
}
Right now it says DB_DBT_MULTIPLE has to be set. I don't know how to do that. Previously without the flags i was only getting the last entry. Any kind of help would be appreciated. Thanks in advance
If I'm reading your code correctly, you are calling get twice with the same key, which returns the same data for obvious reasons.
If you want to iterate over the records, you will need to use a Cursor, see Chapter 4 of Getting Started with Berkeley DB for a good reference for same.