libuv slow in establishing a multiple connections? - c

I am trying to evaluate the performance of libuv because of the promise that it can manage 100,000 of TCP sockets and can generate 10,000 of new TCP sessions/second.
Created the following code snippet based on a gist to validate the performance of libuv. However, when it runs against a server, it's surprisingly slow according to wireshark pcap capture
TCP handshakes were done one after another, I was expecting TCP SYNs to be sent in parallel
The TCP data was sent after all the sessions were established. I had expected the TCP data to be sent as soon as the TCP handshake is complete.
This test was run on Ubuntu 14.04 (64bit core i7 cpu).
Not sure if there is problem with libuv or with this code snippet.
I know this has some memory leaks, but it doesn't matter since I am only doing evaluation.
#include <stdio.h>
#include <stdlib.h>
#include <uv.h>
//based on https://gist.githubusercontent.com/snatchev/5255976/
//raw/8392c42d719bb775053036e32b21affdf932c1b7/libuv-tcp-client.c
static void on_close(uv_handle_t* handle);
static void on_connect(uv_connect_t* req, int status);
static void on_write(uv_write_t* req, int status);
static uv_loop_t *loop;
static uv_buf_t alloc_cb(uv_handle_t* handle, size_t size) {
return uv_buf_init(malloc(size), size);
}
void on_close(uv_handle_t* handle)
{
printf("closed.");
}
void on_write(uv_write_t* req, int status)
{
if (status) {
uv_err_t err = uv_last_error(loop);
fprintf(stderr, "uv_write error: %s\n", uv_strerror(err));
return;
}
printf("wrote.\n");
free(req);
//uv_close((uv_handle_t*)req->handle, on_close);
}
void on_read(uv_stream_t* tcp, ssize_t nread, uv_buf_t buf)
{
printf("on_read. %p\n",tcp);
if(nread >= 0) {
//printf("read: %s\n", tcp->data);
printf("read: %s\n", buf.base);
}
else {
//we got an EOF
uv_close((uv_handle_t*)tcp, on_close);
}
//cargo-culted
free(buf.base);
}
void write2(uv_stream_t* stream, char *data, int len2) {
uv_buf_t buffer[] = {
{.base = data, .len = len2}
};
uv_write_t *req = malloc(sizeof(uv_write_t));
uv_write(req, stream, buffer, 1, on_write);
}
void on_connect(uv_connect_t* connection, int status)
{
if (status < 0) {
printf("failed to connect\n"); return;
}
printf("connected. %p %d\n",connection, status);
uv_stream_t* stream = connection->handle;
free(connection);
write2(stream, "echo world!", 12);
uv_read_start(stream, alloc_cb, on_read);
}
void startConn(char *host, int port) {
uv_tcp_t *pSock = malloc(sizeof(uv_tcp_t));
uv_tcp_init(loop, pSock);
uv_tcp_keepalive(pSock, 1, 60);
struct sockaddr_in dest = uv_ip4_addr(host, port);
uv_connect_t *pConn = malloc(sizeof(uv_connect_t));
printf("allocated %p\n", pConn);
uv_tcp_connect(pConn, pSock, dest, on_connect);
}
int main(int argc, char **argv) {
loop = uv_default_loop();
int i;
for (i=0; i<10; i++)
startConn("0.0.0.0", 1234);
uv_run(loop, UV_RUN_DEFAULT);
}

This example is compiled against libuv 0.1.
After it's adapted to compile and run against libuv1.x, it works as expected. Here is the gist.
There is considerable difference between libuv 0.1 and libuv 1.0. Hope this version could be helpful to those who are looking for a working example of libuv.c. By the way, there are many good examples in libuv repo in github, check the "test" subdirectory.

Related

Send WebSockets message to server

I am trying to work with an API of one device, but it is using a WS interface with enforced Origin header, which is giving me troubles.
In Chrome, I can open the Console while a page with the correct Origin is loaded, create the WS connection, and send/receive messages without difficulties:
Note that sent messages (in green) are always acknowledged by the server.
For reference, this is what happens if I create the connection on a different page, which results in an Origin header mismatch, reported as 404:
To sidestep this problem, I turned to C, because the rest of my program is written in that anyway. This is the code I have right now, based mostly on this answer:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <signal.h>
#include <libwebsockets.h>
#define KGRN "\033[0;32;32m"
#define KCYN "\033[0;36m"
#define KRED "\033[0;32;31m"
#define KYEL "\033[1;33m"
#define KBLU "\033[0;32;34m"
#define KCYN_L "\033[1;36m"
#define KBRN "\033[0;33m"
#define RESET "\033[0m"
static int destroy_flag = 0;
static int connection_flag = 0;
static int writeable_flag = 0;
static void INT_HANDLER(int signo) {
destroy_flag = 1;
}
struct session_data {
int fd;
};
struct pthread_routine_tool {
struct lws_context *context;
struct lws *wsi;
};
static int websocket_write_back(struct lws *wsi_in, char *str, int str_size_in)
{
if (str == NULL || wsi_in == NULL)
return -1;
int n;
int len;
char *out = NULL;
if (str_size_in < 1)
len = strlen(str);
else
len = str_size_in;
out = (char *)malloc(sizeof(char)*(LWS_SEND_BUFFER_PRE_PADDING + len + LWS_SEND_BUFFER_POST_PADDING));
//* setup the buffer*/
memcpy (out + LWS_SEND_BUFFER_PRE_PADDING, str, len );
//* write out*/
n = lws_write(wsi_in, out + LWS_SEND_BUFFER_PRE_PADDING, len, LWS_WRITE_TEXT);
printf(KBLU"[websocket_write_back] %s\n"RESET, str);
//* free the buffer*/
free(out);
return n;
}
static int ws_service_callback(
struct lws *wsi,
enum lws_callback_reasons reason, void *user,
void *in, size_t len)
{
switch (reason) {
case LWS_CALLBACK_CLIENT_ESTABLISHED:
printf(KYEL"[Main Service] Connect with server success.\n"RESET);
connection_flag = 1;
break;
case LWS_CALLBACK_CLIENT_CONNECTION_ERROR:
printf(KRED"[Main Service] Connect with server error.\n"RESET);
destroy_flag = 1;
connection_flag = 0;
break;
case LWS_CALLBACK_CLOSED:
printf(KYEL"[Main Service] LWS_CALLBACK_CLOSED\n"RESET);
destroy_flag = 1;
connection_flag = 0;
break;
case LWS_CALLBACK_CLIENT_RECEIVE:
printf(KCYN_L"[Main Service] Client recvived:%s\n"RESET, (char *)in);
if (writeable_flag)
destroy_flag = 1;
break;
case LWS_CALLBACK_CLIENT_WRITEABLE :
printf(KYEL"[Main Service] On writeable is called. send byebye message\n"RESET);
websocket_write_back(wsi, "{\"command\":\"subscribe\",\"identifier\":\"{\\\"channel\\\":\\\"DevicesChannel\\\",\\\"share_token\\\":\\\"D0E91\\\"}\"}", -1);
websocket_write_back(wsi, "{\"command\":\"message\",\"identifier\":\"{\\\"channel\\\":\\\"DevicesChannel\\\",\\\"share_token\\\":\\\"D0E91\\\"}\",\"data\":\"{\\\"value\\\":100,\\\"action\\\":\\\"set_buzz\\\"}\"}", -1);
writeable_flag = 1;
break;
default:
break;
}
return 0;
}
static void *pthread_routine(void *tool_in)
{
struct pthread_routine_tool *tool = tool_in;
printf(KBRN"[pthread_routine] Good day. This is pthread_routine.\n"RESET);
//* waiting for connection with server done.*/
while(!connection_flag)
usleep(1000*20);
//*Send greeting to server*/
lws_callback_on_writable(tool->wsi);
}
int main(void)
{
//* register the signal SIGINT handler */
struct sigaction act;
act.sa_handler = INT_HANDLER;
act.sa_flags = 0;
sigemptyset(&act.sa_mask);
sigaction( SIGINT, &act, 0);
struct lws_context *context = NULL;
struct lws_context_creation_info info;
struct lws *wsi = NULL;
struct lws_protocols protocol;
memset(&info, 0, sizeof info);
info.port = CONTEXT_PORT_NO_LISTEN;
info.iface = NULL;
info.protocols = &protocol;
info.ssl_cert_filepath = NULL;
info.ssl_private_key_filepath = NULL;
info.extensions = lws_get_internal_extensions();
info.gid = -1;
info.uid = -1;
info.options = 0;
protocol.name = "websockets";
protocol.callback = &ws_service_callback;
protocol.per_session_data_size = sizeof(struct session_data);
protocol.rx_buffer_size = 0;
protocol.id = 0;
protocol.user = NULL;
context = lws_create_context(&info);
printf(KRED"[Main] context created.\n"RESET);
if (context == NULL) {
printf(KRED"[Main] context is NULL.\n"RESET);
return -1;
}
wsi = lws_client_connect(context, "mobu1.herokuapp.com", 443, 1,
"/cable", "mobu1.herokuapp.com", "link.motorbunny.com",
if (wsi == NULL) {
printf(KRED"[Main] wsi create error.\n"RESET);
return -1;
}
printf(KGRN"[Main] wsi create success.\n"RESET);
struct pthread_routine_tool tool;
tool.wsi = wsi;
tool.context = context;
pthread_t pid;
pthread_create(&pid, NULL, pthread_routine, &tool);
pthread_detach(pid);
while(!destroy_flag)
{
lws_service(context, 50);
}
lws_context_destroy(context);
return 0;
}
The result of running the above program is this:
As you can see, the periodic pings from server to my client are being picked up, but the lws_callback_on_writable(wsi); seems to have no effect as the LWS_CALLBACK_CLIENT_WRITEABLE callback never gets called. Additionally, if I call websocket_write_back() directly anywhere else, it doesn't seem to be sending anything to the server, and no acknowledgement is present either.
Is there something obvious I am doing wrong?
EDIT 1:
I found this neat wscat, where I can replicate the results from Chrome:
Now the question is, how can I interface this with my C program in a way that it can wait for the Welcome message from the server, and then send two messages?
And better yet, how to stay connected, so that my program can send multiple commands at different points of time without having to do the handshake all the time?
The reason why the LWS_CALLBACK_CLIENT_WRITEABLE callback never got called was because this particular server uses non-standard handshake. So, to bypass this, I forked a fork of libwsclient and modified the handshake checking function to not fail on mismatch. I also added an optional Origin header.
Now, all I need to do in my original program is
wsclient *client;
char sync_str[6];
void mb_send(int power, char* type)
{
char cmd[2048];
sprintf (cmd, "{\"command\":\"message\",\"identifier\":\"{\\\"channel\\\":\\\"DevicesChannel\\\",\\\"share_token\\\":\\\"%s\\\"}\",\"data\":\"{\\\"value\\\":%d,\\\"action\\\":\\\"set_%s\\\"}\"}",sync_str,power,type);
libwsclient_send(client,cmd);
}
void mb_connect()
{
char cmd[2048];
sprintf (cmd, "{\"command\":\"subscribe\",\"identifier\":\"{\\\"channel\\\":\\\"DevicesChannel\\\",\\\"share_token\\\":\\\"%s\\\"}\"}",sync_str);
libwsclient_send(client,cmd);
mb_send(0,"buzz");
}
int nop()
{
return 0;
}
int main()
{
client = libwsclient_new_extra("wss://mobu1.herokuapp.com/cable","https://link.motorbunny.com");
if(!client) {
fprintf(stderr, "Unable to initialize new WS client.\n");
exit(1);
}
libwsclient_onopen(client, &nop);
libwsclient_onmessage(client, &nop);
libwsclient_onerror(client, &nop);
libwsclient_onclose(client, &nop);
libwsclient_run(client);
...
mb_connect();
...
mb_send(200,"buzz");
mb_send(40,"twirl");
...
mb_send(0,"buzz");
mb_send(0,"twirl");
}
I found an ugly hack to make my C program send WebSocket messages to a server via the wsta program.
It requires a text file, into which my program will append whenever it wants to send a message to the server. The new lines are then picked up in the background by tail -f, and are piped to wsta which maintains the connection. Output can be redirected to /dev/null so that the wsta output doesn't pollute the output of my program, or sent to a file if responses from the server need to be parsed.
The whole script to make this work would look like this (or you could use FIFO pipe with cat instead of a file with tail):
#!/bin/bash
touch commands.txt
tail commands.txt -f -n 0 | wsta --header "Origin: https://link.motorbunny.com" "wss://mobu1.herokuapp.com/cable" &> /dev/null &
./program
In the C program, I just need to write to the commands.txt file:
FILE* cmd;
char sync_str[6];
void mb_connect()
{
fprintf (cmd, "{\"command\":\"subscribe\",\"identifier\":\"{\\\"channel\\\":\\\"DevicesChannel\\\",\\\"share_token\\\":\\\"%s\\\"}\"}\n",sync_str);
fflush(cmd);
}
void mb_send(int power, char* type)
{
fprintf (cmd, "{\"command\":\"message\",\"identifier\":\"{\\\"channel\\\":\\\"DevicesChannel\\\",\\\"share_token\\\":\\\"%s\\\"}\",\"data\":\"{\\\"value\\\":%d,\\\"action\\\":\\\"set_%s\\\"}\"}\n",sync_str,power,type);
fflush(cmd);
}
int main()
{
cmd = fopen ("commands.txt","w");
...
mb_connect();
...
mb_send(200,"buzz");
...
mb_send(0,"buzz");
}

Intercepting winsock recvfrom function gives invalid address error

I'm trying to intercept winsock2 calls to troubleshoot a case where UDP packets between processes on the same host intermittently vanish. When they "vanish" they vanish for something like 20 seconds, which is enough to abort the process being run.
I managed to get the DLL injection to work, and 2 winsock calls are intercepted correctly (I hope) because I can do a simple print, but I need to be able to process the address information so I can track who sends what to whom...
Unfortunately, the upstream process I'm injecting into is apparently calling recvfrom (Yes, not the WSA... equivalent, this is old ported POSIX code) with NULL's for the "from" and "fromlen" parameters. If I don't fiddle with them at all the receive works. If I do the below, I get "Invalid Address" errors (Winsock error 10014).
An I doing something stupid here? Not that I would be surprised.... And yes, it would be easier to rebuild the target application with the desired debugging, but the build environment is down due to a data center migration, and I need ammunition to say "it's not the application."
// dllmain.cpp : Defines the entry point for the DLL application.
#include "stdafx.h"
#include <stdio.h>
#include <stdlib.h>
#include "mhook.h"
#include <winsock2.h>
// typedefs for function pointers...
typedef int(WINAPI *rcvfrom_ptr) (
SOCKET s,
char *buf,
int len,
int flags,
struct sockaddr *from,
int *fromlen
);
typedef int(WINAPI *sendto_ptr) (
_In_ SOCKET s,
_In_ const char *buf,
_In_ int len,
_In_ int flags,
_In_ const struct sockaddr *to,
_In_ int tolen
);
// Function pointers for original calls.
rcvfrom_ptr orig_rcvfrom;
sendto_ptr orig_sendto;
//
// Helper functions.
//
typedef union sockaddrs {
struct sockaddr from;
struct sockaddr_in in_from;
// Need to verify Ipv6 support. may need to remigrate back to VS 2015
//struct sockaddr_in6 in6_from;
} tsockaddrs;
char *printaddr(char *buff,const int buffsz, const tsockaddrs *addr)
{
...
}
// Shim functions.
//
// Still working on getting them to actually work and do what I need.
// But I may as well develop the skeleton
int WINAPI Myrecvfrom(SOCKET s,
char *buf,
int len,
int flags,
struct sockaddr *from,
int *fromlen
)
{
int result;
struct sockaddr *all_froms;
char addrbuff[100] = "";
int newfromlen = sizeof(struct sockaddr);
all_froms = (struct sockaddr *)malloc(sizeof(struct sockaddr));
memset((void *)all_froms,0,sizeof(struct sockaddr));
printf("Receiving Packet!\n");
if (from == NULL) {
printf("\tFrom addr == null, using internal structures\n");
result = (orig_rcvfrom)(s, buf, len, flags, all_froms, &newfromlen);
} else {
printf("\tFrom addr != null, using passed structures\n");
result = (orig_rcvfrom)(s, buf, len, flags, from, fromlen);
memcpy_s((void*)&all_froms,sizeof(all_froms),(void *)from,*fromlen);
if (fromlen != NULL) {
newfromlen=*fromlen;
} else {
newfromlen=sizeof(struct sockaddr);
}
}
if (result >0) {printf("received %d bytes\n",result);}
else if (result == SOCKET_ERROR) {printf("Socket Error %d occurred!\n",WSAGetLastError());}
if (newfromlen >0) {
if (printaddr(addrbuff,sizeof(addrbuff),(tsockaddrs *)all_froms)!=NULL) {
printf("received %d bytes from on port %d from host %s\n",result,((tsockaddrs *)(all_froms))->in_from.sin_port,addrbuff);
}
if (from != NULL) {
memcpy_s((void*)from,sizeof(struct sockaddr),(void*)&all_froms,newfromlen);
}
else
{
}
if (fromlen != NULL) *fromlen=newfromlen;
}
else {
printf("received %d bytes from unknown port and host\n",result);
}
if (all_froms != NULL) {free(all_froms);}
return result;
}
int WINAPI Mysendto(SOCKET s,
const char *buf,
int len,
int flags,
const struct sockaddr *to,
int tolen
)
{
printf("Sending packet!\n");
return orig_sendto(s, buf, len, flags, to, tolen);
}
BOOL AttachHooks(void)
{
BOOL sethooks;
orig_rcvfrom = (rcvfrom_ptr)GetProcAddress(GetModuleHandle(L"ws2_32"), "recvfrom");
sethooks = Mhook_SetHook((PVOID*)&orig_rcvfrom, Myrecvfrom);
if (sethooks) {
orig_sendto = (sendto_ptr)GetProcAddress(GetModuleHandle(L"ws2_32"), "sendto");
sethooks &= Mhook_SetHook((PVOID*)&orig_sendto, Mysendto);
}
return sethooks;
}
BOOL APIENTRY DllMain( HMODULE hModule,
DWORD ul_reason_for_call,
LPVOID lpReserved
)
{
switch (ul_reason_for_call)
{
case DLL_PROCESS_ATTACH:
printf_s("This is an attached DLL!\n");
AttachHooks();
break;
case DLL_THREAD_ATTACH:
case DLL_THREAD_DETACH:
case DLL_PROCESS_DETACH:
break;
}
return TRUE;
}
The cause of the issue is that the ACTUAL size of the reply was more than the size of "struct sockaddr." It in fact was a SOCKADDR_IN6 structure coming back. Used an arbitrary buffer of 200 bytes instead of the 28-byte sockaddr structure and I was able to pull in the data.
In the process, I wound up converting to "peeking" the received data instead of processing it after it was pulled in. When the PC with the source code finishes installing updates, I'll post it.
Working from home is fairly isolating, and there isn't anyone who lives in winsock in my office anyway. It was really helpful to "chat" in the comments.

pf_ring and libpcap if_index not returning

I have recently been getting involved with having to utilize pf_ring / libpcap. I have never developed with libpcap or pf_ring so please forgive what might appear to be a silly question, as network programming is semi new to me... In broad terms what I am trying to do is access the if_index for packets received. I currently have a simple raw packet sniffer created with "C" utilizing pf_ring as shown below:
#include <pcap.h>
#include <pfring.h>
#include <string.h>
#include <stdlib.h>
#define MAXBYTES2CAPTURE 2048
void processRingPacket(const struct pfring_pkthdr* pkthdr, const u_char* packet, const u_char *arg)
{
int i=0, *counter = (int*)arg;
printf("Packet Count: %d ", ++(*counter));
printf("Received Packet Size: %d ", pkthdr->len);
printf("ifIndex: %d ", pkthdr->extended_hdr.if_index);
printf("Payload:\n");
for(i=0; i < pkthdr->len; i++)
{
if(isprint(packet[i]))
{
printf("%c ", packet[i]);
}
else
{
printf(". ");
}
if((i % 16 == 0) && (i != 0) || (i == pkthdr->len-1))
{
printf("\n");
}
}
return;
}
int main()
{
int count = 0;
char *device = "eth0";
printf("Opening Device: %s\n", device);
pfring* ring = pfring_open(device, MAXBYTES2CAPTURE, 0);
pfring_enable_ring(ring);
pfring_loop(ring, processRingPacket, (u_char*)&count, 1);
return 0;
}
Looking at the pfring_pkthdr struct within the pf_ring API, I should be able to do the following:
pkthdr->extended_hdr.if_index
However, when I try to print out the index it just prints 0. I am guessing the if_index is not actually being set, as when I actually call the pf_ring function to get the device if index, I actually receive a value for the specified device:
pfring_get_device_ifindex (pfring *ring, char *device_name, int *if_index)
The problem is I am trying to view the if_index for each packet, hence within the call back function "processRingPacket" there is no way to generically specify the device. I say generically here because there will be two interfaces capturing packets. Any ideas on what my rookie mistake might be?
I think you need to pass in PF_RING_LONG_HEADER as a flag to pfring_open(). So it becomes, pfring_open(device, MAXBYTES2CAPTURE, PF_RING_LONG_HEADER);
If pkthdr->extended_hdr.if_index isn't set in the callback function, you can always pass it in to your callback function in the arg argument.
struct Dev {
int count;
int if_index;
};
...
char *device = "eth0";
struct Dev dev;
dev.count = 0;
dev.if_index = if_nametoindex(device); //from #include <net/in.h>
printf("Opening Device: %s\n", device);
pfring* ring = pfring_open(device, MAXBYTES2CAPTURE, 0);
pfring_enable_ring(ring);
pfring_loop(ring, processRingPacket, (u_char*)&dev, 1);
And recover that in the callback function:
void processRingPacket(const struct pfring_pkthdr* pkthdr, const u_char* packet, const u_char *arg)
{
struct Dev *dev = (struct Dev*)arg;
int i=0, *counter = (int*)&dev->count;
//and use dev->if_index; whenever you need to.

Communicate with zeromq with protobuf serialized messages

I am unable to receive message serialized in protobuf over ZeroMQ sockets using C.
I have serialized message entered by client and send this buffer to server using s_send() function defined in zhelpers.h. The server code is same test code bundled with zeromq package as an examples.
Here is my client side:
#include "amessage.pb-c.h"
#include "zhelpers.h"
int main (void)
{
AMessage msg = AMESSAGE__INIT; // AMessage
void *buf; // Buffer to store serialized data
unsigned len;
printf ("Connecting to server...\n");
void *context = zmq_ctx_new ();
void *requester = zmq_socket (context, ZMQ_REQ);
char buffer[256] = "";
printf("[client] :");
scanf("%s", buffer );
msg.csmsg = buffer;
len = amessage__get_packed_size(&msg);
buf = malloc(len);
printf("[client]: pack msg len : %d\n ", len);
printf("Sent msg : %d\n", buf);
amessage__pack(&msg,buf);
s_send(requester, buf);
zmq_close (requester);
zmq_ctx_destroy (context);
return 0;
}
And server side:
#include "zhelpers.h"
#include <pthread.h>
#include <stdlib.h>
#include "amessage.pb-c.h"
#define MAX_MSG_SIZE 256
static size_t read_buffer (unsigned max_length, unsigned char *out)
{
size_t cur_len = 0, nread;
uint8_t c;
while ((nread=fread(out + cur_len, 1, max_length - cur_len, stdin)) != 0)
{
cur_len += nread;
if (cur_len == max_length)
{
fprintf(stderr, "[server]: max message length exceeded\n");
exit(1);
}
}
return cur_len;
}
static void * worker_routine (void *context)
{
AMessage *msg;
uint8_t buf[MAX_MSG_SIZE];
char buffer[256];
// Socket to talk to dispatcher
void *receiver = zmq_socket (context, ZMQ_REP);
zmq_connect (receiver, "inproc://workers");
while (1) {
uint8_t *string = s_recv (receiver);
if(string == 0)
printf("[server]: Error: In receiving msg.\n");
else
{
size_t msg_len = read_buffer (MAX_MSG_SIZE, string);
printf("[server]: client msg len is: %d.\n", msg_len);
msg = amessage__unpack(NULL, msg_len, string);
if (msg == NULL)
{
fprintf(stderr, "[server]: error unpacking incoming message\n");
exit(1);
}
printf ("[client]: %s \n", msg->csmsg);
}
amessage__free_unpacked(msg, NULL);
free (string);
// Do some 'work'
sleep (1);
}
zmq_close (receiver);
return NULL;
}
int main (void)
{
void *context = zmq_ctx_new ();
void *clients = zmq_socket (context, ZMQ_ROUTER);
zmq_bind (clients, "tcp://*:5555");
void *workers = zmq_socket (context, ZMQ_DEALER);
zmq_bind (workers, "inproc://workers");
// Launch pool of worker threads
int thread_nbr;
for (thread_nbr = 0; thread_nbr < 5; thread_nbr++) {
pthread_t worker;
pthread_create (&worker, NULL, worker_routine, context);
}
// Connect work threads to client threads via a queue proxy
zmq_proxy (clients, workers, NULL);
zmq_close (clients);
zmq_close (workers);
zmq_ctx_destroy (context);
return 0;
}
Any idea what I am doing wrong?
You are using s_send() which expects a C string as an argument, and calls strlen() to determine its size. However, protocol buffers data is binary data, and may contain null bytes anywhere in the message.
Instead use zmq_send() and give the length of the message to the zmq_msg_init_size() function.

linux kernel aio functionality

I am testing kernel asynchronous io functions (not posix aio) and am trying to figure out how it works. The code below is a complete program where I simply write an array repeatedly to a file opened using O_DIRECT. I get an error in the callback function "write missed bytes expect 1024 got 0" (see the fprintf statement in work_done()).
For those not familiar with kernel aio, the code below does the following:
Init some structs
Prepare aio (io_prep_pwrite)
Submit io requests (io_submit)
Check for event completion (io_getevents)
Call a callback function to see if everything went ok.
I get an error at step 5. If I do not open the file using O_DIRECT, things work fine, but it beats the purpose of having async writes.
Can someone tell me what I am doing wrong? Is this the correct usage of kernel aio, for example, is my use of callbacks correct? Are there any restrictions on the usage of O_DIRECT?
I compile using 'gcc -Wall test.c -laio'
Thanks in advance.
/*
* File: myaiocp.c
* Author: kmehta
*
* Created on July 11, 2011, 12:50 PM
*
*
* Testing kernel aio.
* Program creates a 2D matrix and writes it multiple times to create a file of desired size.
* Writes are performed using kernel aio functions (io_prep_pwrite, io_submit, etc.)
*/
#define _GNU_SOURCE
#define _XOPEN_SOURCE 600
#include <stdio.h>
#include <stdlib.h>
#include <getopt.h>
#include <pthread.h>
#include <fcntl.h>
#include <string.h>
#include <sys/uio.h>
#include <sys/time.h>
#include <omp.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>
#include <libaio.h>
char ** buf;
long seg_size;
int seg_rows;
double total_size;
char * filename;
static int wait_count = 0;
void io_task();
void cleanup();
void allocate_2D_matrix(int[]);
int file_open(char *);
void wr_done(io_context_t ctx, struct iocb* iocb, long res, long res2);
int main(int argc, char **argv) {
total_size = 1048576; //1MB
seg_size = 1024; //1kB
seg_rows = 1024;
filename = "aio.out";
int dims[] = {seg_rows, seg_size};
allocate_2D_matrix(dims); //Creates 2D matrix
io_task();
cleanup();
return 0;
}
/*
* Create a 2D matrix
*/
void allocate_2D_matrix(int dims[2]) {
int i;
char *data;
//create the matrix
data = (char *) calloc(1, dims[0] * dims[1] * sizeof (char));
if (data == NULL) {
printf("\nCould not allocate memory for matrix.\n");
exit(1);
}
buf = (char **) malloc(dims[0] * sizeof (char *));
if (buf == NULL) {
printf("\nCould not allocate memory for matrix.\n");
exit(1);
}
for (i = 0; i < dims[0]; i++) {
buf[i] = &(data[i * dims[1]]);
}
}
static void io_error(const char *func, int rc)
{
if (rc == -ENOSYS)
fprintf(stderr, "AIO not in this kernel\n");
else if (rc < 0)
fprintf(stderr, "%s: %s\n", func, strerror(-rc));
else
fprintf(stderr, "%s: error %d\n", func, rc);
exit(1);
}
/*
* Callback function
*/
static void work_done(io_context_t ctx, struct iocb *iocb, long res, long res2)
{
if (res2 != 0) {
io_error("aio write", res2);
}
if (res != iocb->u.c.nbytes) {
fprintf(stderr, "write missed bytes expect %lu got %ld\n",
iocb->u.c.nbytes, res2);
exit(1);
}
wait_count --;
printf("%d ", wait_count);
}
/*
* Wait routine. Get events and call the callback function work_done()
*/
int io_wait_run(io_context_t ctx, long iter)
{
struct io_event events[iter];
struct io_event *ep;
int ret, n;
/*
* get up to aio_maxio events at a time.
*/
ret = n = io_getevents(ctx, iter, iter, events, NULL);
printf("got %d events\n", n);
/*
* Call the callback functions for each event.
*/
for (ep = events ; n-- > 0 ; ep++) {
io_callback_t cb = (io_callback_t)ep->data ; struct iocb *iocb = ep->obj ; cb(ctx, iocb, ep->res, ep->res2);
}
return ret;
}
void io_task() {
long offset = 0;
int bufIndex = 0;
//Open file
int fd = file_open(filename);
//Initialize structures
long i;
long iter = total_size / seg_size; //No. of iterations to reach desired file size (total_size)
io_context_t myctx;
if(0 != io_queue_init(iter, &myctx))
{
perror("Could not initialize io queue");
exit(EXIT_FAILURE);
}
struct iocb * ioq[iter];
//loop through iter times to reach desired file size
for (i = 0; i < iter; i++) {
struct iocb *io = (struct iocb*) malloc(sizeof (struct iocb));
io_prep_pwrite(io, fd, buf[bufIndex], seg_size, offset);
io_set_callback(io, work_done);
ioq[i] = io;
offset += seg_size;
bufIndex ++;
if (bufIndex > seg_rows - 1) //If entire matrix written, start again from index 0
bufIndex = 0;
}
printf("done preparing. Now submitting..\n");
if(iter != io_submit(myctx, iter, ioq))
{
perror("Failure on submit");
exit(EXIT_FAILURE);
}
printf("now awaiting completion..\n");
wait_count = iter;
int res;
while (wait_count) {
res = io_wait_run(myctx, iter);
if (res < 0)
io_error("io_wait_run", res);
}
close(fd);
}
void cleanup() {
free(buf[0]);
free(buf);
}
int file_open(char *filename) {
int fd;
if (-1 == (fd = open(filename, O_DIRECT | O_CREAT | O_WRONLY | O_TRUNC, 0666))) {
printf("\nError opening file. \n");
exit(-1);
}
return fd;
}
First of all, good job using libaio instead of POSIX aio.
Are there any restrictions on the usage of O_DIRECT ?
I'm not 100% sure this is the real problem, but O_DIRECT has some requirements (quoting mostly from TLPI):
The data buffer being transferred must be aligned on a memory boundary that is a multiple of the block size (use posix_memalign)
The file or device offset at which data transfer commences must be a multiple of the block size
The length of the data to be transferred must be a multiple of the block size
At a glance, I can see you are not taking aby precautions to align memory in allocate_2D_matrix.
If I do not open the file using O_DIRECT, things work fine, but it
beats the purpose of having async writes.
This happens not to be the case. Asynchronous I/O works well without O_DIRECT (for instance think of the number of system calls slashed).

Resources