Unix domain sockets 100x slower on Solaris 10 than on Linux? - c

Question

I am benchmarking local socket performance on Linux and Solaris for a project. For some reason I cannot find out, performance on Solaris is roughly 100x worse than on Linux. In Linux, opening a socket, exchanging one very short (2 char) message each way and closing it takes about 10us elapsed time. On Solaris, the same thing takes about 1000us.
Set-up is Solaris 10 developer vm in Virtual Box and Linux both in the same Virtual Box and directly on the same hardware (makes no difference).
Is this a known issue with Solaris? Any ways to work around it? I cannot use a local network connection instead for reasons I cannot go into here.
Code for client and server below. Compile with "cc -fast -m64 -lrt -lsocket -lnsl -o server server.c" and the equivalent for the client. Gcc 3.4.3 as delivered with Solaris 10 gives comparable results. This code has been cut down, for example timeouts have been removed end error handling is minimal.
server.c:
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <ctype.h>
#include <sys/select.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/un.h>
#include <time.h>
#include <string.h>
#include <unistd.h>
#define DIRECTORY "sub/"
#define FULL_PATH "sub/c_socket"
#define MAX_COMMAND_LEN 8192
#define PERMISSIONS 0700
void on_error(int err, char * msg) { // simple convenient error handler
if (err == -1) { // Tests whether 'err' is -1 and
perror(msg); // prints error and msg if so.
exit(-1);
}
}
int main() {
struct sockaddr_un addr;
int srv_fd, inst_fd;
int inst_adr_size;
char c;
int ret;
char readbuf[MAX_COMMAND_LEN];
int num_read;
fd_set rfds;
int fail;
int i;
// make address
memset(&addr, 0, sizeof(addr)); // clear out addr
addr.sun_family = AF_UNIX;
strncpy(addr.sun_path, FULL_PATH, sizeof(addr.sun_path));
// Remove old pseudo file if present
ret = unlink(FULL_PATH);
if (ret == -1 && errno != ENOENT) {
on_error(ret,"\nRemoving old socket file\n");
}
// Remove old directory if present
ret = rmdir(DIRECTORY);
if (ret == -1 && errno != ENOENT) {
on_error(ret, "\nRemoving old socket directory\n");
}
// Re-create new directory with appropriate permissonsm
ret = mkdir(DIRECTORY, PERMISSIONS);
on_error(ret,"\nCreating directoroy for socket file\n");
// create server listening socket
srv_fd = socket(AF_UNIX, SOCK_STREAM, 0);
on_error(srv_fd, "\nSocket creation:\n");
// bind server listening socket to address
ret = bind(srv_fd, (struct sockaddr *) &addr, sizeof(addr));
on_error(ret, "\nSocket binding:\n");
// set file permissions for socket file (somewhat redundant)
ret = chmod(FULL_PATH, PERMISSIONS);
on_error(ret, "\nSetting socket file permissions\n");
// set socket listening and queue length
ret = listen(srv_fd, 10);
on_error(ret, "\nSet socket to listen:\n");
while(1) {
// accept requests
inst_fd = accept(srv_fd, NULL, NULL);
on_error(inst_fd, "\n accepting connection:\n");
// prepare to use select on inst_fd
FD_ZERO(&rfds);
FD_SET(inst_fd, &rfds);
// now interact with the client on the instance socket.
while(1) {
num_read = 0;
while (1) {
// read a line terminated by '\n'
ret = select(inst_fd + 1, &rfds, NULL, NULL, NULL);
on_error(ret, "\nSelect on socket\n");
if (ret == 1) {
// we can read something
ret = recv(inst_fd, readbuf+num_read, MAX_COMMAND_LEN-num_read, 0
on_error(ret, "\nrecv:\n");
if (ret == 0) {
break; // we have EOF
}
num_read += ret;
if (readbuf[num_read - 1] == '\n') {
break;
}
}
} /* reading one input line done */
if (num_read == 0) break; // EOF propagated
// process command: Just send 2 chars back
ret = send(inst_fd, "n\n", 2, 0);
}
close(inst_fd); // clean up
}
// runs forever...
}
client.c:
#include <stdio.h>
#include <time.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#define RCVBUFSIZE 8192 /* Size of receive buffer */
#define FULL_PATH "sub/c_socket"
#define CYCLES 100000
void on_error(int err, char * msg) { // more convenient error output
if (err == -1) { // Tests whether 'err' is -1 and
perror(msg); // prints error and msg if so.
exit(-1);
}
}
int main(int argc, char *argv[]) {
int client_fd;
struct sockaddr_un addr;
char readbuf[RCVBUFSIZE+1];
int num_read;
int ret;
int count;
fd_set rfds;
char * msg = "N\n";
// make address
memset(&addr, 0, sizeof(addr)); // clear out addr
addr.sun_family = AF_UNIX;
strncpy(addr.sun_path, FULL_PATH, sizeof(addr.sun_path));
for(count = 0; count < CYCLES; count++) {
// create socket
client_fd = socket(PF_UNIX, SOCK_STREAM, 0);
on_error(client_fd, "socket() failed");
// prepare to use select on inst_fd
FD_ZERO(&rfds);
FD_SET(client_fd, &rfds);
// connect
ret = connect(client_fd, (struct sockaddr *) &addr, sizeof(addr));
on_error(ret, "connect() failed");
// send msg to server
ret = send(client_fd, msg, 2, 0);
if (ret != 2) {
on_error(-1, "\nnot all bytes sent\n");
}
num_read = 0;
// read until we have a '\n'
while (1) {
ret = select(client_fd + 1, &rfds, NULL, NULL, NULL);
on_error(ret, "\nSelect on socket\n");
if (ret == 1) {
// we can read something
ret = recv(client_fd, readbuf + num_read, RCVBUFSIZE - num_read, 0)
on_error(ret, "\nrecv:\n");
num_read += ret;
if (readbuf[num_read - 1] == '\n') break;
}
}
if (num_read == 0) break;
close(client_fd);
}
return(0);
}

Answer 1

I had a similar issue when I was studying sockets and tried writing an ftp server: because of a bug in the conversion to ascii I ended up writing files one byte at a time, but on linux it was ok, while on windows I ended up with something like 100KB/s on the loop interface... if that is the case, increasing the number of bytes should lessen the difference a lot.
It seems that under linux the act of requesting a system call is simply faster.
PS
I don't know much about the internals of an operating system, so if anyone can share some pointers to understand the issue (like http://yarchive.net/comp/linux/linux_speed.html) I'd be grateful.