What is the frequency of epoll_wait "wakeups"? - c

Question

This question is elaboration and continuation of my previous question - On epoll_pwait, POSIX timers and X11 events. Most of X11 events is either delayed or dropped. I work with XCB window that displays some graphical information. The application has an insteractive input and had more or less static display. Now the requirenments had changed and I need to add some pereodic computations and have a interactive input.
The problem with the new requirenments is that computations goes at high rate. With that, I get inconsistent interaction with the XCB window. Input may lag behind, or change the rate of processing.
The setup is multiplex events with epoll_pwait. The events are signals, X11 events and recently added timers/timeout.
What I understand as of now, I need to separate user interaction from the computations. The problem with my setup, as of now, is that rate of X11 events changes in a way I can't explain.
So I decide to separate waiting on X11 events with the rest of the logic. Can you suggest a proper way to do it? Will having a X11 window in a separate thread/process/epoll set help?
And the actual question, as I look upon it now, is, what is a frequency of epoll_wait wakeups? I plan to have one epoll_wait in a loop. Maybe some processes to be wait on. I understand that epoll_wait will "wakeup" at some random points int time.
Update
My setup is close to this one:
#include <stdbool.h>
#include <string.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include <time.h>
#include <math.h>
#include <sys/types.h>
#include <sys/ioctl.h>
#include <sys/signalfd.h>
#include <sys/epoll.h>
#include <sys/timerfd.h>
#include <sys/socket.h>
#include <xcb/xcb.h>
static struct timespec tm_p, tm_c, tm_diff;
static unsigned long nevents = 0;
static inline void timespec_diff(struct timespec *a, struct timespec *b, struct timespec *res) {
res->tv_sec = a->tv_sec - b->tv_sec;
res->tv_nsec = a->tv_nsec - b->tv_nsec;
if (res->tv_nsec < 0) {
--res->tv_sec;
res->tv_nsec += 1000000000L;
}
}
static double compute(){
double t = 1.0;
double eps = 1e-7;
double eps_2 = eps / 2.0;
clock_gettime(CLOCK_MONOTONIC, &tm_p);
while(t > eps){
t -= eps;
t += eps;
t -= eps_2 + eps_2;
}
clock_gettime(CLOCK_MONOTONIC, &tm_c);
timespec_diff(&tm_c, &tm_p, &tm_diff);
printf(" compute: %ld %f\n", tm_diff.tv_sec, tm_diff.tv_nsec / 1e9);
return (int)t;
}
/* defines for epoll */
#define MAXEVENTS 64
#define SET_EV(_ev,_fd,_events) _ev.data.fd = _fd; _ev.events = _events
static int xcb_get_atom(xcb_connection_t *c,
const char *name,
xcb_atom_t *atom)
{
xcb_intern_atom_cookie_t cookie;
xcb_generic_error_t *error;
xcb_intern_atom_reply_t *reply;
cookie = xcb_intern_atom(c, 0, strlen(name), name);
reply = xcb_intern_atom_reply(c, cookie, &error);
if(NULL == reply){
free(error);
return -1;
}
*atom = reply->atom;
free(reply);
return 0;
}
static int xcb_change_prop_wm_close(xcb_connection_t *c,
xcb_window_t window,
xcb_atom_t wm_p,
xcb_atom_t atom)
{
xcb_void_cookie_t cookie;
xcb_generic_error_t *error;
xcb_atom_enum_t type = XCB_ATOM_ATOM;
uint8_t format = 32;
uint32_t data_len = 1;
cookie = xcb_change_property_checked(c, /* xcb connection */
XCB_PROP_MODE_REPLACE, /* mode */
window, /* window */
wm_p, /* the property to change */
type, /* type of the property */
format, /* format(bits) */
data_len, /* number of elements(see format) */
&atom /* property data */
);
error = xcb_request_check(c, cookie);
if (error) {
free(error);
return -1;
}
return 0;
}
int main()
{
xcb_connection_t *c;
xcb_screen_t *screen;
xcb_window_t win;
xcb_atom_t a_wm_p;
xcb_atom_t wm_p_close;
struct epoll_event ep_ev, *ep_evs = NULL;
struct signalfd_siginfo siginf;
sigset_t mask_sigs, mask_osigs;
int sig_fd = -1, x11_fd = -1, ep_fd = -1, tm_fd = -1;
/* set up signals */
if(sigemptyset(&mask_sigs) < 0){
perror(" * sigemptyset(&mask_sigs)");
goto main_terminate;
}
if(sigaddset(&mask_sigs, SIGINT)){ /* these signals will be blocked. the signals will arrive */
perror(" * sigaddset(&mask_sigs, SIGINT)");
goto main_terminate;
}
if(sigaddset(&mask_sigs, SIGQUIT)){ /* to epoll and not to a default signal handler. */
perror(" * sigaddset(&mask_sigs, SIGQUIT)");
goto main_terminate;
}
/* save old sigmask, replace it with new sigmask */
if(sigprocmask(SIG_BLOCK, &mask_sigs, &mask_osigs) < 0){
perror(" * sigprocmask(SIG_BLOCK, &mask_sigs, &mask_osigs)");
goto main_terminate;
}
/* get signal file descriptor */
if((sig_fd = signalfd(-1, &mask_sigs, 0)) < 0){
perror(" * signalfd(-1, &mask_sigs, 0)");
goto main_terminate;
}
/* set signal fd as non-blocking */
{
int on = 1;
if(ioctl(sig_fd, FIONBIO, (char *)&on) < 0){
perror(" * ioctl(sig_fd, FIONBIO)");
goto main_terminate;
}
}
/* Open the connection to the X server */
c = xcb_connect (NULL, NULL);
/* Get the first screen */
screen = xcb_setup_roots_iterator (xcb_get_setup (c)).data;
/* Ask for our window's Id */
win = xcb_generate_id(c);
/* Create the window */
{
unsigned int cw_mask = XCB_CW_BORDER_PIXEL
| XCB_CW_EVENT_MASK
;
/* values must follow in the incresing order of the cw_mask constants */
unsigned int cw_values[] = {screen->white_pixel,
XCB_EVENT_MASK_KEY_PRESS|XCB_EVENT_MASK_KEY_RELEASE
};
xcb_create_window (c, /* Connection */
XCB_COPY_FROM_PARENT, /* depth (same as root)*/
win, /* window Id */
screen->root, /* parent window */
0, 0, /* x, y */
150, 150, /* width, height */
10, /* border_width */
XCB_WINDOW_CLASS_INPUT_OUTPUT, /* class */
screen->root_visual, /* visual */
cw_mask, cw_values); /* masks */
}
/* get x11 connection file descriptor */
x11_fd = xcb_get_file_descriptor(c);
/* atom WM_PROTOCOLS */
if(xcb_get_atom(c, "WM_PROTOCOLS", &a_wm_p) < 0){
fprintf(stderr, " %s:%s:%d\n", __FILE__, __func__, __LINE__);
return -1;
}
/* atom window close */
if(xcb_get_atom(c, "WM_DELETE_WINDOW", &wm_p_close) < 0){
fprintf(stderr, " %s:%s:%d\n", __FILE__, __func__, __LINE__);
return -1;
}
{ /* wm prop: intercept close */
if(xcb_change_prop_wm_close(c, win, a_wm_p, wm_p_close) < 0){
fprintf(stderr, " %s:%s:%d\n", __FILE__, __func__, __LINE__);
goto main_terminate;
}
}
/* create epoll set file descriptor */
if((ep_fd = epoll_create(1)) < 0){
perror(" * epoll_create");
goto main_terminate;
}
/* allocate events for epoll queue */
if(NULL == (ep_evs = (struct epoll_event*)calloc(MAXEVENTS,sizeof(ep_ev)))){
perror(" * calloc(MAXEVENTS)");
goto main_terminate;
}
{ /* fd timer */
struct itimerspec ts;
if((tm_fd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK)) < 0){
perror(" * timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK)");
goto main_terminate;
}
ts.it_value.tv_sec = 1;
ts.it_value.tv_nsec = 0;
ts.it_interval.tv_sec = 0;
ts.it_interval.tv_nsec = (unsigned long)10e6; /* 10 msec */
if(timerfd_settime(tm_fd, 0, &ts, NULL) < 0){
perror(" * timerfd_settime(tm_fd, 0, &ts, NULL)");
goto main_terminate;
}
}
/* add X11 event */
SET_EV(ep_ev,x11_fd,EPOLLIN);
if(epoll_ctl (ep_fd, EPOLL_CTL_ADD, x11_fd, &ep_ev) < 0){
perror(" * epoll_ctl x11_fd");
goto main_terminate;
}
/* add timer event */
SET_EV(ep_ev,tm_fd,EPOLLIN);
if(epoll_ctl (ep_fd, EPOLL_CTL_ADD, tm_fd, &ep_ev) < 0){
perror(" * epoll_ctl tm_fd");
goto main_terminate;
}
/* add signal event */
SET_EV(ep_ev,sig_fd,EPOLLIN);
if(epoll_ctl (ep_fd, EPOLL_CTL_ADD, sig_fd, &ep_ev) < 0){
perror(" * epoll_ctl sig_fd");
goto main_terminate;
}
/* window title */
const char *title = "epoll_pwait";
xcb_change_property (c,
XCB_PROP_MODE_REPLACE,
win,
XCB_ATOM_WM_NAME,
XCB_ATOM_STRING,
8,
strlen (title),
title );
/* Map the window on the screen */
xcb_map_window (c, win);
/* Make sure commands are sent before we pause, so window is shown */
xcb_flush (c);
while(1){
int n, i, fd, status;
bool f_compute = false;
bool f_exit_sig = false;
bool f_win_close = false;
n = epoll_pwait (ep_fd, ep_evs, MAXEVENTS, -1, &mask_sigs); /* wait, signal safe */
if(n < 0){
fprintf(stderr, " * main(): %s:%s:%d\n", __FILE__, __func__, __LINE__);
status = 1;
goto main_terminate;
}
for(i = 0; i < n; ++i){ /* service epoll events */
fd = ep_evs[i].data.fd;
if(fd == sig_fd){ /* signal */
status = read(fd, &siginf, sizeof(siginf));
if(status != sizeof(siginf)){
fprintf(stderr,"read != sizeof(siginf)");
goto main_terminate;
}
if(siginf.ssi_signo == SIGINT){
printf("got SIGINT\n");
f_exit_sig = true;
}else if(siginf.ssi_signo == SIGQUIT){
printf("got SIGQUIT\n");
f_exit_sig = true;
goto main_terminate;
}else {
printf("got unregistered signal\n");
}
}else if(fd == x11_fd){ /* x11 event */
xcb_generic_event_t *event;
while((event = xcb_poll_for_event(c))){
if (event && (event->response_type == 0)){ /* error recieved */
free(event);
goto main_terminate;
}
switch(event->response_type & ~0x80){
case XCB_CLIENT_MESSAGE: { /* window close */
xcb_client_message_event_t *ce = (xcb_client_message_event_t*)event;
if(ce->data.data32[0] == wm_p_close){ /* window should close */
f_win_close = true;
}
} break;
case XCB_KEY_PRESS: { /* keyboard key press */
printf("XCB_KEY_PRESS\n");
nevents++;
} break;
} /* end switch */
free(event);
} /* end while event loop */
}else if(fd == tm_fd){ /* timer event */
uint64_t overrun;
status = read(fd, &overrun, sizeof(uint64_t));
if(status != EAGAIN) {
//~ printf(" ! timer overruns: %lu\n", overrun);
}
f_compute = true;
}
} /* finish service epoll events */
if(f_exit_sig){ /* exit signal */
goto main_terminate;
}
if(f_win_close){ /* window close */
goto main_terminate;
}
if(f_compute){ /* do some computations */
compute();
xcb_flush(c);
}
} /* end while(1) */
main_terminate:
if(sig_fd != -1){
close(sig_fd);
}
if(tm_fd != -1){
close(tm_fd);
}
if(ep_fd != -1){
close(ep_fd);
}
if(ep_evs){
free(ep_evs);
}
xcb_disconnect(c);
if (sigprocmask(SIG_SETMASK, &mask_osigs, NULL) < 0){
perror(" * sigprocmask(SIG_SETMASK, &mask_osigs, NULL)");
}
printf("received %lu events\n", nevents);
return 0;
}
With the above, I can not reproduce input lag I do have in more verbose program. I test the above with xdotool, send some input X11 events, and my eye can't catch any visible input delay. All events get delivered. For now I can not post a full code that I have problem with.

Answer 1

what is a frequency of epoll_wait wakeups?
I'll assume you are talking about the precision of wakeups. This surely depends on kernel details, but let's just make an experiment: We can write a small program that tests how long epoll_wait takes. This will test both a timeout of zero and a timeout of 1 ms since a timeout of zero might be treated specially.
Output on my system:
empty: avg 0.015636 μs, min 0.014 μs, max 0.065 μs
sleep 1 ms: avg 1108.55 μs, min 1014.72 μs, max 1577.42 μs
epoll_wait 0: avg 0.761084 μs, min 0.38 μs, max 37.787 μs
epoll_wait 1: avg 1108.97 μs, min 1017.22 μs, max 2602.4 μs
Doing nothing (empty) measures less than a microsecond, so the following results should be somewhat reliable. The minimum time is also not zero, so the clock has enough precision for what we are doing.
Sleeping 1 ms sleeps at least 1.014 ms, but there also was a case of 1.5 ms. I guess that means that wakeups are not all that precise.
Using epoll_wait() to do nothing take less than a microsecond. More than doing nothing, but this still does basically nothing, so perhaps this really just measures the syscall overhead...?
Sleeping 1 ms with epoll_wait() behaves more or less the same as sleeping for 1 ms with nanosleep().
If you want to improve this experiment, you could actually register some FDs via epoll_ctl(). It might be that the kernel handles "empty" epoll FDs specially.
#include <stdio.h>
#include <sys/epoll.h>
#include <time.h>
#include <unistd.h>
#define MEASUREMENT_COUNT 10000
#define NUM_EVENTS 42
static int epoll_fd;
double diff_micro_sec(struct timespec *a, struct timespec *b) {
double sec = a->tv_sec - b->tv_sec;
double ns = a->tv_nsec - b->tv_nsec;
return sec * 1e6 + ns / 1e3;
}
static void empty(void) {
}
static void sleep_one_ms(void) {
struct timespec spec;
spec.tv_sec = 0;
spec.tv_nsec = 1000 * 1000;
nanosleep(&spec, NULL);
}
static void epoll_wait_0(void) {
struct epoll_event events[NUM_EVENTS];
epoll_wait(epoll_fd, events, NUM_EVENTS, 0);
}
static void epoll_wait_1(void) {
struct epoll_event events[NUM_EVENTS];
epoll_wait(epoll_fd, events, NUM_EVENTS, 1);
}
static void do_it(const char *name, void (*func)(void)) {
double sum, min, max;
struct timespec a, b;
for (int i = 0; i < MEASUREMENT_COUNT; i++) {
double diff;
clock_gettime(CLOCK_MONOTONIC, &a);
func();
clock_gettime(CLOCK_MONOTONIC, &b);
diff = diff_micro_sec(&b, &a);
if (i == 0) {
sum = diff;
min = diff;
max = diff;
} else {
sum += diff;
if (diff < min)
min = diff;
if (diff > max)
max = diff;
}
}
printf("%14s: avg %g μs, min %g μs, max %g μs\n", name, sum / MEASUREMENT_COUNT, min, max);
}
int main() {
do_it("empty", empty);
do_it("sleep 1 ms", sleep_one_ms);
epoll_fd = epoll_create(1);
do_it("epoll_wait 0", epoll_wait_0);
do_it("epoll_wait 1", epoll_wait_1);
close(epoll_fd);
return 0;
}