OpenSSL DTLS connection never establishes - c

Question

I'm sorry for such a big block of code, but I'm so lost as to what is happening, I have no idea where the problem might be...
I am trying to get a VERY minimal dtls server going and I can't get the client and server to finish handshaking.
Here is my code:
#include <stdio.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <pthread.h>
#include <openssl/ssl.h>
#include <openssl/bio.h>
#include <openssl/err.h>
#include <openssl/rand.h>
#include <openssl/opensslv.h>
int generate_cookie(SSL *ssl, unsigned char *cookie, unsigned int *cookie_len)
{
cookie = "cookie";
cookie_len = 5;
return 1;
}
int verify_cookie(SSL *ssl, const unsigned char *cookie, unsigned int cookie_len)
{
return 1;
}
int dtls_verify_callback (int ok, X509_STORE_CTX *ctx) {
/* This function should ask the user
* if he trusts the received certificate.
* Here we always trust.
*/
return 1;
}
int main() {
char buff[FILENAME_MAX];
getcwd(buff, FILENAME_MAX);
union {
struct sockaddr_storage ss;
struct sockaddr_in s4;
struct sockaddr_in6 s6;
} client_addr;
struct sockaddr_in server_addr;
const int on = 1, off = 0;
memset(&server_addr, 0, sizeof(server_addr));
memset(&client_addr, 0, sizeof(client_addr));
int res;
SSL *ssl;
BIO *bio;
int sock;
struct timeval timeout;
SSL_CTX *ctx = SSL_CTX_new(DTLS_server_method());
SSL_CTX_set_session_cache_mode(ctx, SSL_SESS_CACHE_OFF);
if(!SSL_CTX_use_certificate_file(ctx, "/home/matthew/CLionProjects/OpenSSL.Test/cmake-build-debug/certs/cert.crt", SSL_FILETYPE_PEM))
{
perror("cert");
exit(EXIT_FAILURE);
}
if(!SSL_CTX_use_PrivateKey_file(ctx, "/home/matthew/CLionProjects/OpenSSL.Test/cmake-build-debug/certs/key.key", SSL_FILETYPE_PEM))
{
perror("key");
exit(EXIT_FAILURE);
}
SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER | SSL_VERIFY_CLIENT_ONCE, dtls_verify_callback);
SSL_CTX_set_read_ahead(ctx, 1);
SSL_CTX_set_cookie_generate_cb(ctx, generate_cookie);
SSL_CTX_set_cookie_verify_cb(ctx, &verify_cookie);
server_addr.sin_family = AF_INET;
server_addr.sin_addr.s_addr = INADDR_ANY;
server_addr.sin_port = htons(1114);
if((sock = socket(AF_INET, SOCK_DGRAM, 0)) < 0)
{
perror("socket");
exit(EXIT_FAILURE);
}
if(setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (const char*) &on, (socklen_t) sizeof(on)) < 0)
{
perror("set reuse address");
exit(EXIT_FAILURE);
}
if(setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, (const char*) &on, (socklen_t) sizeof(on)) < 0)
{
perror("set reuse port");
exit(EXIT_FAILURE);
}
if(bind(sock, (const struct sockaddr *) &server_addr, sizeof(server_addr)) < 0)
{
perror("bind");
exit(EXIT_FAILURE);
}
memset(&client_addr, 0, sizeof(struct sockaddr_storage));
/* Create BIO */
bio = BIO_new_dgram(sock, BIO_NOCLOSE);
/* Set and activate timeouts */
timeout.tv_sec = 5;
timeout.tv_usec = 0;
BIO_ctrl(bio, BIO_CTRL_DGRAM_SET_RECV_TIMEOUT, 0, &timeout);
ssl = SSL_new(ctx);
SSL_set_bio(ssl, bio, bio);
SSL_set_options(ssl, SSL_OP_COOKIE_EXCHANGE);
if(!SSL_set_fd(ssl, sock))
{
perror("set fd");
exit(EXIT_FAILURE);
}
res = 0;
while(res <= 0)
{
res = DTLSv1_listen(ssl, (BIO_ADDR *) &client_addr);
if(res < 0)
{
perror("dtls listen"); <--- "Destination address required"
exit(EXIT_FAILURE);
}
}
SSL_accept(ssl);
printf("Hello, World!\n");
return 0;
}
I've added error checks in every nook and cranny I can think of...
What happens is, it loops over DTLSv1_accept() fine until a connection is attempted. As soon as the client tries to connect (sends the Client Hello), DTLSv1_accept() return -1. A call to SSL_get_error gives me 5, which is an SSL_ERROR_SYSCALL.
So I do perror and get Destination address required...
AFAIK the whole point of DTLSv1_listen is to listen for any incoming Client Hellos and populate the BIO_ADDR with the client's address once the handshake is finished...
I am using this to test the server:
openssl s_client -dtls -connect 127.0.0.1:1114
I've spent many hours on this. I am about to give up on OpenSSL and try libressl...
Any help is greatly appreciated.
P.S. the code is complete, in that it should compile and run if you want to try it out.
Thanks!

Answer 1

There are two significant problems in your code sample.
First your code to generate a cookie is not correct. You are supposed to create the
cookie and store it in the location pointed to by cookie, and then fill in the length of the cookie in *cookie_len. So the code should like this:
int generate_cookie(SSL *ssl, unsigned char *cookie, unsigned int *cookie_len)
{
memcpy(cookie, "cookie", 6);
*cookie_len = 6;
return 1;
}
However, the most important error comes later when setting up the SSL object for use. OpenSSL has the concept of a BIO for abstracting the underlying transport layer. In order to do DTLS over UDP you need to use a "dgram" BIO. You do this bit correctly:
/* Create BIO */
bio = BIO_new_dgram(sock, BIO_NOCLOSE);
/* Set and activate timeouts */
timeout.tv_sec = 5;
timeout.tv_usec = 0;
BIO_ctrl(bio, BIO_CTRL_DGRAM_SET_RECV_TIMEOUT, 0, &timeout);
ssl = SSL_new(ctx);
SSL_set_bio(ssl, bio, bio);
So at this point the SSL object is correctly setup for the DTLSv1_listen() call. However you then do this:
if(!SSL_set_fd(ssl, sock))
{
perror("set fd");
exit(EXIT_FAILURE);
}
The SSL_set_fd function is an alternative to the SSL_set_bio function. What it does is take the supplied fd, wraps it in a "socket" BIO and then calls SSL_set_bio() with the result. A socket BIO is primarily useful for standard TLS and can't be used for DTLS. So the effect of the above code is to throw away the dgram BIO that you previously set up.
In my tests, if I made the generate_cookie change I suggested above, and removed the SSL_set_fd line, it all started to work as expected.