I'm producing my 1st projet in c. I want to build a mail client.
I used the socket to do it.
But have some trouble with that methode. I can't send a mail with a gmail smtp server because of an encoding protocol.
So after some research, i understood that i haven't lot's of choice. I have to use the openssl library.
And that's the problem. I understood how to use that library (I hope, you can judge it with my code attached) but i never understand how to add the library.
On stackoverflow i find solutions for linux and android so i create a new post.
So that's my question:
What should I do with the zip file that i download from openssl official website?. And even which zip file do I need to download ?
SOCKET connexion(char* server_name,unsigned short port)
{
char buf[1024]={0};
int res_l= 0;
int nbrecv;
struct sockaddr_in serverSockAddr;
struct hostent * serverHostEnt;
SOCKET to_server_socket = 0;
memset(&serverSockAddr,0, sizeof(serverSockAddr) );
serverHostEnt = gethostbyname( server_name );
if ( serverHostEnt == NULL )
{
res_l = h_errno;
return (SOCKET)-1;
}
memcpy(&serverSockAddr.sin_addr,serverHostEnt->h_addr, serverHostEnt->h_length );
serverSockAddr.sin_port = htons( port );
serverSockAddr.sin_family = AF_INET;
to_server_socket = socket( AF_INET, SOCK_STREAM, 0 );
if( connect( to_server_socket, ( struct sockaddr * ) &serverSockAddr,sizeof( serverSockAddr ) ) < 0 ) return (SOCKET)-3;
while( !buf[0] ) nbrecv = recv( to_server_socket, buf, 1024, 0 );
printf("Welcome message : %s\n",buf);
return to_server_socket;
}
void SendAndReceiveSSL(SSL * ssl, char * messagesend, int n){
{
char bufreceive[1024];
int size,retVal,nbrecv;
size = (int)strlen( messagesend );
retVal = SSL_write( ssl, messagesend, size);
printf("Envoye : %s\n",messagesend) ;
memset(bufreceive,0,1024);
if (n!=1) //n=0 if i don't want an answer of the server.
{
while(!bufreceive[0]) nbrecv = SSL_read( ssl, bufreceive,1024);
printf("Recu : %s\n",bufreceive);
}
}
int mail_ssl(SOCKET sock, SSL ssl, const char* from,const char* to,const char* body)
{
#define SIZEMAX 1000
char buffer[SIZEMAX]; int n=0;
SendAndReceive(sock, "EHLO localhost\r\n",n); //no ssl for the 2 1st sentences//EHLO localhose ... wait 250
SendAndReceive(sock, "STARTTLS\r\n,n)
SendAndReceiveSSL(ssl, "EHLO localhost\r\n",n); //sll for the next
sprintf(buffer,"MAIL FROM: <%s>\r\n",from);
SendAndReceiveSSL(ssl, buffer,n); //MAIL FROM:<******> wait 250
sprintf(buffer,"RCPT TO: <%s>\r\n",to);
SendAndReceiveSSL(ssl, buffer,n); //RCPT TO:<******> wait 250
SendAndReceiveSSL(ssl, "DATA\r\n",n); //DATA wait 340
sprintf(buffer, "Subject: %s\r\n",body); n=1;
SendAndReceiveSSL(ssl, buffer,n); n=0; //Subject : subject \r\n body\r\n ///// DON'T WAIT
SendAndReceiveSSL(ssl, ".\r\n",n); //.\r\n wait 250
SendAndReceiveSSL(ssl, "QUIT\r\n",n);
return 0;
}
SSL_CTX* InitCTX(void)
{
SSL_METHOD *method;
SSL_CTX *ctx;
OpenSSL_add_all_algorithms(); // Load cryptos, et.al.
SSL_load_error_strings(); // Bring in and register error messages
method = TLSv1_2_client_method(); // Create new client-method instance
ctx = SSL_CTX_new(method); // Create new context
if ( ctx == NULL )
{
ERR_print_errors_fp(stderr);
abort();
}
return ctx;
}
void ShowCerts(SSL* ssl)
{
X509 *cert;
char *line;
cert = SSL_get_peer_certificate(ssl); // get the server's certificate
if ( cert != NULL )
{
printf("Server certificates:\n");
line = X509_NAME_oneline(X509_get_subject_name(cert), 0, 0);
printf("Subject: %s\n", line);
free(line); // free the malloc'ed string
line = X509_NAME_oneline(X509_get_issuer_name(cert), 0, 0);
printf("Issuer: %s\n", line);
free(line); // free the malloc'ed string
X509_free(cert); // free the malloc'ed certificate copy
}
else printf("Info: No client certificates configured.\n");
}
int authSSL(SOCKET sock, const char* from,const char* to,const char* body)
{
SSL_CTX *ctx;
SSL *ssl;
SSL_library_init();
ctx = InitCTX();
ssl = SSL_new(ctx);
SSL_set_fd(ssl, sock);
if ( SSL_connect(ssl) == FAIL )
ERR_print_errors_fp(stderr);
else
{
printf("Connected with %s encryption\n", SSL_get_cipher(ssl));
ShowCerts(ssl);
SSL_write(ssl, msg, strlen(msg));
mail_ssl(sock, ssl, from, to, body); //In this function i collect
//all the informations to create the mail and i use all the function above
SSL_free(ssl);
}
close(server); // close socket
SSL_CTX_free(ctx); // release context
return 0;
}
you can use VCPKG package manager, it will download and configure your envirument if you are using Visual Studio
Related
Why does this code result in 301 error when trying to access sites that has a .net suffix
void test(const char * host, const char *index)
{
BIO *bio, *out;
SSL_CTX * ctx;
SSL * ssl;
int len;
char tmpbuf[1024];
ERR_load_crypto_strings();
char ready[1204];
char format[] = "%s:http";
sprintf(ready, format , host);
char req_template[] = "GET %s HTTP/1.1\r\nHost: %s\r\nConnection: close\r\n\r\n";
char ready_request[1024];
sprintf(ready_request , req_template , index, host);
const SSL_METHOD * method = SSLv23_client_method();
if (!method)
exit(-1);
ctx = SSL_CTX_new(method);
if (!ctx)
exit(-1);
if (!SSL_CTX_load_verify_locations(ctx,"/etc/ssl/certs/ca-certificates.crt","/etc/ssl/certs/"))
{
SSL_CTX_free(ctx);
exit(-1);
}
bio = BIO_new_ssl_connect(ctx);
BIO_get_ssl(bio, &ssl);
SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY);
char temp[1024];
sprintf(temp, "%s:https",host);
if (BIO_set_conn_hostname(bio, temp) < 0)
{
memset(temp, 0, sizeof(temp));
sprintf(temp, "%s:http", host);
bio = BIO_new_connect(temp);
if (BIO_do_connect(bio) < 0 )
{
BIO_free_all(bio);
SSL_CTX_free(ctx);
exit(-1);
}
}
printf("###\n%s\n###\n",ready_request);
out = BIO_new_fp(stdout, BIO_NOCLOSE);
if(BIO_do_connect(bio) <= 0)
exit(-1);
BIO_puts(bio,ready_request);
for(;;)
{
len = BIO_read(bio, tmpbuf, 1024);
if(len <= 0) break;
BIO_write(out, tmpbuf, len);
}
BIO_free(bio);
BIO_free(out);
}
int main()
{
test("openssl.org", "/docs/manpages.html");
test("pastebin.com", "/raw/j0BnRwBw");
test("pastebin.com", "/j0BnRwBw");
}
for some reason that i can't figure out the first time test is called it returns a 301 status code but the two time test is called it returns the html code or the paste code with out any problems
Does this have anything to do with the fact that the websites use different technologies or if they have some sort of firewall, I believe pastebin uses cloudflare to protect it self, I also tried using User-Agent but still got the same result
Just add www. as a prefix to the host header and for BIO_set_conn_hostname you should use the format www.<hostname>.com:https or www.<host>.org:http for BIO_new_connect
for some reason, the docs do not mention this
void test(const char * host, const char *index)
{
BIO *bio, *out;
SSL_CTX * ctx;
SSL * ssl;
int len;
char tmpbuf[1024];
ERR_load_crypto_strings();
char req_template[] = "GET %s HTTP/1.1\r\nHost: www.%s\r\nConnection: close\r\n\r\n";
char ready_request[1024];
sprintf(ready_request , req_template , index, host);
const SSL_METHOD * method = SSLv23_client_method();
if (!method)
exit(-1);
ctx = SSL_CTX_new(method);
if (!ctx)
exit(-1);
if (!SSL_CTX_load_verify_locations(ctx,"/etc/ssl/certs/ca-certificates.crt","/etc/ssl/certs/"))
{
SSL_CTX_free(ctx);
exit(-1);
}
bio = BIO_new_ssl_connect(ctx);
BIO_get_ssl(bio, &ssl);
SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY);
char temp[1024];
sprintf(temp, "www.%s:https",host);
if (BIO_set_conn_hostname(bio, temp) < 0)
{
memset(temp, 0, sizeof(temp));
sprintf(temp, "www.%s:http", host);
bio = BIO_new_connect(temp);
if (BIO_do_connect(bio) < 0 )
{
BIO_free_all(bio);
SSL_CTX_free(ctx);
exit(-1);
}
}
printf("###\n%s\n###\n",ready_request);
out = BIO_new_fp(stdout, BIO_NOCLOSE);
if(BIO_do_connect(bio) <= 0)
exit(-1);
BIO_puts(bio,ready_request);
for(;;)
{
len = BIO_read(bio, tmpbuf, 1024);
if(len <= 0) break;
BIO_write(out, tmpbuf, len);
}
BIO_free(bio);
BIO_free(out);
}
The first request https://openssl.org/docs/manpages.html returns:
HTTP/1.1 301 Moved Permanently
...
Location: https://www.openssl.org/docs/manpages.html
...
You make another request to that url. To demonstrate it working, I changed your first test case to read:
test("www.openssl.org", "/docs/manpages.html");
// ^^^^
and the server now returns the response you were expecting:
HTTP/1.1 200 OK
...
I have the following code, and I am getting SIGSEGV on the line:
if ( SSL_connect(ssl) == FAIL )
The fault Im getting is:
Program received signal SIGSEGV, Segmentation fault.
0x00007ffffe5a41e0 in __GI___libc_malloc (bytes=104) at malloc.c:2926
2926 malloc.c: No such file or directory.
The program basically is designed to take loads of data and push it into firebase.
The first one element, is to check if we are registered, the next bit is to actually do the registration.
Cutting the program back to basics, we have the following opening gambit:
int main(int argc, char *argv[]) {
int iRegistered = checkRegistered();
int result = registerCar();
}
If we swap those two lines, so we register before we check the registration, then we don't get a SIGSEGV.
Here's the checkRegistration function:
int checkRegistered() {
int firebaseRegistered = 0;
char *carId;
carId = (char *) malloc(256);
strcpy(carId, "aabbccddeeffgg" );
char *payload;
payload = (char *) malloc(1024);
sprintf(payload, "{ \"carid\": \"%s\" }", carId);
char *response;
response = (char *) malloc(1024);
int result = firebase("isCarRegistered", payload, &response);
if (result == 0) {
// Process JSON Response
cJSON *json = cJSON_Parse(response);
if (json == NULL) {
//
} else {
cJSON *json_registered = NULL;
json_registered = cJSON_GetObjectItemCaseSensitive(json, "registered");
firebaseRegistered = json_registered->valueint;
}
}
free(response);
free(payload);
free(carId);
return firebaseRegistered;
}
And the registerCar function.
They're basically mostly the same format - construct a message, send it to firebase, process the JSON response. We use cJSON to decompile the data returned from Firebase, though we could potentially use it to also compile. But one thing at a time.
You'll see a number of free() statements - I've been trying to work out how best to complete this - ie, generate a char* locally, pass by reference ** to a function, let the function perform the malloc/realloc based on the sizes it can calculate and then we can free it from the calling code once we have dealth with the data. Though I also get a SIGSEGV from that as well.
int registerCar() {
int iResponse = 0;
char *carId;
carId = (char *) malloc(256);
char *authCode;
authCode = (char *) malloc(12);
char *payload;
payload = (char *) malloc(1024);
sprintf(payload, "{ }");
char *response;
response = (char *) malloc(1024);
int result = firebase("registerCar", payload, &response);
if (result == 0) {
// Process JSON Response
cJSON *json = cJSON_Parse(response);
if (json == NULL) {
//
} else {
cJSON *json_auth = NULL;
cJSON *json_car = NULL;
json_auth = cJSON_GetObjectItemCaseSensitive(json, "authcode");
json_car = cJSON_GetObjectItemCaseSensitive(json, "carid");
iResponse = 1;
}
}
free(response);
free(payload);
return iResponse;
}
Here's the firebase routine, it takes a function, a payload and generates a response. Interestingly here, char firebaseLocal and charfirebaseMessage is not always null before the initial malloc.
int firebase(char *firebaseFunction, char *firebasePayload, char **firebaseResponse) {
char buf[1024];
char *firebaseLocal;
char *firebaseMessage;
firebaseMessage = (char *) malloc(1024);
SSL_CTX *ctx;
int server;
SSL *ssl;
int bytes;
ctx = InitCTX();
server = OpenConnection(HOST, atoi(PORT));
ssl = SSL_new(ctx); /* create new SSL connection state */
SSL_set_fd(ssl, server); /* attach the socket descriptor */
if ( SSL_connect(ssl) == FAIL ) /* perform the connection */
ERR_print_errors_fp(stderr);
else {
ShowCerts(ssl); /* get any certs */
char *firebasePost;
generatePostMessage(firebaseFunction, firebasePayload, &firebasePost);
SSL_write(ssl, firebasePost, strlen(firebasePost));
bytes = SSL_read(ssl, buf, sizeof(buf)); /* get reply & decrypt */
buf[bytes] = 0;
//SSL_free(ssl); /* release connection state */
strcpy(firebaseMessage, buf);
firebaseLocal = strstr(firebaseMessage, "\r\n\r\n");
if (firebaseLocal != NULL) {
firebaseLocal +=4;
}
strcpy(*firebaseResponse, firebaseLocal);
}
free(firebaseMessage);
close(server); /* close socket */
SSL_CTX_free(ctx); /* release context */
return 0;
}
This is from an implementation I found on secure sockets.
int OpenConnection(const char *hostname, int port)
{ int sd;
struct hostent *host;
struct sockaddr_in addr;
if ( (host = gethostbyname(hostname)) == NULL )
{
perror(hostname);
abort();
}
sd = socket(PF_INET, SOCK_STREAM, 0);
bzero(&addr, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_port = htons(port);
addr.sin_addr.s_addr = *(long*)(host->h_addr);
if ( connect(sd, (struct sockaddr*)&addr, sizeof(addr)) != 0 )
{
close(sd);
perror(hostname);
abort();
}
return sd;
}
This is from an implementation I found on secure sockets.
SSL_CTX* InitCTX(void)
{
SSL_METHOD *method;
SSL_CTX *ctx;
SSL_library_init();
OpenSSL_add_all_algorithms(); /* Load cryptos, et.al. */
SSL_load_error_strings(); /* Bring in and register error messages */
method = TLSv1_2_client_method(); /* Create new client-method instance */
ctx = SSL_CTX_new(method); /* Create new context */
if ( ctx == NULL )
{
ERR_print_errors_fp(stderr);
abort();
}
return ctx;
}
This is from an implementation I found on secure sockets.
void ShowCerts(SSL* ssl)
{ X509 *cert;
char *line;
cert = SSL_get_peer_certificate(ssl); /* get the server's certificate */
if ( cert != NULL )
{
printf("Server certificates:\n");
line = X509_NAME_oneline(X509_get_subject_name(cert), 0, 0);
printf("Subject: %s\n", line);
free(line); /* free the malloc'ed string */
line = X509_NAME_oneline(X509_get_issuer_name(cert), 0, 0);
printf("Issuer: %s\n", line);
free(line); /* free the malloc'ed string */
X509_free(cert); /* free the malloc'ed certificate copy */
}
else
printf("Info: No client certificates configured.\n");
}
This is something that I wrote to generate a post message from message
void generatePostMessage(char *firebaseFunction, char *firebaseMessage, char **response) {
int intPayloadSize = strlen(firebaseMessage);
char *charPayloadSize;
charPayloadSize = (char *) malloc(8);
sprintf(charPayloadSize, "%d", intPayloadSize);
char *postmessage = "POST /%s HTTP/1.1\r\n"
"Host: us-central1-carconnect-e763e.cloudfunctions.net\r\n"
"User-Agent: USER_AGENT\r\n"
"Content-Type: application/json\r\n"
"Accept: text/plain\r\n"
"Content-Length: %d\r\n\r\n"
"%s";
// Allocate size of postmessage less the inserts, plus the payload size, plus the payload size digits, plus null
int responseLength = (strlen(postmessage) - 4) + intPayloadSize + strlen(charPayloadSize)+1;
// Round up Four Bytes.
int responseIncrease = responseLength % 4;
if (responseIncrease > 0) {
responseLength += (4 - responseIncrease);
}
*response = (char *) malloc(responseLength);
sprintf(*response, postmessage, firebaseFunction, intPayloadSize, firebaseMessage);
}
As advised, whether the registration or registration check is called first, the first call works fine.
If I perform the registration before the check, then both commands work fine. Further testing also does confirm the problem is the registration check. I can perform registration several times without fail. The registration check and any follow up calls fail completely at the SSL_connect line. I don't know why.
The SSL_free command in the firebase connection always fails. I also get a SIGSEGV if I try to free(firebasePost) after the SSL_Write - which suggests I cannot free a pointer that has been passed by reference and mallocced in a function.
Part of me wonders whether any of this is caused by the fact Im debugging on Windows. I've always had problems with malloc() on Windows just not working the way I would expect.
The problem, or at least one of them, is in generatePostMessage. Not enough buffer is allocated for response. sprintf will then run off the end of the allocated buffer and cause heap corruption, which manifests itself on next invocation of malloc. Try:
int responseLength = strlen(firebaseFunction) + (strlen(postmessage) - 4) + intPayloadSize + strlen(charPayloadSize)+1;
I'm looking for a way to restrict client certificates to specific set of self-signed certificates on the server side using the OpenSSL API.
There is a set of trusted self-signed certificates, say ./dir/*.pem. I want to reject connections, if they don't supply one of those certificates.
I can achieve almost desired behaviour by comparing server and client certificate fingerprints in the SSL context verification callback:
SSL_CTX *ctx;
...
SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, verify_callback);
static inline int get_fingerprint(X509* cert, unsigned char *md, unsigned int *n)
{
return X509_digest(cert, EVP_sha1(), md, n);
}
static inline int compare_certificates(X509 *c1, X509 *c2)
{
unsigned char md1[EVP_MAX_MD_SIZE], md2[EVP_MAX_MD_SIZE];
unsigned int n1, n2;
if (!(get_fingerprint(c1, md1, &n1) && get_fingerprint(c2, md2, &n2))) {
return -1;
}
return memcmp(md1, md2, n1);
}
static int verify_callback(int preverify_ok, X509_STORE_CTX *ctx)
{
SSL *ssl = X509_STORE_CTX_get_ex_data(ctx, SSL_get_ex_data_X509_STORE_CTX_idx());
int err = X509_STORE_CTX_get_error(ctx);
/* Allow self-signed certificates */
if (!preverify_ok && err == X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT) {
preverify_ok = 1;
}
if (0 != compare_certificates(ctx->current_cert, SSL_CTX_get0_certificate(ssl->ctx))) {
/* Peer certificate doesn't match the server certificate */
preverify_ok = 0;
}
/* More checks ... */
return preverify_ok;
}
So if the server and client certificate fingerprints match, verification passes. Otherwise, connection is closed by the server.
I might compute fingerprints of the trusted certificates somewhere in initialization phase, then check them in a loop within the verify_callback. However, I don't like this idea. There should be easier way to do this.
I thought SSL_CTX_load_verify_locations() is just what I was looking for(but it looks like it's not; i'll explain why):
SSL_CTX_load_verify_locations() specifies the locations for ctx, at which CA certificates for verification purposes are located.
...
If CAfile is not NULL, it points to a file of CA certificates in PEM format. The file can contain several CA certificates...
The certificates in CApath are only looked up when required, e.g. when building the certificate chain or when actually performing the verification of a peer certificate.
(man 3 SSL_CTX_load_verify_locations)
Well, I guess SSL_VERIFY_FAIL_IF_NO_PEER_CERT implies verifying the peer certificate. Then it looks like all I need to do is to make a bundle of trusted certificates and pass it to SSL_CTX_load_verify_locations():
bundle_file=CAbundle.pem
cd ./dir
rm -f $bundle_file
for i in *.pem; do
openssl x509 -in $i -text >> $bundle_file
done
c_rehash .
SSL_CTX *ctx;
const char *cafile = "dir/CAbundle.pem";
const char *capath = NULL;
...
if (!SSL_CTX_load_verify_locations(ctx, cafile, capath)) {
/* Unable to set verify locations ... */
}
cert_names = SSL_load_client_CA_file(cafile);
if (cert_names != NULL) {
SSL_CTX_set_client_CA_list(ctx, cert_names);
} else {
/* Handle error ... */
}
All looks good. But the server still accepts connections with different peer certificates.
I've reproduced this behaviour using standard OpenSSL utilities here: https://gist.github.com/rosmanov/d960a5d58a96bdb730303c5b8e86f951
So my question is: how do I configure the server to accept only peers providing only specific certificates?
Update
I've found that the "whitelist" of certificates (CA bundle) actually works,
when I remove the following from the verify_callback:
if (!preverify_ok && err == X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT) {
preverify_ok = 1;
}
So without this block everything just works. The server responds to a client connected with one of certificates listed in CAbundle.pem. If a client connects with different certificate, the server closes connection.
However, there is a strange thing. In both cases openssl s_client outputs:
Verify return code: 18 (self signed certificate)
Then maybe
if (!preverify_ok
&& err == X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT
&& allow_self_signed
&& !cafile
&& !capath) {
preverify_ok = 1;
}
?
Update 2
Now I understand why openssl s_client outputs Verify return code: 18 (self signed...). It doesn't trust server's certificate unless -CAfile or -CApath contains the server certificate. And the server certificate is self-signed.
An explanation (for commandline) and a halfanswer (for library):
I (this time fully) redid your gist and was reminded of an inconsistency here. Commandline openssl xxx utilities are mostly designed as test/debugging tools, and in particular:
s_client normally (except anonymous, SRP, etc) receives a cert chain from the server, but uses a callback that only logs what it got and ignores/overrides any error; this is the block
depth=0 C = AU, ST = StateA, L = CityA, O = CompanyA, CN = localhost, emailAddress = a#gmail.com
verify error:num=18:self signed certificate
verify return:1
depth=0 C = AU, ST = StateA, L = CityA, O = CompanyA, CN = localhost, emailAddress = a#gmail.com
verify return:1
just after CONNECTED(fd) in your s_client output, but as you see in spite of the error it continues with the handshake resulting in a usable connection.
s_server is a more complicated. It does not request a cert from client by default, only if you specify -verify or -Verify (which set SSL_VERIFY_PEER which is not the default for server), and if it does request a cert client has the option whether to send one (with associated proof in CertVerify). If client does send chain, s_server uses the same callback as s_client which overrides any error and continues with the connection; this in your s_server output with the same verify error:num-18... which actually means 'root (including selfsigned which is its own root) in received chain but not in local truststore'. If client does not send chain, -verify continues, but -Verify (which also sets SSL_VERIFY_FAIL_IF_NO_PEER_CERT) aborts the handshake with alert 40 and returns an error, so the s_server output is very different:
verify depth is 0, must return a certificate
Using default temp DH parameters
Using default temp ECDH parameters
ACCEPT
ERROR
140679792887624:error:140890C7:SSL routines:SSL3_GET_CLIENT_CERTIFICATE:peer did not return a certificate:s3_srvr.c:3271:
shutting down SSL
CONNECTION CLOSED
ACCEPT
But a program using the library should work. I hacked up this simple test from parts of some other programs (hence the odd indentation):
/* SO36821430 2016-04-25 */
#include <stdio.h>
#if defined(_WIN32)&&!defined(WIN32)
#define WIN32 /*anything*/
#endif
#ifdef WIN32
#include <winsock2.h>
typedef int socklen_t;
#define SOCKERR WSAGetLastError()
#include "openssl/applink.c"
#else
#include <unistd.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#ifndef INADDR_NONE
#define INADDR_NONE (ipaddr_t)-1
#endif
typedef int SOCKET;
enum { INVALID_SOCKET = -1, SOCKET_ERROR = -1 };
#define SOCKERR errno
#define closesocket close
#endif
#include "openssl/ssl.h"
#include "openssl/err.h"
#include "openssl/rand.h"
void sockerr (const char *what){
fprintf (stderr, "%s %d %s\n", what, SOCKERR, strerror(SOCKERR));
}
void sslerrn (const char *what){
fprintf (stderr, "* %s failed:\n", what);
ERR_print_errors_fp (stderr);
}
void sslerr (const char *what, int rv){
fprintf (stderr, "* %s return %d:\n", what, rv);
ERR_print_errors_fp (stderr);
}
void sslerrx (SSL * ssl, const char *what, int rv){
int rc = SSL_get_error (ssl, rv);
if( rv == -1 && rc == SSL_ERROR_SYSCALL ) sockerr (what);
else fprintf (stderr, "* %s return %d,%d\n", what, rv, rc);
ERR_print_errors_fp (stderr);
}
void subj_oneline (X509 * cert, FILE *fp){
X509_NAME * subj = X509_get_subject_name (cert);
BIO *bmem = BIO_new (BIO_s_mem()); char *ptr; int n;
X509_NAME_print_ex (bmem, subj, 0, XN_FLAG_ONELINE);
n = (int) BIO_get_mem_data (bmem, &ptr);
if( n <= 0 ) ptr = "?", n = 1;
fwrite (ptr,1,n,fp);
}
const char * inaddr;
int inport;
char buf [9999];
int main (int argc, char* argv[] )
{
int rv;
struct sockaddr_in sin; socklen_t sinlen;
SOCKET s1, s2; SSL_CTX *ctx = NULL;
time_t now; struct tm * tm;
#ifdef WIN32
struct WSAData wsa;
rv = WSAStartup (MAKEWORD(1,1), &wsa);
if(rv){ printf ("WSAStartup %d\n", rv); exit(1); }
#endif
if( argc < 2 || argc > 6 )
printf ("usage: %s port key cert CAcerts\n", argv[0]), exit(1);
sin.sin_addr.s_addr = INADDR_ANY;
sin.sin_port = htons (atoi(argv[1]));
sin.sin_family = AF_INET;
/**/
SSL_library_init();
SSL_load_error_strings();
ctx = SSL_CTX_new (SSLv23_server_method());
if( !ctx ){ sslerrn("CTX_new"); exit(1); }
SSL_CTX_set_options (ctx, SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3);
rv = SSL_CTX_use_PrivateKey_file (ctx, argv[2], SSL_FILETYPE_PEM);
if( rv != 1 ){ sslerr ("use_PrivateKey_file",rv); exit(1); }
rv = SSL_CTX_use_certificate_file (ctx, argv[3], SSL_FILETYPE_PEM);
if( rv != 1 ){ sslerr ("use_certificate_file",rv); exit(1); }
SSL_CTX_set_verify (ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL);
if( !SSL_CTX_load_verify_locations (ctx, argv[4], NULL) ){
sslerrn ("load_verify_locations"); exit(1); }
SSL_CTX_set_client_CA_list (ctx, SSL_load_client_CA_file (argv[4]));
/**/
if( (s1 = socket (AF_INET,SOCK_STREAM,0)) == INVALID_SOCKET ){
sockerr ("socket()"); exit(1); }
if( bind (s1, (struct sockaddr*)&sin, sizeof sin) < 0 ){
sockerr ("bind()"); exit(1); }
if( listen (s1, 5) < 0 ){
sockerr ("listen()"); exit(1); }
do{
sinlen = sizeof sin;
if( (s2 = accept (s1, (struct sockaddr*)&sin, &sinlen)) == INVALID_SOCKET ){
sockerr ("accept()"); exit(1); }
now = time(NULL); tm = localtime(&now);
printf ("+ %s %u #%02d.%02d.%02d\n", inet_ntoa (sin.sin_addr),
ntohs (sin.sin_port), tm->tm_hour, tm->tm_min, tm->tm_sec);
/**/
SSL * ssl = SSL_new (ctx);
if( !ssl ){ sslerrn("SSL_new"); goto next; }
SSL_set_fd (ssl, s2);
if( (rv = SSL_accept(ssl)) < 0 ){
sslerrx (ssl, "SSL_accept", rv); goto next; }
{ X509 * cert = SSL_get_peer_certificate (ssl);
/*EVP_PKEY * key = cert? X509_get_pubkey (cert): NULL;*/
fprintf (stdout, "=%ld", SSL_get_verify_result (ssl));
if( cert ) putchar (':'), subj_oneline (cert, stdout);
putchar ('\n');
}
while( (rv = SSL_read (ssl, buf, sizeof buf)) > 0 )
printf ("%d: %.*s\n", rv, rv, buf);
sslerrx (ssl, "SSL_read", rv);
next:
if( ssl ) SSL_free (ssl);
/**/
now = time(NULL); tm = localtime(&now);
printf ("- %s %u #%02d.%02d.%02d\n", inet_ntoa (sin.sin_addr),
ntohs (sin.sin_port), tm->tm_hour, tm->tm_min, tm->tm_sec);
closesocket (s2);
} while (1);
return 0;
}
When run with $port cert1.key cert1.pem CAbundle.pem and connected from client using cert2.key & cert2.pem this aborts the handshake with alert 48 unknown_ca and returns an error as desired:
+ 127.0.0.1 46765 #22.07.36
* SSL_accept return -1,1
140240689366696:error:14089086:SSL routines:ssl3_get_client_certificate:certificate verify failed:s3_srvr.c:3270:
- 127.0.0.1 46765 #22.07.36
HTH.
If you want a whitelist of specific client certificates, you can prepare an indexed list in memory when you initialize.
For example, you can use PEM_X509_INFO_read to read a concatenated file of all client certificates in PEM format. This will give you a STACK_OF(X509_INFO)* of certificates. The number of certificates can be found with sk_X509_INFO_num, and you can see each certificate at sk_X509_INFO_value(..)->x509.
Then, for example, you can simply build an in-memory index and qsort by compare_x509.
Now, when your verify callback is called, just do a bsearch on your index by compare_x509, and either the certificate is on your whitelist, or it isn't.
You can accept the match on the result of compare_x5099, or of course you could double-check by verifying the full certificate once the search finds a match in the index.
I use the below server.c source, i generated
sinful-host-cert.pem
sinful-host.key
as described here: Elliptic Curve CA Guide
When running the program get the following errors:
140722397161136:error:10071065:elliptic curve routines:func(113):reason(101):ec_lib.c:995:
140722397161136:error:0B080075:x509 certificate routines:func(128):reason(117):x509_cmp.c:346:
I compiled using:
gcc server.c -ldl -lcrypto -lssl -o Server
The error occurs at this line I think
if (SSL_CTX_use_PrivateKey_file(ctx, KeyFile, SSL_FILETYPE_PEM) <= 0)
server.c
#include <errno.h>
#include <unistd.h>
#include <malloc.h>
#include <string.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <resolv.h>
#include "openssl/ssl.h"
#include "openssl/err.h"
#define FAIL -1
int OpenListener(int port)
{ int sd;
struct sockaddr_in addr;
sd = socket(PF_INET, SOCK_STREAM, 0);
bzero(&addr, sizeof(addr));
inet_aton("10.8.0.26", &addr.sin_addr);
addr.sin_family = AF_INET;
addr.sin_port = htons(port);
//addr.sin_addr.s_addr = INADDR_ANY;
if ( bind(sd, (struct sockaddr*)&addr, sizeof(addr)) != 0 )
{
perror("can't bind port");
abort();
}
if ( listen(sd, 10) != 0 )
{
perror("Can't configure listening port");
abort();
}
return sd;
}
SSL_CTX* InitServerCTX(void)
{ const SSL_METHOD *method;
SSL_CTX *ctx;
OpenSSL_add_all_algorithms(); /* load & register all cryptos, etc. */
SSL_load_error_strings(); /* load all error messages */
//method = SSLv23_server_method();
method = TLSv1_2_server_method(); /* create new server-method instance */
ctx = SSL_CTX_new(method); /* create new context from method */
if ( ctx == NULL )
{
ERR_print_errors_fp(stderr);
abort();
}
return ctx;
}
void LoadCertificates(SSL_CTX* ctx, char* CertFile, char* KeyFile)
{
//New lines
if (SSL_CTX_set_cipher_list(ctx, "ECDHE-ECDSA-AES128-GCM-SHA256") != 1)
ERR_print_errors_fp(stderr);
if (SSL_CTX_load_verify_locations(ctx, CertFile, KeyFile) != 1)
ERR_print_errors_fp(stderr);
if (SSL_CTX_set_default_verify_paths(ctx) != 1)
ERR_print_errors_fp(stderr);
//End new lines
/* set the local certificate from CertFile */
if (SSL_CTX_use_certificate_file(ctx, CertFile, SSL_FILETYPE_PEM) <= 0)
{
ERR_print_errors_fp(stderr);
abort();
}
printf("FFFF\n");
/* set the private key from KeyFile (may be the same as CertFile) */
if (SSL_CTX_use_PrivateKey_file(ctx, KeyFile, SSL_FILETYPE_PEM) <= 0)
{
ERR_print_errors_fp(stderr);
abort();
}
printf("GGGG\n");
/* verify private key */
if (!SSL_CTX_check_private_key(ctx))
{
fprintf(stderr, "Private key does not match the public certificate\n");
abort();
}
//New lines - Force the client-side have a certificate
//SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL);
//SSL_CTX_set_verify_depth(ctx, 4);
//End new lines
}
void ShowCerts(SSL* ssl)
{ X509 *cert;
char *line;
cert = SSL_get_peer_certificate(ssl); /* Get certificates (if available) */
if ( cert != NULL )
{
printf("Server certificates:\n");
line = X509_NAME_oneline(X509_get_subject_name(cert), 0, 0);
printf("Subject: %s\n", line);
free(line);
line = X509_NAME_oneline(X509_get_issuer_name(cert), 0, 0);
printf("Issuer: %s\n", line);
free(line);
X509_free(cert);
}
else
printf("No certificates.\n");
}
void Servlet(SSL* ssl) /* Serve the connection -- threadable */
{ char buf[1024];
char reply[1024];
int sd, bytes, err;
const char* HTMLecho="<html><body><pre>%s</pre></body></html>\n\n";
printf("huhupre\n");
err = SSL_accept(ssl);
if ( err <= 0 ) { /* do SSL-protocol accept */
printf("%d\n",err);
ERR_print_errors_fp(stderr);
}
else
{
printf("XXXXXX\n");
//SSL_write(ssl, "huhu\n\r", 8);
ShowCerts(ssl); /* get any certificates */
bytes = SSL_read(ssl, buf, sizeof(buf)); /* get request */
if ( bytes > 0 )
{
buf[bytes] = 0;
printf("Client msg: \"%s\"\n", buf);
sprintf(reply, HTMLecho, buf); /* construct reply */
SSL_write(ssl, reply, strlen(reply)); /* send reply */
}
else
ERR_print_errors_fp(stderr);
}
sd = SSL_get_fd(ssl); /* get socket connection */
SSL_free(ssl); /* release SSL state */
close(sd); /* close connection */
}
int main()
{ SSL_CTX *ctx;
int server;
char portnum[]="5000";
char CertFile[] = "sinful-host-cert.pem";
char KeyFile[] = "sinful-host.key";
SSL_library_init();
ctx = InitServerCTX(); /* initialize SSL */
LoadCertificates(ctx, CertFile, KeyFile); /* load certs */
server = OpenListener(atoi(portnum)); /* create server socket */
printf("%d while\n", server);
while (1)
{ struct sockaddr_in addr;
socklen_t len = sizeof(addr);
SSL *ssl;
int client = accept(server, (struct sockaddr*)&addr, &len); /* accept connection as usual */
printf("Connection: %s:%d\n",inet_ntoa(addr.sin_addr), ntohs(addr.sin_port));
ssl = SSL_new(ctx); /* get new SSL state with context */
if (ssl == NULL) {
ERR_print_errors_fp(stderr);
return 0;
}
SSL_set_fd(ssl, client); /* set connection socket to SSL state */
Servlet(ssl); /* service connection */
}
close(server); /* close server socket */
SSL_CTX_free(ctx); /* release context */
}
as described here: Elliptic Curve CA Guide...
This page has so many errors and omissions I would discard it. The first red flag is the white text and black background. That tells me someone less experienced is providing the page...
From the page:
openssl ecparam -list-curves
This should be -list_curves, not -list-curves.
From the page:
openssl ecparam -out sinful.key -name sect283k1 -genkey
This should be:
openssl ecparam -param_enc named_curve -out sinful.key -name sect283k1 -genkey
If you don't use a named curve, then you will have lots of problems later, like when a client attempts to connect to the server. Here, named curve is the OID for a curve like secp256k1, and not the domain parameters like p, a, b, G, etc.
The "lots of problems later" is documented at the OpenSSL wiki Elliptic Curve Cryptography, Named Curves. Here are some of the problems you will experience:
Client: 139925962778272:error:14094410:SSL routines:SSL3_READ_BYTES:sslv3 alert handshake failure:s3_pkt.c:1256:SSL alert number 40
Client: 139925962778272:error:1409E0E5:SSL routines:SSL3_WRITE_BYTES:ssl handshake failure:s3_pkt.c:596
Server: 140339533272744:error:1408A0C1:SSL routines:SSL3_GET_CLIENT_HELLO:no shared cipher:s3_srvr.c:1353
Also, for maximum interoperability, you should use secp256k1. A close second is secp521r1.
Also, use of/lack of -*form in the openssl ecparam and openssl req commands are discussed below.
SSL_CTX* InitServerCTX(void) { ... }
This code block has quite a few problems. The most notable is lack of the ECDH callback. Where are you setting the SSL_CTX_set_tmp_ecdh callback (OpenSSL 1.0.1 and below), or where is the call to SSL_CTX_set_ecdh_auto (OpenSSL 1.0.2 and above)?
Others include the default protocol, the default cipher list, weak and wounded ciphers, the inclusion of anonymous protocols, and compression. For a partial example of code to provide a server context, see 'No Shared Cipher' Error with EDH-RSA-DES-CBC3-SHA.
The error occurs at this line I think
if (SSL_CTX_use_PrivateKey_file(ctx, KeyFile, SSL_FILETYPE_PEM) <= 0)
I think that traces back to that defective page you referenced. This:
openssl req -x509 -new -key sinful.key -out sinful-ca.pem -outform PEM -days 3650
Should probably be (note the addition of -keyform)
openssl req -x509 -new -key sinful.key -keyform PEM -out sinful-ca.pem -outform PEM -days 3650
Or
if (SSL_CTX_use_PrivateKey_file(ctx, KeyFile, SSL_FILETYPE_ASN1) <= 0)
In general, always use the *form option for a command, whether its -keyform, -certform, -inform, -outform, etc. OpenSSL does not always get it right (even though its supposed to use PEM by default).
The error occurs at this line I think
if (SSL_CTX_use_PrivateKey_file(ctx, KeyFile, SSL_FILETYPE_PEM) <= 0)
If the private key has a password, then you will need to provide a Password Callback or strip the password from the file.
Its OK to strip the password because there's no difference in storing a plaintext private key; or a encrypted private key with the passphrase in a configuration file next to the key. In both cases, the only effective security you have is the filesystem ACLs.
Related, this is known as the Unattended Key Storage problem. Guttman discusses it in his book Engineering Security. Its a problem without a solution.
Here's some more complete error information... It looks like you are using an old version of OpenSSL, and that does not provide the newer error codes.
When running the program get the following errors:
140722397161136:error:10071065:elliptic curve routines:func(113):reason(101):ec_lib.c:995
140722397161136:error:0B080075:x509 certificate routines:func(128):reason(117):x509_cmp.c:346
First, the 0x10071065 error:
$ /usr/local/ssl/macosx-x64/bin/openssl errstr 0x10071065
error:10071065:elliptic curve routines:EC_POINT_cmp:incompatible objects
The 0x10071065 usually means the client and the server are using incompatible EC fields. In this case, you should use either secp256k1 or secp521r1.
Second, the 0x0B080075 error:
$ /usr/local/ssl/macosx-x64/bin/openssl errstr 0x0B080075
error:0B080075:x509 certificate routines:X509_check_private_key:unknown key type
I'm guessing that there's a mismatch in the certificate and private key. But its only a guess. I would (1) clear the named curve issue, (2) clear the sect283k1 issue, and (3) clear the down level library issue (see below). After clearing those issues, then see if this issue remains.
It looks like you are using an old version of OpenSSL, and that does not provide the newer error codes...
Be sure you are running OpenSSL 1.0.0 or above. 0.9.8 had limited EC support, but it was not really cut-in in force until 1.0.0. Better, use OpenSSL 1.0.2.
OpenSSL_add_all_algorithms(); /* load & register all cryptos, etc. */
SSL_load_error_strings(); /* load all error messages */
Also see Library Initialization on the OpenSSL wiki.
if (SSL_CTX_set_cipher_list(ctx, "ECDHE-ECDSA-AES128-GCM-SHA256")
This will get you into trouble on some versions of OS X and iOS due to a bug in the SecureTransport library. Apple only fixed it on some versions of their operating systems.
If you plan on servicing Apple hardwarez, then you will need one additional non-ECDHE-ECDSA cipher. And you need to use the server side context option SSL_OP_SAFARI_ECDHE_ECDSA_BUG.
Related, Apple is pretty bold about not fixing their security bugs. You have the broken ECDHE-ECDSA cipher suites; and gems like CVE-2015-1130 (Hidden Backdoor with Root).
Here's what my ECDH callback looks like in OpenSSL 1.0.1 and below. OpenSSL 1.0.2 should use SSL_CTX_set_ecdh_auto. Its C++ code, but its easy enough to convert back to C code. Also see SL_CTX_set_tmp_ecdh_callback semantics in 1.0.1 on the OpenSSL mailing list.
The code below could be more robust. The callback should fetch the certificate with SSL_get_certificate (not SSL_get_peer_certificate), query the certificate for the EC field, and then provide a temporary key in the appropriate field, like secp256k1 or secp571k1. (It works because my certificates use secp256, and EcdhCallback uses secp256 as its default).
SSL_get_certificate is not documented. But it is used in <openssl src>/apps/s_cb.c. That's the "self documenting" code OpenSSL is famous for.
using SSL_ptr = std::shared_ptr<SSL>;
using SSL_CTX_ptr = std::shared_ptr<SSL_CTX>;
using EC_KEY_ptr = std::unique_ptr<EC_KEY, decltype(&::EC_KEY_free)>;
using EC_GROUP_ptr = std::unique_ptr<EC_GROUP, decltype(&::EC_GROUP_free)>;
using EC_POINT_ptr = std::unique_ptr<EC_POINT, decltype(&::EC_POINT_free)>;
using EVP_PKEY_ptr = std::unique_ptr<EVP_PKEY, decltype(&::EVP_PKEY_free)>;
using BIO_MEM_ptr = std::unique_ptr<BIO, decltype(&::BIO_free)>;
using BIO_FILE_ptr = std::unique_ptr<BIO, decltype(&::BIO_free)>;
...
SSL_CTX* CreateServerContext(const string & domain)
{
const SSL_METHOD* method = SSLv23_server_method();
ASSERT(method != NULL);
SSL_CTX_ptr t(SSL_CTX_new(method), ::SSL_CTX_free);
ASSERT(t.get() != NULL);
long flags = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3;
flags |= SSL_OP_NO_COMPRESSION;
flags |= SSL_OP_SAFARI_ECDHE_ECDSA_BUG;
flags |= SSL_OP_CIPHER_SERVER_PREFERENCE;
SSL_CTX_set_options(t.get(), flags);
string ciphers = "HIGH:!aNULL:!RC4:!MD5";
rc = SSL_CTX_set_cipher_list(t.get(), ciphers.c_str());
...
LogDebug("GetServerContext: setting ECDH callback");
SSL_CTX_set_tmp_ecdh_callback(t.get(), EcdhCallback);
...
return t.release();
}
EC_KEY* EcdhCallback(SSL *ssl, int is_export, int keylength)
{
UNUSED(ssl);
UNUSED(is_export);
UNUSED(keylength);
/* This callback is OK, but OpenSSL calls it in a broken fashion. */
/* With 1.0.1e and 1.0.1f, the value is 1024-bits. That is more */
/* appropriate for RSA.... We'll try and rewrite it here. */
if (keylength >= 1024)
{
keylength = 256;
LogRelevant("EcdhCallback: field size is wrong, using 256-bit group");
}
#if defined(ALLOW_ECDH_192_PARAMS)
if (keylength <= 192 + 4)
return ECDH192();
#endif
if (keylength <= 224 + 4)
return ECDH224();
else if (keylength <= 256 + 4)
return ECDH256();
else if (keylength <= 384 + 4)
return ECDH384();
else if (keylength <= 521 + 4)
return ECDH521();
return ECDH521();
}
#if defined(ALLOW_ECDH_192_PARAMS)
static EC_KEY* ECDH192()
{
static EC_KEY_ptr key(NULL, NULL);
static once_flag flag;
call_once(flag, []()
{
key = EC_KEY_ptr(InitEcdhkey(192), ::EC_KEY_free);
ASSERT(key.get());
if(!key.get())
LogError("ECDH192: InitEcdhkey failed");
});
return key.get();
}
#endif
static EC_KEY* ECDH224()
{
static EC_KEY_ptr key(NULL, NULL);
static once_flag flag;
call_once(flag, []()
{
key = EC_KEY_ptr(InitEcdhkey(224), ::EC_KEY_free);
ASSERT(key.get());
if(!key.get())
LogError("ECDH224: InitEcdhkey failed");
});
return key.get();
}
static EC_KEY* ECDH256()
{
static EC_KEY_ptr key(NULL, NULL);
static once_flag flag;
call_once(flag, []()
{
key = EC_KEY_ptr(InitEcdhkey(256), ::EC_KEY_free);
ASSERT(key.get());
if(!key.get())
LogError("ECDH256: InitEcdhkey failed");
});
return key.get();
}
static EC_KEY* ECDH384()
{
static EC_KEY_ptr key(NULL, NULL);
static once_flag flag;
call_once(flag, []()
{
key = EC_KEY_ptr(InitEcdhkey(384), ::EC_KEY_free);
ASSERT(key.get());
if(!key.get())
LogError("ECDH384: InitEcdhkey failed");
});
return key.get();
}
static EC_KEY* ECDH521()
{
static EC_KEY_ptr key(NULL, NULL);
static once_flag flag;
call_once(flag, []()
{
key = EC_KEY_ptr(InitEcdhkey(521), ::EC_KEY_free);
ASSERT(key.get());
if(!key.get())
LogError("ECDH521: InitEcdhkey failed");
});
return key.get();
}
static EC_KEY* InitEcdhkey(int bits)
{
if (bits <= 160 + 4)
bits = 160;
else if (bits <= 192 + 4)
bits = 192;
else if (bits <= 224 + 4)
bits = 224;
else if (bits <= 256 + 4)
bits = 256;
else if (bits <= 384 + 4)
bits = 384;
else if (bits <= 521 + 4)
bits = 521;
else
bits = 521;
EC_KEY* key = EC_KEY_new_by_curve_name(CurveToNidByBits(bits));
unsigned long err = ERR_get_error();
ASSERT(key != NULL);
if (key == NULL)
{
ostringstream oss;
oss << "InitEcdhkey: EC_KEY_new_by_curve_name failed for ";
oss << bits << "-bit key, error " << err << ", 0x" << err;
LogError(oss);
}
return key;
}
I'm trying to implement a web prefetching system. The purpose of a system like this is to “predict” future requests and prefetch them.
The system builds a predictive model from web navigation logs (Squid access.log files). The model is a dependency graph, where a node representing URL A has an arc to a node representing URL B if URL B has been requested immediately after URL A.
Once the model is built, the system receives queries of URLs requested by users, and make “predictions” based on the graph. Predictions are resources (URLs) very likely to be requested in the future. So, based on predictions, the system prefetches these resources to store them in cache prior to users' requests.
I'm using the following testing scenario:
A process simulate multiple clients, requesting URLs in a file using libcurl. The process runs in a different PC from the prefetching system. PCs are connected directly via an ethernet cable
Requests made by the client simulator are always the same URLs in the same relative time from the first request made. All requests are going to port 3128 (Prefetch PC Squid listen port) (port 80 DNAT to port 3128 in the client).
The prefetching system runs in a CentOS 6.3 box, kernel 2.6.32-71.el6.i686, 2 core Intel Pentium 4 3.00GHz processor, 4 GB RAM.
The prefetching system is one process with multiple threads. The main thread creates the predictive model and generates predictions based on queries. A “listener” thread reads URLs requested by users and prefetches predicted URLs using libcurl. “Listening” means reading from a named pipe (called url_fifo) URLs captured live on an interface using tshark:
stdbuf -o0 tshark -i eth1 tcp port 3128 and "tcp[((tcp[12:1] & 0xf0) >> 2):4] = 0x47455420" -T fields -e http.request.full_uri >> url_fifo
Each 10 minutes (1 cycle) the model is updated based on requests from the last cycle. The client tells the system when a cycle ends and so the model is updated. Once the model is updated, the system tells the client to start requesting URLs from the next cycle.
Here is the situation: Sometimes reading from the named pipe freezes. No URLs are read from the pipe even though tshark keeps capturing URLs and redirecting them to the named pipe. After an hour (or a couple of hours) all “buffered” URLs are read in less than 10 minutes. After that, reading from the pipe keeps going ok again. This situation doesn't happen always (50% of times freezes, 50% no).
It seems that there is a buffering issue, since tshark keeps capturing URLs and all requests are correctly logged in Squid's access.log.
In the beginning, I ran tshark with the -l option, so that its output becomes line buffered. Then I started using stdbuf -o0 (no buffering). Anyway the situation still happens.
In the system code, I also tried opening and reading the named pipe as a stream (FILE *) and set the stream as no buffered or line buffered (using setvbuf() function). The situation still happened.
In some cycles requests are faster than in other cycles. Anyway, it doesn't seems to be a fast producer slow consumer issue, since in many repetitions of the test all URLs are correctly read and processed without any freezes.
Is there something am I missing related to named pipes and buffering? I'd really appreciate some guidance.
Assume networking (interfaces, routing, iptables, squid) is ok. I've not had any issues related to it.
Code (assume necessary header files are included):
functions.c
#define BUFLEN 512
#define QUEUE_LEN 64
#define THREADS_LEN 2
pthread_mutex_t model_lock;
pthread_cond_t model_cond, listen_cond;
pthread_t queries_thread, listen_thread;
short int model_is_updating, model_can_update, program_shutdown;
/* Program execution statistics */
Status * program_status;
/* Thread pool */
threadpool_t *pool;
/* program execution */
int
run(void)
{
Graph_Adj_List * gr = NULL; /* Graph as an adjacency list */
char ** reports = NULL;
unsigned report_counter = 0;
/* Init program status */
program_status = status_init();
/* Load list of custom web navigation reports to be used to build the initial
* version of the predictive model */
reports = file_load_reports(program_config.reports_file);
if (!reports)
return 0;
/* Init mutex and cond */
pthread_mutex_init(&model_lock, NULL);
pthread_cond_init(&model_cond, NULL);
pthread_cond_init(&listen_cond, NULL);
/* Lock */
pthread_mutex_lock (&model_lock);
/* Start first cycle */
status_start_cycle(program_status);
/* Create initial version of the predictive model */
gr = create_model_from_files(reports, &report_counter, program_config.reports_limit);
if (!gr)
{
/* Unlock */
pthread_mutex_unlock (&model_lock);
return 0;
}
/* Unlock */
pthread_mutex_unlock (&model_lock);
/* Start threads */
if (pthread_create(&queries_thread, NULL, fifo_predictions_threaded, (void *)gr) ||
pthread_create(&listen_thread, NULL, listen_end_of_cycle, NULL))
program_shutdown = 1;
/* main loop */
while(!program_shutdown)
{
/* lock */
pthread_mutex_lock (&model_lock);
/* wait for clients' announcement of the end of requests from current cycle */
while (!model_can_update)
pthread_cond_wait(&model_cond, &model_lock);
/* set updating flag */
model_is_updating = 1;
/* Update predictive model, based on Squid's access.log from (about to finish)
* current cycle */
adj_list_update_access(gr, program_config.access_file);
/* Save statistics related to the current cycle and finish it */
status_finish_cycle(program_status);
/* Check if last custom report has been read */
if (!reports[report_counter])
{
program_shutdown = 1;
pthread_mutex_unlock (&model_lock);
break;
}
/* Start a new cycle */
status_start_cycle(program_status);
/* Read a new custom report and update the predictive model */
update_model(gr, reports[report_counter]);
report_counter++;
/* Updating is done */
model_is_updating = 0;
/* Model can't be updated until client announces the end of the cycle
* that has just started */
model_can_update = 0;
/* Tell client to start sending requests from the new cycle */
if (!signal_start_cycle())
{
program_shutdown = 1;
pthread_mutex_unlock (&model_lock);
break;
}
/* Signal listener thread that a new cycle has begin */
pthread_cond_signal(&listen_cond);
/* Unlock */
pthread_mutex_unlock (&model_lock);
}
/* Finish threads */
pthread_cancel(listen_thread);
pthread_cancel(queries_thread);
pthread_join(listen_thread, NULL);
pthread_join(queries_thread, NULL);
/* Free memory */
adj_list_free_all2(&gr);
file_reports_free_all(&reports);
pthread_cond_destroy(&model_cond);
pthread_cond_destroy(&listen_cond);
pthread_mutex_destroy(&model_lock);
status_free(&program_status);
return 1;
}
void *
fifo_predictions_threaded(void * data)
{
Graph_Adj_List * gr = (Graph_Adj_List *) data;
/* Set thread cancel type */
pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
while (!program_shutdown)
{
pthread_mutex_lock(&model_lock);
/* Pause reading from named pipe while the model is being updated */
while(model_is_updating)
pthread_cond_wait(&listen_cond, &model_lock);
pthread_mutex_unlock(&model_lock);
/* Read URLs from named pipe */
fifo_predictions(gr, program_config.fifo);
}
pthread_exit(NULL);
return NULL;
}
int
fifo_predictions(Graph_Adj_List * gr, const u8 * fifo)
{
u8 cad[BUFLEN] = { '\0' };
u8 * ini = NULL, * fin = NULL, * fullurl = NULL;
int i, fifo_descriptor, read_urls = 0, fullurl_len = 0, incomplete_url = 1;
FILE * fifo_file = NULL;
/* Open fifo in blocking mode */
fifo_descriptor = open(CHAR_CAST fifo, O_RDONLY);
/* Open fifo as a stream */
// fifo_file = fopen(fifo, "r");
// if (!fifo_file)
if (fifo_descriptor == -1)
return 0;
/* If fifo is opened as a stream, set it line buffered */
// setlinebuf(fifo_file);
do
{
if ((i = read(fifo_descriptor, cad, BUFLEN - 1)) == -1)
// if ( fgets(cad, BUFLEN-1, fifo_file) == NULL)
ERROR(__FILE__, __FUNCTION__, __LINE__, "Fifo read error");
else
{
// i = strlen(cad);
cad[i] = '\0';
read_urls = 0;
if (i > 0)
{
int j = 0;
for (j = 0, ini = cad, fin = NULL ; cad[j] != '\0'; j++)
{
if (cad[j] == '\n')
{
/* Save URL */
fin = &cad[j];
ini = (*ini == '\n' ? ini + 1 : ini);
/* Check if string is a continuation of the previously read URL */
read_urls = fin - ini;
read_urls = read_urls >= 0 ? read_urls : 0;
/* Save URL in fullurl string */
fullurl = realloc(fullurl, fullurl_len + read_urls + 1);
memcpy(&fullurl[fullurl_len], ini, read_urls);
fullurl[fullurl_len + read_urls] = '\0';
ini = fin;
incomplete_url = fullurl_len = 0;
/* Ask the model for predictions and fetch them */
fetch_url_predictions2(gr, fullurl);
u8_free(&fullurl);
} else
incomplete_url = 1;
}
if (incomplete_url)
{
ini = (*ini == '\n' ? ini + 1 : ini);
read_urls = &cad[j] - ini;
read_urls = read_urls >= 0 ? read_urls : 0;
fullurl = realloc(fullurl, fullurl_len + read_urls);
memcpy(&fullurl[fullurl_len], ini, read_urls);
fullurl_len += read_urls;
}
}
}
} while (i > 0);
close(fifo_descriptor);
// fclose (fifo_file);
return 1;
}
int
fetch_url_predictions2(Graph_Adj_List * gr, u8 * in_url)
{
String * string_url = NULL;
Headnode * head = NULL;
LinkedList * list = NULL;
LinkedListElem * elem = NULL;
/* Use custom string type */
string_url = string_create_no_len(in_url);
if (!string_url)
return 0;
pthread_mutex_lock(&model_lock);
/* Get URL node */
head = adj_list_get_node(gr, string_url);
if (head)
{
/* Get predictions (URLs) as a linked list */
list = adj_list_predictions_to_list(head);
if (!list)
{
string_free_all(&string_url);
return 0;
}
pthread_mutex_unlock(&model_lock);
/* Callback fetches URLs */
list->callback = &curl_callback_void;
if (!pool)
pool = threadpool_create(THREADS_LEN, QUEUE_LEN, 0);
/* Load URLs to be fetched to threadpool's task queue */
for (elem = list->first; elem; elem = elem->next)
{
CallbackArg arg;
arg.data = arg.copy(elem->data);
threadpool_add_copy_arg(pool, list->callback, &arg, 1, sizeof(arg), 0);
}
linked_list_free_all(&list);
}
pthread_mutex_unlock(&model_lock);
string_free_all(&string_url);
return 1;
}
fetch.c
void
curl_callback_void(void * data)
{
CallbackArg * arg = (CallbackArg *) data;
char * url = (char *) arg->data;
fetch_url(url);
}
static size_t
write_data(void *buffer, size_t size, size_t nmemb, void *userp)
{
return size * nmemb;
}
int
fetch_url(char * url)
{
CURL *curl;
CURLcode res;
struct timeval time;
char * time_string = NULL;
curl = curl_easy_init();
if (curl)
{
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL);
curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 15);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10);
/* Perform the request, res will get the return code */
res = curl_easy_perform(curl);
gettimeofday(&time, NULL);
time_string = timeval_to_str(&time);
/* Check for errors */
if (res != CURLE_OK)
{
fprintf(stderr, "\ntime %s curl_easy_perform() (url %s) failed: %s\n",
time_string, url, curl_easy_strerror(res));
}
else
{
fprintf(stderr, "\ntime %s curl_easy_perform() (url %s) fetched ok\n",
time_string, url);
}
fflush(stderr);
free (time_string);
curl_easy_cleanup(curl);
}
return 0;
}
network.c
/*
* Code based on Beej's Networking Guide
*/
#define MSG_LEN 5
#define QUEUE_SIZE 5
extern pthread_mutex_t model_lock;
extern pthread_cond_t model_cond;
extern short int model_can_update, program_shutdown;
extern Config program_config;
// get sockaddr, IPv4 or IPv6:
static void *
get_in_addr(struct sockaddr *sa) {
if (sa->sa_family == AF_INET) {
return &(((struct sockaddr_in*) sa)->sin_addr);
}
return &(((struct sockaddr_in6*) sa)->sin6_addr);
}
void *
listen_end_of_cycle(void * data)
{
int sockfd, new_fd; // listen on sock_fd, new connection on new_fd
struct addrinfo hints, *servinfo, *p;
struct sockaddr_storage their_addr; // connector's address information
socklen_t sin_size;
int yes = 1;
char s[INET_ADDRSTRLEN], msg[MSG_LEN], *str = NULL;
int rv;
int read_bytes;
struct timeval actual_time;
/* Set thread cancel type */
pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL );
memset(&hints, 0, sizeof hints);
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
hints.ai_flags = AI_PASSIVE; // use my IP
if ((rv = getaddrinfo(NULL, program_config.listen_port, &hints, &servinfo))
!= 0) {
fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(rv));
return "error";
}
// loop through all the results and bind to the first we can
for (p = servinfo; p != NULL ; p = p->ai_next) {
if ((sockfd = socket(p->ai_family, p->ai_socktype, p->ai_protocol))
== -1) {
perror("server: socket");
continue;
}
if (setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int))
== -1) {
perror("setsockopt");
return "error";
}
if (bind(sockfd, p->ai_addr, p->ai_addrlen) == -1) {
close(sockfd);
perror("server: bind");
continue;
}
break;
}
if (p == NULL ) {
fprintf(stderr, "server: failed to bind\n");
return "error";
}
freeaddrinfo(servinfo); // all done with this structure
if (listen(sockfd, QUEUE_SIZE) == -1) {
perror("listen");
return "error";
}
while (!program_shutdown)
{
sin_size = sizeof their_addr;
new_fd = accept(sockfd, (struct sockaddr *) &their_addr, &sin_size);
if (new_fd == -1) {
perror("accept");
continue;
}
inet_ntop(their_addr.ss_family,
get_in_addr((struct sockaddr *) &their_addr), s, sizeof s);
if ((read_bytes = recv(new_fd, msg, MSG_LEN - 1, 0)) == -1) {
perror("recv");
continue;
}
close(new_fd);
msg[read_bytes] = '\0';
/* Check received message */
if (strcmp(msg, "DONE")) {
perror("Not valid message");
continue;
}
printf("\ngot \"DONE\" from %s\n", s);
fflush(stdout);
/* Lock */
pthread_mutex_lock(&model_lock);
/* Flag used by main thread to allow model update */
model_can_update = 1;
/* Signal model can be updated */
pthread_cond_signal(&model_cond);
/* Unlock */
pthread_mutex_unlock(&model_lock);
}
close(sockfd);
pthread_exit(NULL);
return "ok";
}
int signal_start_cycle(void) {
int sockfd;
struct addrinfo hints, *servinfo, *p;
int rv;
char s[INET6_ADDRSTRLEN], *str = NULL;
struct timeval actual_time, aux_time;
struct timeval connect_timeout = { 15, 0 }, max_connect_time = { 0, 0 };
short int connected = 0;
memset(&hints, 0, sizeof hints);
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
if ((rv = getaddrinfo(program_config.client_ip, program_config.client_port,
&hints, &servinfo)) != 0) {
fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(rv));
return 0;
}
gettimeofday(&aux_time, NULL);
timeval_add(aux_time, connect_timeout, &max_connect_time);
/* Try several times to connect to the remote side */
do {
// loop through all the results and connect to the first we can
for (p = servinfo; p != NULL ; p = p->ai_next) {
if ((sockfd = socket(p->ai_family, p->ai_socktype, p->ai_protocol))
== -1) {
perror("client: socket");
continue;
}
gettimeofday(&actual_time, NULL )
printf("\ntrying to connect %s\n", program_config.client_ip);
fflush(stdout);
if (connect(sockfd, p->ai_addr, p->ai_addrlen) == -1) {
close(sockfd);
perror("client: connect");
continue;
}
connected = 1;
break;
}
} while (!connected && !timeval_greater_than(actual_time, max_connect_time));
if (p == NULL ) {
fprintf(stderr, "client: failed to connect\n");
return 0;
}
inet_ntop(p->ai_family, get_in_addr((struct sockaddr *) p->ai_addr), s,
sizeof s);
printf("\nMAIN THREAD: connecting to %s\n", s);
fflush(stdout);
freeaddrinfo(servinfo); // all done with this structure
if (send(sockfd, "DONE", 4, 0) == -1)
{
perror("send");
return 0;
}
printf("\nsent \"DONE\" to %s\n", s);
fflush(stdout);
close(sockfd);
return 1;
}