I'm dealing with code written by a 3rd party (which I don't want to radically change as I'll explain) which opens a tcp connection to talk json-rpc with a server. I want to add ssl to this as the server has an ssl port for the same comms. EDIT: I want to avoid using external proxies (like stunnel, which works).
Currently, the socket is opened by socket() and passed to curl via CURLOPT_OPENSOCKETFUNCTION and libcurl is told to stop after connecting (perform no data transfer) via CURLOPT_CONNECT_ONLY.
I tried enabling the SSL options in CURL starting with curl_easy_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_ALL); but they had no effect, i.e. the connection itself succeeded in that the function connect() returned true (CURL was happy) but the data is still being sent unencrypted.
My understanding is: the socket is created externally (and passed to CURL which is told to not handle data transfer) and then written to directly, not via CURL or OpenSSL, so nothing actually encrypts the transfer ... despite enabling SSL in CURL.
I also tried to point CURL to https:// by doing sprintf(sctx->curl_url, "https%s", strstr(url, "://")); and then I get:
SSL: couldn't create a context: error:140A90A1:lib(20):func(169):reason(161)
I've also tried using OpenSSL directly on the opened socket, like here https://stackoverflow.com/a/41321247/1676217, by initializing OpenSSL and setting its file descriptor to sctx->sock which is the same socket passed to CURL (i.e. calling SSL_set_fd(ssl, sctx->sock);) but SSL_Connect() fails.
I can't change the code entirely to remove CURL or to remove the external socket creation as both are used in other parts of the code, though I can attempt to adapt it.
Basically, I'd like to keep the function declarations for thread_recv_line() and thread_send_line() and make them use SSL.
If anyone proficient with OpenSSL, CURL and/or sockets could show me how to do it within this code snippet, I'd be very grateful.
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#define socket_blocks() (errno == EAGAIN || errno == EWOULDBLOCK)
#define RBUFSIZE 2048
#define RECVSIZE (RBUFSIZE - 4)
struct ctx {
char *url;
CURL *curl;
char *curl_url;
char curl_err_str[CURL_ERROR_SIZE];
curl_socket_t sock;
size_t sockbuf_size;
char *sockbuf;
pthread_mutex_t sock_lock;
};
static curl_socket_t opensocket_grab_cb(void *clientp, curlsocktype purpose,
struct curl_sockaddr *addr)
{
curl_socket_t *sock = (curl_socket_t*) clientp;
*sock = socket(addr->family, addr->socktype, addr->protocol);
return *sock;
}
bool connect(struct ctx *sctx, const char *url)
{
CURL *curl;
int rc;
pthread_mutex_lock(&sctx->sock_lock);
if (sctx->curl)
curl_easy_cleanup(sctx->curl);
sctx->curl = curl_easy_init();
if (!sctx->curl) {
pthread_mutex_unlock(&sctx->sock_lock);
return false;
}
curl = sctx->curl;
if (!sctx->sockbuf) {
sctx->sockbuf = (char*) calloc(RBUFSIZE, 1);
sctx->sockbuf_size = RBUFSIZE;
}
sctx->sockbuf[0] = '\0';
pthread_mutex_unlock(&sctx->sock_lock);
if (url != sctx->url) {
free(sctx->url);
sctx->url = strdup(url);
}
free(sctx->curl_url);
sctx->curl_url = (char*) malloc(strlen(url));
sprintf(sctx->curl_url, "http%s", strstr(url, "://"));
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
curl_easy_setopt(curl, CURLOPT_URL, sctx->curl_url);
curl_easy_setopt(curl, CURLOPT_FRESH_CONNECT, 1);
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 30);
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, sctx->curl_err_str);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
curl_easy_setopt(curl, CURLOPT_TCP_NODELAY, 1);
curl_easy_setopt(curl, CURLOPT_HTTPPROXYTUNNEL, 1);
curl_easy_setopt(curl, CURLOPT_SOCKOPTFUNCTION, sockopt_keepalive_cb);
curl_easy_setopt(curl, CURLOPT_OPENSOCKETFUNCTION, opensocket_grab_cb);
curl_easy_setopt(curl, CURLOPT_OPENSOCKETDATA, &sctx->sock);
curl_easy_setopt(curl, CURLOPT_CONNECT_ONLY, 1);
rc = curl_easy_perform(curl);
if (rc) {
curl_easy_cleanup(curl);
sctx->curl = NULL;
return false;
}
return true;
}
static bool send_line(curl_socket_t sock, char *s)
{
size_t sent = 0;
int len = (int) strlen(s);
s[len++] = '\n';
while (len > 0) {
struct timeval timeout = {0, 0};
int n;
fd_set wd;
FD_ZERO(&wd);
FD_SET(sock, &wd);
if (select((int) (sock + 1), NULL, &wd, NULL, &timeout) < 1)
return false;
n = send(sock, s + sent, len, 0);
if (n < 0) {
if (!socket_blocks())
return false;
n = 0;
}
sent += n;
len -= n;
}
return true;
}
bool thread_send_line(struct ctx *sctx, char *s)
{
bool ret = false;
pthread_mutex_lock(&sctx->sock_lock);
ret = send_line(sctx->sock, s);
pthread_mutex_unlock(&sctx->sock_lock);
return ret;
}
static bool socket_full(curl_socket_t sock, int timeout)
{
struct timeval tv;
fd_set rd;
FD_ZERO(&rd);
FD_SET(sock, &rd);
tv.tv_sec = timeout;
tv.tv_usec = 0;
if (select((int)(sock + 1), &rd, NULL, NULL, &tv) > 0)
return true;
return false;
}
static void buffer_append(struct ctx *sctx, const char *s)
{
size_t old, n;
old = strlen(sctx->sockbuf);
n = old + strlen(s) + 1;
if (n >= sctx->sockbuf_size) {
sctx->sockbuf_size = n + (RBUFSIZE - (n % RBUFSIZE));
sctx->sockbuf = (char*) realloc(sctx->sockbuf, sctx->sockbuf_size);
}
strcpy(sctx->sockbuf + old, s);
}
char *thread_recv_line(struct ctx *sctx)
{
ssize_t len, buflen;
char *tok, *sret = NULL;
if (!strstr(sctx->sockbuf, "\n")) {
bool ret = true;
time_t rstart;
time(&rstart);
if (!socket_full(sctx->sock, 60))
return sret;
do {
char s[RBUFSIZE];
ssize_t n;
memset(s, 0, RBUFSIZE);
n = recv(sctx->sock, s, RECVSIZE, 0);
if (!n) {
ret = false;
break;
}
if (n < 0) {
if (!socket_blocks() || !socket_full(sctx->sock, 1)) {
ret = false;
break;
}
} else
buffer_append(sctx, s);
} while (time(NULL) - rstart < 60 && !strstr(sctx->sockbuf, "\n"));
if (!ret)
return sret;
}
buflen = (ssize_t) strlen(sctx->sockbuf);
tok = strtok(sctx->sockbuf, "\n");
if (!tok)
return sret;
sret = strdup(tok);
len = (ssize_t) strlen(sret);
if (buflen > len + 1)
memmove(sctx->sockbuf, sctx->sockbuf + len + 1, buflen - len + 1);
else
sctx->sockbuf[0] = '\0';
}
This is not what you asked for, but you should consider placing a separate TCP-to-TLS proxy (such as HAProxy) between your code and remote server.
You will have to neither mess with TLS protocol details, nor modify existing code. Such setup will likely be safer and faster than anything you could write yourself.
Libcurl establishes an SSL connection with the server, even with CURLOPT_CONNECT_ONLY. It is not possible to establish another SSL connection on top of the same TCP connection. The only way to continue is to use the same SSL connection that libcurl has already established.
Fortunately it is possible to retrieve the SSL connection data from the curl handle using curl_easy_getinfo and CURLINFO_TLS_SSL_PTR. It should return a pointer of type SSL*.
The thread_send_line and thread_recv_line functions should not use the socket directly, but call SSL_read and SSL_write on the retrieved SSL pointer instead.
See CURLINFO_TLS_SSL_PTR for more information. Make sure you are getting an SSL pointer and not an SSL_CTX pointer.
Something like this should work:
bool thread_send_line(struct ctx *sctx, char *s)
{
bool ret = false;
struct curl_tlssessioninfo* session;
curl_easy_getinfo (sctx->curl, CURLINFO_TLS_SSL_PTR, &session);
SSL* ssl = session->internals;
pthread_mutex_lock(&sctx->sock_lock);
int len = (int) strlen(s);
s[len++] = '\n';
ret = SSL_write(ssl, s, len);
pthread_mutex_unlock(&sctx->sock_lock);
return ret == len;
}
Related
I am trying to develop a C application to upload a file on HTTP endpoint.
I used below command from terminal to upload file, which works fine,
curl -F file=#/filename.txt -O http://serveraddress/download
Since, I need it in C application I took reference from https://curl.haxx.se/libcurl/c/postit2-formadd.html which is also working fine. the file supplied is getting uploaded successfully.
I want to upload the file similar way but the file is not present on local storage. The file is being downloaded in an application using Curl. From the beginning, we have the size of the file which will be downloaded. I want to stream that file directly to the HTTP endpoint as above(multipart file upload) because I don't have enough local storage to download entire file and then upload.
File--> CurlDownloadHandle-->Write() ----- Read()-->CurlUploadHandle---File
I am not sure how to use HTTPPOST, HTTPPOSTFILED and all. Is there anyway I can achieve this streaming?
Any suggestion is much appreciated.
Thanks.
I have figured out below method to achieve my requirement.
File streaming can be achieved using curl_formadd(for older version of libcurl)and curl_mime(for newer version > 7.58.0) APIs.
Using Mime APIs:
CURL *curl_handle;
curl_mime *mime;
curl_mimepart *part;
mime = curl_mime_init(curl_handle);
part = curl_mime_addpart(mime);
curl_mime_name(part, "sendfile");
curl_mime_filename(part, "uploadfile.txt");
curl_mime_data_cb(part, file_size, read_buffer, NULL, NULL, NULL);
curl_easy_setopt(curl_handle, CURLOPT_URL, url);
curl_easy_setopt(curl_handle, CURLOPT_MIMEPOST, mime);
curl_easy_perform(curl_handle);
The read callback function given in curl_mime_data_cb() reads from the buffer/queue, this buffer/queue data is getting filled by other application which is downloading the file using libcurl.
I hope it helps someone.
I recommend you to try it with xmlstream please check out the code below maybe it will help.
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <expat.h>
#include <curl/curl.h>
struct MemoryStruct {
char *memory;
size_t size;
};
struct ParserStruct {
int ok;
size_t tags;
size_t depth;
struct MemoryStruct characters;
};
static void startElement(void *userData, const XML_Char *name,
const XML_Char **atts)
{
struct ParserStruct *state = (struct ParserStruct *) userData;
state->tags++;
state->depth++;
/* Get a clean slate for reading in character data. */
free(state->characters.memory);
state->characters.memory = NULL;
state->characters.size = 0;
}
static void characterDataHandler(void *userData, const XML_Char *s, int len)
{
struct ParserStruct *state = (struct ParserStruct *) userData;
struct MemoryStruct *mem = &state->characters;
char *ptr = realloc(mem->memory, mem->size + len + 1);
if(!ptr) {
/* Out of memory. */
fprintf(stderr, "Not enough memory (realloc returned NULL).\n");
state->ok = 0;
return;
}
mem->memory = ptr;
memcpy(&(mem->memory[mem->size]), s, len);
mem->size += len;
mem->memory[mem->size] = 0;
}
static void endElement(void *userData, const XML_Char *name)
{
struct ParserStruct *state = (struct ParserStruct *) userData;
state->depth--;
printf("%5lu %10lu %s\n", state->depth, state->characters.size, name);
}
static size_t parseStreamCallback(void *contents, size_t length, size_t nmemb,
void *userp)
{
XML_Parser parser = (XML_Parser) userp;
size_t real_size = length * nmemb;
struct ParserStruct *state = (struct ParserStruct *) XML_GetUserData(parser);
/* Only parse if we're not already in a failure state. */
if(state->ok && XML_Parse(parser, contents, real_size, 0) == 0) {
int error_code = XML_GetErrorCode(parser);
fprintf(stderr, "Parsing response buffer of length %lu failed"
" with error code %d (%s).\n",
real_size, error_code, XML_ErrorString(error_code));
state->ok = 0;
}
return real_size;
}
int main(void)
{
CURL *curl_handle;
CURLcode res;
XML_Parser parser;
struct ParserStruct state;
/* Initialize the state structure for parsing. */
memset(&state, 0, sizeof(struct ParserStruct));
state.ok = 1;
/* Initialize a namespace-aware parser. */
parser = XML_ParserCreateNS(NULL, '\0');
XML_SetUserData(parser, &state);
XML_SetElementHandler(parser, startElement, endElement);
XML_SetCharacterDataHandler(parser, characterDataHandler);
/* Initialize a libcurl handle. */
curl_global_init(CURL_GLOBAL_DEFAULT);
curl_handle = curl_easy_init();
curl_easy_setopt(curl_handle, CURLOPT_URL,
"https://www.w3schools.com/xml/simple.xml");
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, parseStreamCallback);
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *)parser);
printf("Depth Characters Closing Tag\n");
/* Perform the request and any follow-up parsing. */
res = curl_easy_perform(curl_handle);
if(res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
}
else if(state.ok) {
/* Expat requires one final call to finalize parsing. */
if(XML_Parse(parser, NULL, 0, 1) == 0) {
int error_code = XML_GetErrorCode(parser);
fprintf(stderr, "Finalizing parsing failed with error code %d (%s).\n",
error_code, XML_ErrorString(error_code));
}
else {
printf(" --------------\n");
printf(" %lu tags total\n", state.tags);
}
}
/* Clean up. */
free(state.characters.memory);
XML_ParserFree(parser);
curl_easy_cleanup(curl_handle);
curl_global_cleanup();
return 0;
}
I have the following code, and I am getting SIGSEGV on the line:
if ( SSL_connect(ssl) == FAIL )
The fault Im getting is:
Program received signal SIGSEGV, Segmentation fault.
0x00007ffffe5a41e0 in __GI___libc_malloc (bytes=104) at malloc.c:2926
2926 malloc.c: No such file or directory.
The program basically is designed to take loads of data and push it into firebase.
The first one element, is to check if we are registered, the next bit is to actually do the registration.
Cutting the program back to basics, we have the following opening gambit:
int main(int argc, char *argv[]) {
int iRegistered = checkRegistered();
int result = registerCar();
}
If we swap those two lines, so we register before we check the registration, then we don't get a SIGSEGV.
Here's the checkRegistration function:
int checkRegistered() {
int firebaseRegistered = 0;
char *carId;
carId = (char *) malloc(256);
strcpy(carId, "aabbccddeeffgg" );
char *payload;
payload = (char *) malloc(1024);
sprintf(payload, "{ \"carid\": \"%s\" }", carId);
char *response;
response = (char *) malloc(1024);
int result = firebase("isCarRegistered", payload, &response);
if (result == 0) {
// Process JSON Response
cJSON *json = cJSON_Parse(response);
if (json == NULL) {
//
} else {
cJSON *json_registered = NULL;
json_registered = cJSON_GetObjectItemCaseSensitive(json, "registered");
firebaseRegistered = json_registered->valueint;
}
}
free(response);
free(payload);
free(carId);
return firebaseRegistered;
}
And the registerCar function.
They're basically mostly the same format - construct a message, send it to firebase, process the JSON response. We use cJSON to decompile the data returned from Firebase, though we could potentially use it to also compile. But one thing at a time.
You'll see a number of free() statements - I've been trying to work out how best to complete this - ie, generate a char* locally, pass by reference ** to a function, let the function perform the malloc/realloc based on the sizes it can calculate and then we can free it from the calling code once we have dealth with the data. Though I also get a SIGSEGV from that as well.
int registerCar() {
int iResponse = 0;
char *carId;
carId = (char *) malloc(256);
char *authCode;
authCode = (char *) malloc(12);
char *payload;
payload = (char *) malloc(1024);
sprintf(payload, "{ }");
char *response;
response = (char *) malloc(1024);
int result = firebase("registerCar", payload, &response);
if (result == 0) {
// Process JSON Response
cJSON *json = cJSON_Parse(response);
if (json == NULL) {
//
} else {
cJSON *json_auth = NULL;
cJSON *json_car = NULL;
json_auth = cJSON_GetObjectItemCaseSensitive(json, "authcode");
json_car = cJSON_GetObjectItemCaseSensitive(json, "carid");
iResponse = 1;
}
}
free(response);
free(payload);
return iResponse;
}
Here's the firebase routine, it takes a function, a payload and generates a response. Interestingly here, char firebaseLocal and charfirebaseMessage is not always null before the initial malloc.
int firebase(char *firebaseFunction, char *firebasePayload, char **firebaseResponse) {
char buf[1024];
char *firebaseLocal;
char *firebaseMessage;
firebaseMessage = (char *) malloc(1024);
SSL_CTX *ctx;
int server;
SSL *ssl;
int bytes;
ctx = InitCTX();
server = OpenConnection(HOST, atoi(PORT));
ssl = SSL_new(ctx); /* create new SSL connection state */
SSL_set_fd(ssl, server); /* attach the socket descriptor */
if ( SSL_connect(ssl) == FAIL ) /* perform the connection */
ERR_print_errors_fp(stderr);
else {
ShowCerts(ssl); /* get any certs */
char *firebasePost;
generatePostMessage(firebaseFunction, firebasePayload, &firebasePost);
SSL_write(ssl, firebasePost, strlen(firebasePost));
bytes = SSL_read(ssl, buf, sizeof(buf)); /* get reply & decrypt */
buf[bytes] = 0;
//SSL_free(ssl); /* release connection state */
strcpy(firebaseMessage, buf);
firebaseLocal = strstr(firebaseMessage, "\r\n\r\n");
if (firebaseLocal != NULL) {
firebaseLocal +=4;
}
strcpy(*firebaseResponse, firebaseLocal);
}
free(firebaseMessage);
close(server); /* close socket */
SSL_CTX_free(ctx); /* release context */
return 0;
}
This is from an implementation I found on secure sockets.
int OpenConnection(const char *hostname, int port)
{ int sd;
struct hostent *host;
struct sockaddr_in addr;
if ( (host = gethostbyname(hostname)) == NULL )
{
perror(hostname);
abort();
}
sd = socket(PF_INET, SOCK_STREAM, 0);
bzero(&addr, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_port = htons(port);
addr.sin_addr.s_addr = *(long*)(host->h_addr);
if ( connect(sd, (struct sockaddr*)&addr, sizeof(addr)) != 0 )
{
close(sd);
perror(hostname);
abort();
}
return sd;
}
This is from an implementation I found on secure sockets.
SSL_CTX* InitCTX(void)
{
SSL_METHOD *method;
SSL_CTX *ctx;
SSL_library_init();
OpenSSL_add_all_algorithms(); /* Load cryptos, et.al. */
SSL_load_error_strings(); /* Bring in and register error messages */
method = TLSv1_2_client_method(); /* Create new client-method instance */
ctx = SSL_CTX_new(method); /* Create new context */
if ( ctx == NULL )
{
ERR_print_errors_fp(stderr);
abort();
}
return ctx;
}
This is from an implementation I found on secure sockets.
void ShowCerts(SSL* ssl)
{ X509 *cert;
char *line;
cert = SSL_get_peer_certificate(ssl); /* get the server's certificate */
if ( cert != NULL )
{
printf("Server certificates:\n");
line = X509_NAME_oneline(X509_get_subject_name(cert), 0, 0);
printf("Subject: %s\n", line);
free(line); /* free the malloc'ed string */
line = X509_NAME_oneline(X509_get_issuer_name(cert), 0, 0);
printf("Issuer: %s\n", line);
free(line); /* free the malloc'ed string */
X509_free(cert); /* free the malloc'ed certificate copy */
}
else
printf("Info: No client certificates configured.\n");
}
This is something that I wrote to generate a post message from message
void generatePostMessage(char *firebaseFunction, char *firebaseMessage, char **response) {
int intPayloadSize = strlen(firebaseMessage);
char *charPayloadSize;
charPayloadSize = (char *) malloc(8);
sprintf(charPayloadSize, "%d", intPayloadSize);
char *postmessage = "POST /%s HTTP/1.1\r\n"
"Host: us-central1-carconnect-e763e.cloudfunctions.net\r\n"
"User-Agent: USER_AGENT\r\n"
"Content-Type: application/json\r\n"
"Accept: text/plain\r\n"
"Content-Length: %d\r\n\r\n"
"%s";
// Allocate size of postmessage less the inserts, plus the payload size, plus the payload size digits, plus null
int responseLength = (strlen(postmessage) - 4) + intPayloadSize + strlen(charPayloadSize)+1;
// Round up Four Bytes.
int responseIncrease = responseLength % 4;
if (responseIncrease > 0) {
responseLength += (4 - responseIncrease);
}
*response = (char *) malloc(responseLength);
sprintf(*response, postmessage, firebaseFunction, intPayloadSize, firebaseMessage);
}
As advised, whether the registration or registration check is called first, the first call works fine.
If I perform the registration before the check, then both commands work fine. Further testing also does confirm the problem is the registration check. I can perform registration several times without fail. The registration check and any follow up calls fail completely at the SSL_connect line. I don't know why.
The SSL_free command in the firebase connection always fails. I also get a SIGSEGV if I try to free(firebasePost) after the SSL_Write - which suggests I cannot free a pointer that has been passed by reference and mallocced in a function.
Part of me wonders whether any of this is caused by the fact Im debugging on Windows. I've always had problems with malloc() on Windows just not working the way I would expect.
The problem, or at least one of them, is in generatePostMessage. Not enough buffer is allocated for response. sprintf will then run off the end of the allocated buffer and cause heap corruption, which manifests itself on next invocation of malloc. Try:
int responseLength = strlen(firebaseFunction) + (strlen(postmessage) - 4) + intPayloadSize + strlen(charPayloadSize)+1;
I'm trying to implement a web prefetching system. The purpose of a system like this is to “predict” future requests and prefetch them.
The system builds a predictive model from web navigation logs (Squid access.log files). The model is a dependency graph, where a node representing URL A has an arc to a node representing URL B if URL B has been requested immediately after URL A.
Once the model is built, the system receives queries of URLs requested by users, and make “predictions” based on the graph. Predictions are resources (URLs) very likely to be requested in the future. So, based on predictions, the system prefetches these resources to store them in cache prior to users' requests.
I'm using the following testing scenario:
A process simulate multiple clients, requesting URLs in a file using libcurl. The process runs in a different PC from the prefetching system. PCs are connected directly via an ethernet cable
Requests made by the client simulator are always the same URLs in the same relative time from the first request made. All requests are going to port 3128 (Prefetch PC Squid listen port) (port 80 DNAT to port 3128 in the client).
The prefetching system runs in a CentOS 6.3 box, kernel 2.6.32-71.el6.i686, 2 core Intel Pentium 4 3.00GHz processor, 4 GB RAM.
The prefetching system is one process with multiple threads. The main thread creates the predictive model and generates predictions based on queries. A “listener” thread reads URLs requested by users and prefetches predicted URLs using libcurl. “Listening” means reading from a named pipe (called url_fifo) URLs captured live on an interface using tshark:
stdbuf -o0 tshark -i eth1 tcp port 3128 and "tcp[((tcp[12:1] & 0xf0) >> 2):4] = 0x47455420" -T fields -e http.request.full_uri >> url_fifo
Each 10 minutes (1 cycle) the model is updated based on requests from the last cycle. The client tells the system when a cycle ends and so the model is updated. Once the model is updated, the system tells the client to start requesting URLs from the next cycle.
Here is the situation: Sometimes reading from the named pipe freezes. No URLs are read from the pipe even though tshark keeps capturing URLs and redirecting them to the named pipe. After an hour (or a couple of hours) all “buffered” URLs are read in less than 10 minutes. After that, reading from the pipe keeps going ok again. This situation doesn't happen always (50% of times freezes, 50% no).
It seems that there is a buffering issue, since tshark keeps capturing URLs and all requests are correctly logged in Squid's access.log.
In the beginning, I ran tshark with the -l option, so that its output becomes line buffered. Then I started using stdbuf -o0 (no buffering). Anyway the situation still happens.
In the system code, I also tried opening and reading the named pipe as a stream (FILE *) and set the stream as no buffered or line buffered (using setvbuf() function). The situation still happened.
In some cycles requests are faster than in other cycles. Anyway, it doesn't seems to be a fast producer slow consumer issue, since in many repetitions of the test all URLs are correctly read and processed without any freezes.
Is there something am I missing related to named pipes and buffering? I'd really appreciate some guidance.
Assume networking (interfaces, routing, iptables, squid) is ok. I've not had any issues related to it.
Code (assume necessary header files are included):
functions.c
#define BUFLEN 512
#define QUEUE_LEN 64
#define THREADS_LEN 2
pthread_mutex_t model_lock;
pthread_cond_t model_cond, listen_cond;
pthread_t queries_thread, listen_thread;
short int model_is_updating, model_can_update, program_shutdown;
/* Program execution statistics */
Status * program_status;
/* Thread pool */
threadpool_t *pool;
/* program execution */
int
run(void)
{
Graph_Adj_List * gr = NULL; /* Graph as an adjacency list */
char ** reports = NULL;
unsigned report_counter = 0;
/* Init program status */
program_status = status_init();
/* Load list of custom web navigation reports to be used to build the initial
* version of the predictive model */
reports = file_load_reports(program_config.reports_file);
if (!reports)
return 0;
/* Init mutex and cond */
pthread_mutex_init(&model_lock, NULL);
pthread_cond_init(&model_cond, NULL);
pthread_cond_init(&listen_cond, NULL);
/* Lock */
pthread_mutex_lock (&model_lock);
/* Start first cycle */
status_start_cycle(program_status);
/* Create initial version of the predictive model */
gr = create_model_from_files(reports, &report_counter, program_config.reports_limit);
if (!gr)
{
/* Unlock */
pthread_mutex_unlock (&model_lock);
return 0;
}
/* Unlock */
pthread_mutex_unlock (&model_lock);
/* Start threads */
if (pthread_create(&queries_thread, NULL, fifo_predictions_threaded, (void *)gr) ||
pthread_create(&listen_thread, NULL, listen_end_of_cycle, NULL))
program_shutdown = 1;
/* main loop */
while(!program_shutdown)
{
/* lock */
pthread_mutex_lock (&model_lock);
/* wait for clients' announcement of the end of requests from current cycle */
while (!model_can_update)
pthread_cond_wait(&model_cond, &model_lock);
/* set updating flag */
model_is_updating = 1;
/* Update predictive model, based on Squid's access.log from (about to finish)
* current cycle */
adj_list_update_access(gr, program_config.access_file);
/* Save statistics related to the current cycle and finish it */
status_finish_cycle(program_status);
/* Check if last custom report has been read */
if (!reports[report_counter])
{
program_shutdown = 1;
pthread_mutex_unlock (&model_lock);
break;
}
/* Start a new cycle */
status_start_cycle(program_status);
/* Read a new custom report and update the predictive model */
update_model(gr, reports[report_counter]);
report_counter++;
/* Updating is done */
model_is_updating = 0;
/* Model can't be updated until client announces the end of the cycle
* that has just started */
model_can_update = 0;
/* Tell client to start sending requests from the new cycle */
if (!signal_start_cycle())
{
program_shutdown = 1;
pthread_mutex_unlock (&model_lock);
break;
}
/* Signal listener thread that a new cycle has begin */
pthread_cond_signal(&listen_cond);
/* Unlock */
pthread_mutex_unlock (&model_lock);
}
/* Finish threads */
pthread_cancel(listen_thread);
pthread_cancel(queries_thread);
pthread_join(listen_thread, NULL);
pthread_join(queries_thread, NULL);
/* Free memory */
adj_list_free_all2(&gr);
file_reports_free_all(&reports);
pthread_cond_destroy(&model_cond);
pthread_cond_destroy(&listen_cond);
pthread_mutex_destroy(&model_lock);
status_free(&program_status);
return 1;
}
void *
fifo_predictions_threaded(void * data)
{
Graph_Adj_List * gr = (Graph_Adj_List *) data;
/* Set thread cancel type */
pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
while (!program_shutdown)
{
pthread_mutex_lock(&model_lock);
/* Pause reading from named pipe while the model is being updated */
while(model_is_updating)
pthread_cond_wait(&listen_cond, &model_lock);
pthread_mutex_unlock(&model_lock);
/* Read URLs from named pipe */
fifo_predictions(gr, program_config.fifo);
}
pthread_exit(NULL);
return NULL;
}
int
fifo_predictions(Graph_Adj_List * gr, const u8 * fifo)
{
u8 cad[BUFLEN] = { '\0' };
u8 * ini = NULL, * fin = NULL, * fullurl = NULL;
int i, fifo_descriptor, read_urls = 0, fullurl_len = 0, incomplete_url = 1;
FILE * fifo_file = NULL;
/* Open fifo in blocking mode */
fifo_descriptor = open(CHAR_CAST fifo, O_RDONLY);
/* Open fifo as a stream */
// fifo_file = fopen(fifo, "r");
// if (!fifo_file)
if (fifo_descriptor == -1)
return 0;
/* If fifo is opened as a stream, set it line buffered */
// setlinebuf(fifo_file);
do
{
if ((i = read(fifo_descriptor, cad, BUFLEN - 1)) == -1)
// if ( fgets(cad, BUFLEN-1, fifo_file) == NULL)
ERROR(__FILE__, __FUNCTION__, __LINE__, "Fifo read error");
else
{
// i = strlen(cad);
cad[i] = '\0';
read_urls = 0;
if (i > 0)
{
int j = 0;
for (j = 0, ini = cad, fin = NULL ; cad[j] != '\0'; j++)
{
if (cad[j] == '\n')
{
/* Save URL */
fin = &cad[j];
ini = (*ini == '\n' ? ini + 1 : ini);
/* Check if string is a continuation of the previously read URL */
read_urls = fin - ini;
read_urls = read_urls >= 0 ? read_urls : 0;
/* Save URL in fullurl string */
fullurl = realloc(fullurl, fullurl_len + read_urls + 1);
memcpy(&fullurl[fullurl_len], ini, read_urls);
fullurl[fullurl_len + read_urls] = '\0';
ini = fin;
incomplete_url = fullurl_len = 0;
/* Ask the model for predictions and fetch them */
fetch_url_predictions2(gr, fullurl);
u8_free(&fullurl);
} else
incomplete_url = 1;
}
if (incomplete_url)
{
ini = (*ini == '\n' ? ini + 1 : ini);
read_urls = &cad[j] - ini;
read_urls = read_urls >= 0 ? read_urls : 0;
fullurl = realloc(fullurl, fullurl_len + read_urls);
memcpy(&fullurl[fullurl_len], ini, read_urls);
fullurl_len += read_urls;
}
}
}
} while (i > 0);
close(fifo_descriptor);
// fclose (fifo_file);
return 1;
}
int
fetch_url_predictions2(Graph_Adj_List * gr, u8 * in_url)
{
String * string_url = NULL;
Headnode * head = NULL;
LinkedList * list = NULL;
LinkedListElem * elem = NULL;
/* Use custom string type */
string_url = string_create_no_len(in_url);
if (!string_url)
return 0;
pthread_mutex_lock(&model_lock);
/* Get URL node */
head = adj_list_get_node(gr, string_url);
if (head)
{
/* Get predictions (URLs) as a linked list */
list = adj_list_predictions_to_list(head);
if (!list)
{
string_free_all(&string_url);
return 0;
}
pthread_mutex_unlock(&model_lock);
/* Callback fetches URLs */
list->callback = &curl_callback_void;
if (!pool)
pool = threadpool_create(THREADS_LEN, QUEUE_LEN, 0);
/* Load URLs to be fetched to threadpool's task queue */
for (elem = list->first; elem; elem = elem->next)
{
CallbackArg arg;
arg.data = arg.copy(elem->data);
threadpool_add_copy_arg(pool, list->callback, &arg, 1, sizeof(arg), 0);
}
linked_list_free_all(&list);
}
pthread_mutex_unlock(&model_lock);
string_free_all(&string_url);
return 1;
}
fetch.c
void
curl_callback_void(void * data)
{
CallbackArg * arg = (CallbackArg *) data;
char * url = (char *) arg->data;
fetch_url(url);
}
static size_t
write_data(void *buffer, size_t size, size_t nmemb, void *userp)
{
return size * nmemb;
}
int
fetch_url(char * url)
{
CURL *curl;
CURLcode res;
struct timeval time;
char * time_string = NULL;
curl = curl_easy_init();
if (curl)
{
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL);
curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 15);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10);
/* Perform the request, res will get the return code */
res = curl_easy_perform(curl);
gettimeofday(&time, NULL);
time_string = timeval_to_str(&time);
/* Check for errors */
if (res != CURLE_OK)
{
fprintf(stderr, "\ntime %s curl_easy_perform() (url %s) failed: %s\n",
time_string, url, curl_easy_strerror(res));
}
else
{
fprintf(stderr, "\ntime %s curl_easy_perform() (url %s) fetched ok\n",
time_string, url);
}
fflush(stderr);
free (time_string);
curl_easy_cleanup(curl);
}
return 0;
}
network.c
/*
* Code based on Beej's Networking Guide
*/
#define MSG_LEN 5
#define QUEUE_SIZE 5
extern pthread_mutex_t model_lock;
extern pthread_cond_t model_cond;
extern short int model_can_update, program_shutdown;
extern Config program_config;
// get sockaddr, IPv4 or IPv6:
static void *
get_in_addr(struct sockaddr *sa) {
if (sa->sa_family == AF_INET) {
return &(((struct sockaddr_in*) sa)->sin_addr);
}
return &(((struct sockaddr_in6*) sa)->sin6_addr);
}
void *
listen_end_of_cycle(void * data)
{
int sockfd, new_fd; // listen on sock_fd, new connection on new_fd
struct addrinfo hints, *servinfo, *p;
struct sockaddr_storage their_addr; // connector's address information
socklen_t sin_size;
int yes = 1;
char s[INET_ADDRSTRLEN], msg[MSG_LEN], *str = NULL;
int rv;
int read_bytes;
struct timeval actual_time;
/* Set thread cancel type */
pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL );
memset(&hints, 0, sizeof hints);
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
hints.ai_flags = AI_PASSIVE; // use my IP
if ((rv = getaddrinfo(NULL, program_config.listen_port, &hints, &servinfo))
!= 0) {
fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(rv));
return "error";
}
// loop through all the results and bind to the first we can
for (p = servinfo; p != NULL ; p = p->ai_next) {
if ((sockfd = socket(p->ai_family, p->ai_socktype, p->ai_protocol))
== -1) {
perror("server: socket");
continue;
}
if (setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int))
== -1) {
perror("setsockopt");
return "error";
}
if (bind(sockfd, p->ai_addr, p->ai_addrlen) == -1) {
close(sockfd);
perror("server: bind");
continue;
}
break;
}
if (p == NULL ) {
fprintf(stderr, "server: failed to bind\n");
return "error";
}
freeaddrinfo(servinfo); // all done with this structure
if (listen(sockfd, QUEUE_SIZE) == -1) {
perror("listen");
return "error";
}
while (!program_shutdown)
{
sin_size = sizeof their_addr;
new_fd = accept(sockfd, (struct sockaddr *) &their_addr, &sin_size);
if (new_fd == -1) {
perror("accept");
continue;
}
inet_ntop(their_addr.ss_family,
get_in_addr((struct sockaddr *) &their_addr), s, sizeof s);
if ((read_bytes = recv(new_fd, msg, MSG_LEN - 1, 0)) == -1) {
perror("recv");
continue;
}
close(new_fd);
msg[read_bytes] = '\0';
/* Check received message */
if (strcmp(msg, "DONE")) {
perror("Not valid message");
continue;
}
printf("\ngot \"DONE\" from %s\n", s);
fflush(stdout);
/* Lock */
pthread_mutex_lock(&model_lock);
/* Flag used by main thread to allow model update */
model_can_update = 1;
/* Signal model can be updated */
pthread_cond_signal(&model_cond);
/* Unlock */
pthread_mutex_unlock(&model_lock);
}
close(sockfd);
pthread_exit(NULL);
return "ok";
}
int signal_start_cycle(void) {
int sockfd;
struct addrinfo hints, *servinfo, *p;
int rv;
char s[INET6_ADDRSTRLEN], *str = NULL;
struct timeval actual_time, aux_time;
struct timeval connect_timeout = { 15, 0 }, max_connect_time = { 0, 0 };
short int connected = 0;
memset(&hints, 0, sizeof hints);
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
if ((rv = getaddrinfo(program_config.client_ip, program_config.client_port,
&hints, &servinfo)) != 0) {
fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(rv));
return 0;
}
gettimeofday(&aux_time, NULL);
timeval_add(aux_time, connect_timeout, &max_connect_time);
/* Try several times to connect to the remote side */
do {
// loop through all the results and connect to the first we can
for (p = servinfo; p != NULL ; p = p->ai_next) {
if ((sockfd = socket(p->ai_family, p->ai_socktype, p->ai_protocol))
== -1) {
perror("client: socket");
continue;
}
gettimeofday(&actual_time, NULL )
printf("\ntrying to connect %s\n", program_config.client_ip);
fflush(stdout);
if (connect(sockfd, p->ai_addr, p->ai_addrlen) == -1) {
close(sockfd);
perror("client: connect");
continue;
}
connected = 1;
break;
}
} while (!connected && !timeval_greater_than(actual_time, max_connect_time));
if (p == NULL ) {
fprintf(stderr, "client: failed to connect\n");
return 0;
}
inet_ntop(p->ai_family, get_in_addr((struct sockaddr *) p->ai_addr), s,
sizeof s);
printf("\nMAIN THREAD: connecting to %s\n", s);
fflush(stdout);
freeaddrinfo(servinfo); // all done with this structure
if (send(sockfd, "DONE", 4, 0) == -1)
{
perror("send");
return 0;
}
printf("\nsent \"DONE\" to %s\n", s);
fflush(stdout);
close(sockfd);
return 1;
}
Below is my code for building a SOAP request:
#include <stdio.h>
#include <string.h>
#include <curl/curl.h>
/* Auxiliary function that waits on the socket. */
static int wait_on_socket(curl_socket_t sockfd, int for_recv, long timeout_ms)
{
struct timeval tv;
fd_set infd, outfd, errfd;
int res;
tv.tv_sec = timeout_ms / 1000;
tv.tv_usec= (timeout_ms % 1000) * 1000;
FD_ZERO(&infd);
FD_ZERO(&outfd);
FD_ZERO(&errfd);
FD_SET(sockfd, &errfd); /* always check for error */
if(for_recv)
{
FD_SET(sockfd, &infd);
}
else
{
FD_SET(sockfd, &outfd);
}
/* select() returns the number of signalled sockets or -1 */
res = select(sockfd + 1, &infd, &outfd, &errfd, &tv);
return res;
}
int main(void)
{
CURL *ch;
CURLcode res;
struct curl_slist *headerlist=NULL;
struct MemoryStruct *bodyStruct=NULL;
char _gatineauSoapReq[2000]="";
const char *WriteMemoryCallback;
/* Minimalistic http request */
curl_socket_t sockfd; /* socket */
long sockextr;
size_t iolen;
ch = curl_easy_init();
curl_easy_setopt(ch, CURLOPT_URL, "http://thermo.sdsu.edu/servlet/ThermodynamicProperties/ThermodynamicPropertiesService");
curl_easy_setopt(ch, CURLOPT_POST, 1);
curl_easy_setopt(ch, CURLOPT_HEADER, 1);
curl_easy_setopt(ch, CURLOPT_VERBOSE, 1);
curl_easy_setopt(ch, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
if ((bodyStruct = (struct MemoryStruct *) malloc(sizeof(struct MemoryStruct))) == NULL) exit(1);
curl_easy_setopt(ch, CURLOPT_FILE, bodyStruct);
curl_slist_free_all(headerlist);
headerlist = curl_slist_append(headerlist, "Content-Type: text/xml");
headerlist = curl_slist_append(headerlist, "SOAPAction: \"http://blabla.com/blabla_services/ValidateNow\"");
sprintf(_gatineauSoapReq, "<?xml version=\"1.0\" encoding=\"utf-8\"?>\
<S:Envelope xmlns:S="http://schemas.xmlsoap.org/soap/envelope/">
<S:Body>
<ns2:getSpeciesInformationResponse
xmlns:ns2="http://ThermodynamicProperties/">
<return>
{"Nist":true,"Abinitio":[["3-21G","MP2",""]],"Nasa":true,"Chemkin":true,"Burcat":true}
</return>
</ns2:getSpeciesInformationResponse>
</S:Body>
</S:Envelope>
curl_easy_setopt(ch, CURLOPT_POSTFIELDS, _gatineauSoapReq);
curl_easy_setopt(ch, CURLOPT_HTTPHEADER, headerlist);
curl_easy_perform(ch);
//res = curl_easy_perform(curl);
if(CURLE_OK != res)
{
printf("Error: %s\n", strerror(res));
return 1;
}
/* Extract the socket from the curl handle - we'll need it for waiting.
* Note that this API takes a pointer to a 'long' while we use
* curl_socket_t for sockets otherwise.
*/
res = curl_easy_getinfo(curl, CURLINFO_LASTSOCKET, &sockextr);
if(CURLE_OK != res)
{
printf("Error: %s\n", curl_easy_strerror(res));
return 1;
}
sockfd = sockextr;
/* wait for the socket to become ready for sending */
if(!wait_on_socket(sockfd, 0, 60000L))
{
printf("Error: timeout.\n");
return 1;
}
puts("Sending request.");
/* Send the request. Real applications should check the iolen
* to see if all the request has been sent */
//res = curl_easy_send(curl,, strlen(request), &iolen);
if(CURLE_OK != res)
{
printf("Error: %s\n", curl_easy_strerror(res));
return 1;
}
puts("Reading response.");
/* read the response */
//for(;;)
{
printf("ok1 \n");
char buf[1024];
wait_on_socket(sockfd, 1, 60000L);
res = curl_easy_recv(curl, buf, 1024, &iolen);
printf("ok2 \n");
if(CURLE_OK != res)
//break;
printf("Error: %s\n", strerror(res));
//printf("ok3 \n");
//printf("Received %u bytes.\n", iolen);
printf("data %s \n", buf);
}
/* always cleanup */
curl_easy_cleanup(curl);
return 0;
}
Also along with building the SOAP request, how should one send the request, should we use the curl_easy_send().
I am getting the following error:
curls.c: In function âmainâ:
curls.c:57:45: warning: incompatible implicit declaration of built-in function âmallocâ
curls.c:57:59: error: invalid application of âsizeofâ to incomplete type âstruct MemoryStructâ
curls.c:57:91: warning: incompatible implicit declaration of built-in function âexitâ
curls.c:63:72: warning: backslash and newline separated by space
curls.c:64:22: error: expected â)â before âhttpâ
curls.c:69:60: error: invalid suffix "G" on integer constant
curls.c:146:1: error: expected â;â before â}â token
It seems you have syntax mistakes or missing declarations - have a look at the error lines you provide from your log.
Can I send a Json string using libcurl. I am new to both. Any kind of help would be appreciated. Basically i want to send a Json string using libcurl simple send and receive in C
My code is below:
#include <stdio.h>
#include <string.h>
#include <curl/curl.h>
/* Auxiliary function that waits on the socket. */
static int wait_on_socket(curl_socket_t sockfd, int for_recv, long timeout_ms)
{
struct timeval tv;
fd_set infd, outfd, errfd;
int res;
tv.tv_sec = timeout_ms / 1000;
tv.tv_usec= (timeout_ms % 1000) * 1000;
FD_ZERO(&infd);
FD_ZERO(&outfd);
FD_ZERO(&errfd);
FD_SET(sockfd, &errfd); /* always check for error */
if(for_recv)
{
FD_SET(sockfd, &infd);
}
else
{
FD_SET(sockfd, &outfd);
}
/* select() returns the number of signalled sockets or -1 */
res = select(sockfd + 1, &infd, &outfd, &errfd, &tv);
return res;
}
int main(void)
{
CURL *curl;
CURLcode res;
/* Minimalistic http request */
const char *request = "reactantsJSON={"O=O":{"N":1}}&productsJSON=["O=O","[O]"]&temperature=2273.15&pressure=101.325";
curl_socket_t sockfd; /* socket */
long sockextr;
size_t iolen;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "http://gibbs.sdsu.edu:8080/axis2/services/GibbsMinimization/solveTP");
/* Do not do the transfer - only connect to host */
curl_easy_setopt(curl, CURLOPT_CONNECT_ONLY, 1L);
res = curl_easy_perform(curl);
if(CURLE_OK != res)
{
printf("Error: %s\n", strerror(res));
return 1;
}
/* Extract the socket from the curl handle - we'll need it for waiting.
* Note that this API takes a pointer to a 'long' while we use
* curl_socket_t for sockets otherwise.
*/
res = curl_easy_getinfo(curl, CURLINFO_LASTSOCKET, &sockextr);
if(CURLE_OK != res)
{
printf("Error: %s\n", curl_easy_strerror(res));
return 1;
}
sockfd = sockextr;
/* wait for the socket to become ready for sending */
if(!wait_on_socket(sockfd, 0, 60000L))
{
printf("Error: timeout.\n");
return 1;
}
puts("Sending request.");
/* Send the request. Real applications should check the iolen
* to see if all the request has been sent */
res = curl_easy_send(curl, request, strlen(request), &iolen);
if(CURLE_OK != res)
{
printf("Error: %s\n", curl_easy_strerror(res));
return 1;
}
puts("Reading response.");
/* read the response */
for(;;)
{
char buf[1024];
wait_on_socket(sockfd, 1, 60000L);
res = curl_easy_recv(curl, buf, 1024, &iolen);
if(CURLE_OK != res)
break;
printf("Received %u bytes.\n", iolen);
}
/* always cleanup */
curl_easy_cleanup(curl);
}
return 0;
}
the error I am getting is that it says a ";" is missing in line 40, which is nothing but the json string, even though I ahve a ";" on that line. It looks like to send json string i need to do something, which I am not aware of. Do I need to specify some flags while compiling the code.
const char *request = "reactantsJSON={"O=O":{"N":1}}&productsJSON=["O=O","[O]"]&temperature=2273.15&pressure=101.325";
You need to escape " in your string literal.
const char *request = "reactantsJSON={\"O=O\":{\"N\":1}}&productsJSON=[\"O=O\",\"[O]\"]&temperature=2273.15&pressure=101.325";