File stream to HTTP server using libcurl - c

Question

I am trying to develop a C application to upload a file on HTTP endpoint.
I used below command from terminal to upload file, which works fine,
curl -F file=#/filename.txt -O http://serveraddress/download
Since, I need it in C application I took reference from https://curl.haxx.se/libcurl/c/postit2-formadd.html which is also working fine. the file supplied is getting uploaded successfully.
I want to upload the file similar way but the file is not present on local storage. The file is being downloaded in an application using Curl. From the beginning, we have the size of the file which will be downloaded. I want to stream that file directly to the HTTP endpoint as above(multipart file upload) because I don't have enough local storage to download entire file and then upload.
File--> CurlDownloadHandle-->Write() ----- Read()-->CurlUploadHandle---File
I am not sure how to use HTTPPOST, HTTPPOSTFILED and all. Is there anyway I can achieve this streaming?
Any suggestion is much appreciated.
Thanks.

Answer 1

I have figured out below method to achieve my requirement.
File streaming can be achieved using curl_formadd(for older version of libcurl)and curl_mime(for newer version > 7.58.0) APIs.
Using Mime APIs:
CURL *curl_handle;
curl_mime *mime;
curl_mimepart *part;
mime = curl_mime_init(curl_handle);
part = curl_mime_addpart(mime);
curl_mime_name(part, "sendfile");
curl_mime_filename(part, "uploadfile.txt");
curl_mime_data_cb(part, file_size, read_buffer, NULL, NULL, NULL);
curl_easy_setopt(curl_handle, CURLOPT_URL, url);
curl_easy_setopt(curl_handle, CURLOPT_MIMEPOST, mime);
curl_easy_perform(curl_handle);
The read callback function given in curl_mime_data_cb() reads from the buffer/queue, this buffer/queue data is getting filled by other application which is downloading the file using libcurl.
I hope it helps someone.

Answer 2

I recommend you to try it with xmlstream please check out the code below maybe it will help.
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <expat.h>
#include <curl/curl.h>
struct MemoryStruct {
char *memory;
size_t size;
};
struct ParserStruct {
int ok;
size_t tags;
size_t depth;
struct MemoryStruct characters;
};
static void startElement(void *userData, const XML_Char *name,
const XML_Char **atts)
{
struct ParserStruct *state = (struct ParserStruct *) userData;
state->tags++;
state->depth++;
/* Get a clean slate for reading in character data. */
free(state->characters.memory);
state->characters.memory = NULL;
state->characters.size = 0;
}
static void characterDataHandler(void *userData, const XML_Char *s, int len)
{
struct ParserStruct *state = (struct ParserStruct *) userData;
struct MemoryStruct *mem = &state->characters;
char *ptr = realloc(mem->memory, mem->size + len + 1);
if(!ptr) {
/* Out of memory. */
fprintf(stderr, "Not enough memory (realloc returned NULL).\n");
state->ok = 0;
return;
}
mem->memory = ptr;
memcpy(&(mem->memory[mem->size]), s, len);
mem->size += len;
mem->memory[mem->size] = 0;
}
static void endElement(void *userData, const XML_Char *name)
{
struct ParserStruct *state = (struct ParserStruct *) userData;
state->depth--;
printf("%5lu %10lu %s\n", state->depth, state->characters.size, name);
}
static size_t parseStreamCallback(void *contents, size_t length, size_t nmemb,
void *userp)
{
XML_Parser parser = (XML_Parser) userp;
size_t real_size = length * nmemb;
struct ParserStruct *state = (struct ParserStruct *) XML_GetUserData(parser);
/* Only parse if we're not already in a failure state. */
if(state->ok && XML_Parse(parser, contents, real_size, 0) == 0) {
int error_code = XML_GetErrorCode(parser);
fprintf(stderr, "Parsing response buffer of length %lu failed"
" with error code %d (%s).\n",
real_size, error_code, XML_ErrorString(error_code));
state->ok = 0;
}
return real_size;
}
int main(void)
{
CURL *curl_handle;
CURLcode res;
XML_Parser parser;
struct ParserStruct state;
/* Initialize the state structure for parsing. */
memset(&state, 0, sizeof(struct ParserStruct));
state.ok = 1;
/* Initialize a namespace-aware parser. */
parser = XML_ParserCreateNS(NULL, '\0');
XML_SetUserData(parser, &state);
XML_SetElementHandler(parser, startElement, endElement);
XML_SetCharacterDataHandler(parser, characterDataHandler);
/* Initialize a libcurl handle. */
curl_global_init(CURL_GLOBAL_DEFAULT);
curl_handle = curl_easy_init();
curl_easy_setopt(curl_handle, CURLOPT_URL,
"https://www.w3schools.com/xml/simple.xml");
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, parseStreamCallback);
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *)parser);
printf("Depth Characters Closing Tag\n");
/* Perform the request and any follow-up parsing. */
res = curl_easy_perform(curl_handle);
if(res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
}
else if(state.ok) {
/* Expat requires one final call to finalize parsing. */
if(XML_Parse(parser, NULL, 0, 1) == 0) {
int error_code = XML_GetErrorCode(parser);
fprintf(stderr, "Finalizing parsing failed with error code %d (%s).\n",
error_code, XML_ErrorString(error_code));
}
else {
printf(" --------------\n");
printf(" %lu tags total\n", state.tags);
}
}
/* Clean up. */
free(state.characters.memory);
XML_ParserFree(parser);
curl_easy_cleanup(curl_handle);
curl_global_cleanup();
return 0;
}