I am writing a function in C and it'll work on Debian 8. What the function does is just downloading a file and scanning it by using cURL and ClamAV. So, there are two functions I wrote url2file() and clamav_scan(). And I want to insert url2file() into clamav_scan(). But, it'll show an error message 'Can't allocate memory'... why? If I delete a line url2file() in clamav_scan(), it works very well.
This below is my code I've written.
Thank you.
// this function scans a file by using ClamAV
int clamav_scan(const char* file_name)
{
int ret = 0;
struct cl_engine* engine = NULL; // clamAV engine
unsigned int signo = 0; // clamAV signature number
const char* path = NULL; // clamAV DB path
const char* virname = NULL; // virus name
unsigned long int scanned = 0; // size of scanned data
char target[MAX_PATH] = "./"; // path of the scan file
strcat(target, file_name);
if ((ret = cl_init(CL_INIT_DEFAULT)) != CL_SUCCESS) {
printf("cl_init() failed: %s\n", cl_strerror(ret));
exit(1);
}
if ((engine = cl_engine_new()) == NULL) {
printf("cl_engine_new() failed\n");
exit(1);
}
if ((path = cl_retdbdir()) == NULL) {
printf("cl_retdbdir() failed\n");
cl_engine_free(engine);
exit(1);
}
if ((ret = cl_load(path, engine, &signo, CL_DB_STDOPT)) != CL_SUCCESS) {
printf("cl_load() failed: %s\n", cl_strerror(ret));
cl_engine_free(engine);
exit(1);
}
if ((ret = cl_engine_compile(engine)) != CL_SUCCESS) {
printf("cl_engine_compile() failed: %s\n", cl_strerror(ret));
cl_engine_free(engine);
exit(1);
}
// this is the inserted function I said
// this function downloads a file by using cURL
url2file(url, file_name);
printf("Scanning %s ...\n", target);
if ((ret = cl_scanfile((const char*)target, &virname, &scanned, engine, CL_SCAN_STDOPT)) == CL_VIRUS) {
printf("Virus detected: %s\n", virname);
}
else {
printf("No virus detected\n");
if (ret != CL_CLEAN)
printf("Error: %s\n", cl_strerror(ret)); // shows message here
}
printf("scanned: %lu\n", scanned);
cl_engine_free(engine);
return EXIT_SUCCESS;
}
I referred to
https://curl.haxx.se/libcurl/c/url2file.html
https://raw.githubusercontent.com/vrtadmin/clamav-faq/master/manual/clamdoc.pdf
.
.
.
+++ added url2file() function
static size_t write_data(void* ptr, size_t size, size_t nmemb, void* stream)
{
size_t written = fwrite(ptr, size, nmemb, (FILE*)stream);
return written;
}
int url2file(const char* url, const char* pagefilename)
{
CURL *curl_handle;
FILE *pagefile;
curl_global_init(CURL_GLOBAL_ALL);
/* init the curl session */
curl_handle = curl_easy_init();
/* set URL to get here */
curl_easy_setopt(curl_handle, CURLOPT_URL, url);
/* Switch on full protocol/debug output while testing */
curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, 1L);
/* disable progress meter, set to 0L to enable and disable debug output */
curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 1L);
/* send all data to this function */
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_data);
/* open the file */
pagefile = fopen(pagefilename, "wb");
if (pagefile) {
/* write the page body to this file handle */
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, pagefile);
/* get it! */
curl_easy_perform(curl_handle);
/* close the header file */
fclose(pagefile);
}
/* cleanup curl stuff */
curl_easy_cleanup(curl_handle);
curl_global_cleanup();
return 0;
}
Related
I am trying to develop a C application to upload a file on HTTP endpoint.
I used below command from terminal to upload file, which works fine,
curl -F file=#/filename.txt -O http://serveraddress/download
Since, I need it in C application I took reference from https://curl.haxx.se/libcurl/c/postit2-formadd.html which is also working fine. the file supplied is getting uploaded successfully.
I want to upload the file similar way but the file is not present on local storage. The file is being downloaded in an application using Curl. From the beginning, we have the size of the file which will be downloaded. I want to stream that file directly to the HTTP endpoint as above(multipart file upload) because I don't have enough local storage to download entire file and then upload.
File--> CurlDownloadHandle-->Write() ----- Read()-->CurlUploadHandle---File
I am not sure how to use HTTPPOST, HTTPPOSTFILED and all. Is there anyway I can achieve this streaming?
Any suggestion is much appreciated.
Thanks.
I have figured out below method to achieve my requirement.
File streaming can be achieved using curl_formadd(for older version of libcurl)and curl_mime(for newer version > 7.58.0) APIs.
Using Mime APIs:
CURL *curl_handle;
curl_mime *mime;
curl_mimepart *part;
mime = curl_mime_init(curl_handle);
part = curl_mime_addpart(mime);
curl_mime_name(part, "sendfile");
curl_mime_filename(part, "uploadfile.txt");
curl_mime_data_cb(part, file_size, read_buffer, NULL, NULL, NULL);
curl_easy_setopt(curl_handle, CURLOPT_URL, url);
curl_easy_setopt(curl_handle, CURLOPT_MIMEPOST, mime);
curl_easy_perform(curl_handle);
The read callback function given in curl_mime_data_cb() reads from the buffer/queue, this buffer/queue data is getting filled by other application which is downloading the file using libcurl.
I hope it helps someone.
I recommend you to try it with xmlstream please check out the code below maybe it will help.
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <expat.h>
#include <curl/curl.h>
struct MemoryStruct {
char *memory;
size_t size;
};
struct ParserStruct {
int ok;
size_t tags;
size_t depth;
struct MemoryStruct characters;
};
static void startElement(void *userData, const XML_Char *name,
const XML_Char **atts)
{
struct ParserStruct *state = (struct ParserStruct *) userData;
state->tags++;
state->depth++;
/* Get a clean slate for reading in character data. */
free(state->characters.memory);
state->characters.memory = NULL;
state->characters.size = 0;
}
static void characterDataHandler(void *userData, const XML_Char *s, int len)
{
struct ParserStruct *state = (struct ParserStruct *) userData;
struct MemoryStruct *mem = &state->characters;
char *ptr = realloc(mem->memory, mem->size + len + 1);
if(!ptr) {
/* Out of memory. */
fprintf(stderr, "Not enough memory (realloc returned NULL).\n");
state->ok = 0;
return;
}
mem->memory = ptr;
memcpy(&(mem->memory[mem->size]), s, len);
mem->size += len;
mem->memory[mem->size] = 0;
}
static void endElement(void *userData, const XML_Char *name)
{
struct ParserStruct *state = (struct ParserStruct *) userData;
state->depth--;
printf("%5lu %10lu %s\n", state->depth, state->characters.size, name);
}
static size_t parseStreamCallback(void *contents, size_t length, size_t nmemb,
void *userp)
{
XML_Parser parser = (XML_Parser) userp;
size_t real_size = length * nmemb;
struct ParserStruct *state = (struct ParserStruct *) XML_GetUserData(parser);
/* Only parse if we're not already in a failure state. */
if(state->ok && XML_Parse(parser, contents, real_size, 0) == 0) {
int error_code = XML_GetErrorCode(parser);
fprintf(stderr, "Parsing response buffer of length %lu failed"
" with error code %d (%s).\n",
real_size, error_code, XML_ErrorString(error_code));
state->ok = 0;
}
return real_size;
}
int main(void)
{
CURL *curl_handle;
CURLcode res;
XML_Parser parser;
struct ParserStruct state;
/* Initialize the state structure for parsing. */
memset(&state, 0, sizeof(struct ParserStruct));
state.ok = 1;
/* Initialize a namespace-aware parser. */
parser = XML_ParserCreateNS(NULL, '\0');
XML_SetUserData(parser, &state);
XML_SetElementHandler(parser, startElement, endElement);
XML_SetCharacterDataHandler(parser, characterDataHandler);
/* Initialize a libcurl handle. */
curl_global_init(CURL_GLOBAL_DEFAULT);
curl_handle = curl_easy_init();
curl_easy_setopt(curl_handle, CURLOPT_URL,
"https://www.w3schools.com/xml/simple.xml");
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, parseStreamCallback);
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *)parser);
printf("Depth Characters Closing Tag\n");
/* Perform the request and any follow-up parsing. */
res = curl_easy_perform(curl_handle);
if(res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
}
else if(state.ok) {
/* Expat requires one final call to finalize parsing. */
if(XML_Parse(parser, NULL, 0, 1) == 0) {
int error_code = XML_GetErrorCode(parser);
fprintf(stderr, "Finalizing parsing failed with error code %d (%s).\n",
error_code, XML_ErrorString(error_code));
}
else {
printf(" --------------\n");
printf(" %lu tags total\n", state.tags);
}
}
/* Clean up. */
free(state.characters.memory);
XML_ParserFree(parser);
curl_easy_cleanup(curl_handle);
curl_global_cleanup();
return 0;
}
I am writing a c program using libcurl. I am connecting to a server running on localhost and then saving this retrieved data from server in a file. When I use the filename such as "file.txt", then I can write and read my file. But when I try to change the filename and append it with current data and time, I get an error "Failed Writing received data to disk/application".
Following is my code:
` int main(int argc, char *argv[])
{
float freq;
CURL *curl;
CURLcode res;
FILE *fp, *file_r;
char *buffer;
buffer = malloc (100000);
char url[1024], name[20];
char *filename;
filename = (char *) malloc (100);
if (filename == NULL)
{
printf("Error in memory allocation");
}`
/* Store the data retireved from webpage in a file and name the file with current date and time */
time_t now;
time (&now);
struct tm *t = localtime (&now);
strftime (name, 20, "%d %m %Y %H:%M:%S", t);
// printf ("%s\n", name);
filename[0] ='\0'; // Ensure the memory is an emtpy string;
strcat (filename, "file_new_");
strcat (filename, name);
strcat (filename, ".txt");
printf ("%s\n", filename);
printf("enter your input frequency value\n");
scanf("%f", &freq);
sprintf(url, "http://127.0.0.1:8080/?action=update_frequency&update_frequency=%f",freq);
/* In windows, this will init the winsock stuff */
curl_global_init(CURL_GLOBAL_ALL);
/* get a curl handle */
curl = curl_easy_init();
if(curl)
{
curl_easy_setopt(curl, CURLOPT_URL, "http://127.0.0.1:8080");
/* Now specify the username & pwd */
curl_easy_setopt(curl, CURLOPT_USERPWD, "admin:12345");
/* Specify the web request */
curl_easy_setopt(curl, CURLOPT_URL, url);
/* Storing the received data in a file */
fp = fopen (filename, "w+");
curl_easy_setopt (curl, CURLOPT_WRITEDATA, fp);
/* Perform the request, res will get the return code */
res = curl_easy_perform(curl);
/* Check for errors */
if (res != CURLE_OK)
{
fprintf(stderr, "curl_easy_perform() failed: %s\n",curl_easy_strerror(res));
}
fclose (fp);
/* To read the return msg from server after performing a request */
file_r= fopen (filename, "r");
fseek (file_r, SEEK_SET, 0);
while (fgets (buffer, 100000, file_r)!=NULL)
{
;
}
printf( "%s\n", buffer);
fclose (file_r);
}
/* always cleanup */
curl_easy_cleanup(curl);
curl_global_cleanup();
return 0;
}
If anyone has any idea, would be of great help.
Thanks much
You can't use the character : in a filename on Windows, so that's why it fails to save. Change the separator to something else.
Is fopen("tftp://1.1.1.1/file.txt","rb"); a valid statement? Can urls be opened using fopen in C programming?
No, but you can use libcurl, an example:
#include <stdio.h>
#include <curl/curl.h>
/*
* This is an example showing how to get a single file from an FTP server.
* It delays the actual destination file creation until the first write
* callback so that it won't create an empty file in case the remote file
* doesn't exist or something else fails.
*/
struct FtpFile {
const char *filename;
FILE *stream;
};
static size_t my_fwrite(void *buffer, size_t size, size_t nmemb, void *stream)
{
struct FtpFile *out=(struct FtpFile *)stream;
if(out && !out->stream) {
/* open file for writing */
out->stream=fopen(out->filename, "wb");
if(!out->stream)
return -1; /* failure, can't open file to write */
}
return fwrite(buffer, size, nmemb, out->stream);
}
int main(void)
{
CURL *curl;
CURLcode res;
struct FtpFile ftpfile={
"curl.tar.gz", /* name to store the file as if succesful */
NULL
};
curl_global_init(CURL_GLOBAL_DEFAULT);
curl = curl_easy_init();
if(curl) {
/*
* You better replace the URL with one that works!
*/
curl_easy_setopt(curl, CURLOPT_URL,
"ftp://ftp.example.com/pub/www/utilities/curl/curl-7.9.2.tar.gz");
/* Define our callback to get called when there's data to be written */
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, my_fwrite);
/* Set a pointer to our struct to pass to the callback */
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &ftpfile);
/* Switch on full protocol/debug output */
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
res = curl_easy_perform(curl);
/* always cleanup */
curl_easy_cleanup(curl);
if(CURLE_OK != res) {
/* we failed */
fprintf(stderr, "curl told us %d\n", res);
}
}
if(ftpfile.stream)
fclose(ftpfile.stream); /* close the local file */
curl_global_cleanup();
return 0;
}
Or (as pointed out by #Paul) you can pipe a process (E.g.: wget url) with popen:
#include <stdio.h>
FILE *popen(const char *command, const char *mode);
int pclose(FILE *stream);
int main(void)
{
/* wget -q = silent mode */
FILE *cmd = popen("wget -q -O - ftp://debian.org/debian-security/README.security", "r");
char result[1024];
while (fgets(result, sizeof(result), cmd) != NULL)
printf("%s", result);
pclose(cmd);
return 0;
}
The fopen in <stdio.h> doesn't do that.
However, nothing prevents someone from writing a function called fopen() that does something else.
FILE *popen(const char *command, const char *mode) can be used to spawn a process running an appropriate command line tool such as tftp or wget, and thereby accomplish this downloading of a remote resource into a file descriptor accessible from C code. The syntax for a popen() call is very similar to what you have shown. It is missing the program name for the download utility, though. A bare url or ftp address won't work for popen().
See:
fopen man page
popen man page
Also note:
The PHP language version of fopen() does open bare URLs. But PHP != C
It's not that easy as simply using fopen but it can be done.
You need to use libcurl. Take a look here.
From the site:
/*****************************************************************************
*
* This example requires libcurl 7.9.7 or later.
*/
#include <stdio.h>
#include <string.h>
#ifndef WIN32
#include <sys/time.h>
#endif
#include <stdlib.h>
#include <errno.h>
#include <curl/curl.h>
enum fcurl_type_e {
CFTYPE_NONE=0,
CFTYPE_FILE=1,
CFTYPE_CURL=2
};
struct fcurl_data
{
enum fcurl_type_e type; /* type of handle */
union {
CURL *curl;
FILE *file;
} handle; /* handle */
char *buffer; /* buffer to store cached data*/
size_t buffer_len; /* currently allocated buffers length */
size_t buffer_pos; /* end of data in buffer*/
int still_running; /* Is background url fetch still in progress */
};
typedef struct fcurl_data URL_FILE;
/* exported functions */
URL_FILE *url_fopen(const char *url,const char *operation);
int url_fclose(URL_FILE *file);
int url_feof(URL_FILE *file);
size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file);
char * url_fgets(char *ptr, size_t size, URL_FILE *file);
void url_rewind(URL_FILE *file);
/* we use a global one for convenience */
CURLM *multi_handle;
/* curl calls this routine to get more data */
static size_t write_callback(char *buffer,
size_t size,
size_t nitems,
void *userp)
{
char *newbuff;
size_t rembuff;
URL_FILE *url = (URL_FILE *)userp;
size *= nitems;
rembuff=url->buffer_len - url->buffer_pos; /* remaining space in buffer */
if(size > rembuff) {
/* not enough space in buffer */
newbuff=realloc(url->buffer,url->buffer_len + (size - rembuff));
if(newbuff==NULL) {
fprintf(stderr,"callback buffer grow failed\n");
size=rembuff;
}
else {
/* realloc suceeded increase buffer size*/
url->buffer_len+=size - rembuff;
url->buffer=newbuff;
}
}
memcpy(&url->buffer[url->buffer_pos], buffer, size);
url->buffer_pos += size;
return size;
}
/* use to attempt to fill the read buffer up to requested number of bytes */
static int fill_buffer(URL_FILE *file, size_t want)
{
fd_set fdread;
fd_set fdwrite;
fd_set fdexcep;
struct timeval timeout;
int rc;
/* only attempt to fill buffer if transactions still running and buffer
* doesnt exceed required size already
*/
if((!file->still_running) || (file->buffer_pos > want))
return 0;
/* attempt to fill buffer */
do {
int maxfd = -1;
long curl_timeo = -1;
FD_ZERO(&fdread);
FD_ZERO(&fdwrite);
FD_ZERO(&fdexcep);
/* set a suitable timeout to fail on */
timeout.tv_sec = 60; /* 1 minute */
timeout.tv_usec = 0;
curl_multi_timeout(multi_handle, &curl_timeo);
if(curl_timeo >= 0) {
timeout.tv_sec = curl_timeo / 1000;
if(timeout.tv_sec > 1)
timeout.tv_sec = 1;
else
timeout.tv_usec = (curl_timeo % 1000) * 1000;
}
/* get file descriptors from the transfers */
curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
/* In a real-world program you OF COURSE check the return code of the
function calls. On success, the value of maxfd is guaranteed to be
greater or equal than -1. We call select(maxfd + 1, ...), specially
in case of (maxfd == -1), we call select(0, ...), which is basically
equal to sleep. */
rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
switch(rc) {
case -1:
/* select error */
break;
case 0:
default:
/* timeout or readable/writable sockets */
curl_multi_perform(multi_handle, &file->still_running);
break;
}
} while(file->still_running && (file->buffer_pos < want));
return 1;
}
/* use to remove want bytes from the front of a files buffer */
static int use_buffer(URL_FILE *file,int want)
{
/* sort out buffer */
if((file->buffer_pos - want) <=0) {
/* ditch buffer - write will recreate */
if(file->buffer)
free(file->buffer);
file->buffer=NULL;
file->buffer_pos=0;
file->buffer_len=0;
}
else {
/* move rest down make it available for later */
memmove(file->buffer,
&file->buffer[want],
(file->buffer_pos - want));
file->buffer_pos -= want;
}
return 0;
}
URL_FILE *url_fopen(const char *url,const char *operation)
{
/* this code could check for URLs or types in the 'url' and
basicly use the real fopen() for standard files */
URL_FILE *file;
(void)operation;
file = malloc(sizeof(URL_FILE));
if(!file)
return NULL;
memset(file, 0, sizeof(URL_FILE));
if((file->handle.file=fopen(url,operation)))
file->type = CFTYPE_FILE; /* marked as URL */
else {
file->type = CFTYPE_CURL; /* marked as URL */
file->handle.curl = curl_easy_init();
curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file);
curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L);
curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
if(!multi_handle)
multi_handle = curl_multi_init();
curl_multi_add_handle(multi_handle, file->handle.curl);
/* lets start the fetch */
curl_multi_perform(multi_handle, &file->still_running);
if((file->buffer_pos == 0) && (!file->still_running)) {
/* if still_running is 0 now, we should return NULL */
/* make sure the easy handle is not in the multi handle anymore */
curl_multi_remove_handle(multi_handle, file->handle.curl);
/* cleanup */
curl_easy_cleanup(file->handle.curl);
free(file);
file = NULL;
}
}
return file;
}
int url_fclose(URL_FILE *file)
{
int ret=0;/* default is good return */
switch(file->type) {
case CFTYPE_FILE:
ret=fclose(file->handle.file); /* passthrough */
break;
case CFTYPE_CURL:
/* make sure the easy handle is not in the multi handle anymore */
curl_multi_remove_handle(multi_handle, file->handle.curl);
/* cleanup */
curl_easy_cleanup(file->handle.curl);
break;
default: /* unknown or supported type - oh dear */
ret=EOF;
errno=EBADF;
break;
}
if(file->buffer)
free(file->buffer);/* free any allocated buffer space */
free(file);
return ret;
}
int url_feof(URL_FILE *file)
{
int ret=0;
switch(file->type) {
case CFTYPE_FILE:
ret=feof(file->handle.file);
break;
case CFTYPE_CURL:
if((file->buffer_pos == 0) && (!file->still_running))
ret = 1;
break;
default: /* unknown or supported type - oh dear */
ret=-1;
errno=EBADF;
break;
}
return ret;
}
size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
{
size_t want;
switch(file->type) {
case CFTYPE_FILE:
want=fread(ptr,size,nmemb,file->handle.file);
break;
case CFTYPE_CURL:
want = nmemb * size;
fill_buffer(file,want);
/* check if theres data in the buffer - if not fill_buffer()
* either errored or EOF */
if(!file->buffer_pos)
return 0;
/* ensure only available data is considered */
if(file->buffer_pos < want)
want = file->buffer_pos;
/* xfer data to caller */
memcpy(ptr, file->buffer, want);
use_buffer(file,want);
want = want / size; /* number of items */
break;
default: /* unknown or supported type - oh dear */
want=0;
errno=EBADF;
break;
}
return want;
}
char *url_fgets(char *ptr, size_t size, URL_FILE *file)
{
size_t want = size - 1;/* always need to leave room for zero termination */
size_t loop;
switch(file->type) {
case CFTYPE_FILE:
ptr = fgets(ptr,size,file->handle.file);
break;
case CFTYPE_CURL:
fill_buffer(file,want);
/* check if theres data in the buffer - if not fill either errored or
* EOF */
if(!file->buffer_pos)
return NULL;
/* ensure only available data is considered */
if(file->buffer_pos < want)
want = file->buffer_pos;
/*buffer contains data */
/* look for newline or eof */
for(loop=0;loop < want;loop++) {
if(file->buffer[loop] == '\n') {
want=loop+1;/* include newline */
break;
}
}
/* xfer data to caller */
memcpy(ptr, file->buffer, want);
ptr[want]=0;/* allways null terminate */
use_buffer(file,want);
break;
default: /* unknown or supported type - oh dear */
ptr=NULL;
errno=EBADF;
break;
}
return ptr;/*success */
}
void url_rewind(URL_FILE *file)
{
switch(file->type) {
case CFTYPE_FILE:
rewind(file->handle.file); /* passthrough */
break;
case CFTYPE_CURL:
/* halt transaction */
curl_multi_remove_handle(multi_handle, file->handle.curl);
/* restart */
curl_multi_add_handle(multi_handle, file->handle.curl);
/* ditch buffer - write will recreate - resets stream pos*/
if(file->buffer)
free(file->buffer);
file->buffer=NULL;
file->buffer_pos=0;
file->buffer_len=0;
break;
default: /* unknown or supported type - oh dear */
break;
}
}
/* Small main program to retrive from a url using fgets and fread saving the
* output to two test files (note the fgets method will corrupt binary files if
* they contain 0 chars */
int main(int argc, char *argv[])
{
URL_FILE *handle;
FILE *outf;
int nread;
char buffer[256];
const char *url;
if(argc < 2)
url="http://192.168.7.3/testfile";/* default to testurl */
else
url=argv[1];/* use passed url */
/* copy from url line by line with fgets */
outf=fopen("fgets.test","w+");
if(!outf) {
perror("couldn't open fgets output file\n");
return 1;
}
handle = url_fopen(url, "r");
if(!handle) {
printf("couldn't url_fopen() %s\n", url);
fclose(outf);
return 2;
}
while(!url_feof(handle)) {
url_fgets(buffer,sizeof(buffer),handle);
fwrite(buffer,1,strlen(buffer),outf);
}
url_fclose(handle);
fclose(outf);
/* Copy from url with fread */
outf=fopen("fread.test","w+");
if(!outf) {
perror("couldn't open fread output file\n");
return 1;
}
handle = url_fopen("testfile", "r");
if(!handle) {
printf("couldn't url_fopen() testfile\n");
fclose(outf);
return 2;
}
do {
nread = url_fread(buffer, 1,sizeof(buffer), handle);
fwrite(buffer,1,nread,outf);
} while(nread);
url_fclose(handle);
fclose(outf);
/* Test rewind */
outf=fopen("rewind.test","w+");
if(!outf) {
perror("couldn't open fread output file\n");
return 1;
}
handle = url_fopen("testfile", "r");
if(!handle) {
printf("couldn't url_fopen() testfile\n");
fclose(outf);
return 2;
}
nread = url_fread(buffer, 1,sizeof(buffer), handle);
fwrite(buffer,1,nread,outf);
url_rewind(handle);
buffer[0]='\n';
fwrite(buffer,1,1,outf);
nread = url_fread(buffer, 1,sizeof(buffer), handle);
fwrite(buffer,1,nread,outf);
url_fclose(handle);
fclose(outf);
return 0;/* all done */
}
Question 1: When url is downloaded using libcurl, how to preserve the original name of downloaded file ? LibCurl asks programmer to generate filename. Which is maybe easy when URL has the
e.g. in below url its easy to figure out target name is vimqrc.pdf.
http://tnerual.eriogerg.free.fr/vimqrc.pdf)
but when URL is dynamically generating target name e.g.below URL downloads AdbeRdr1010_eu_ES.exe. with wget (no arguments except URL) and curl (argument -O)
http://get.adobe.com/reader/download/?installer=Reader_10.1_Basque_for_Windows&standalone=1%22
How does curl (-O) or wget figures out name of
//invoked as ./a.out <URL>
#include <stdio.h>
#include <curl/curl.h>
char *location = "/tmp/test/out";
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
size_t written = fwrite(ptr, size, nmemb, stream);
return written;
}
int main(int argc, char *argv[])
{
CURL *curl;
CURLcode res;
int ret = -1;
if (argc!= 2) {
//invoked as ./a.out <URL>
return -1;
}
curl = curl_easy_init();
if (!curl) {
goto bail;
}
FILE *fp = fopen(location, "wb");
curl_easy_setopt(curl, CURLOPT_URL, argv[1]); //invoked as ./a.out <URL>
/* example.com is redirected, so we tell libcurl to follow redirection */
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
/* Perform the request, res will get the return code */
res = curl_easy_perform(curl);
/* Check for errors */
if(res != CURLE_OK)
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
/* always cleanup */
curl_easy_cleanup(curl);
ret = 0;
fclose(fp);
bail:
return ret;
}
I found the answer in libcurl source code. Looks like "remote name" is part of the "content-disposition" tag from the header. Libcurl is parsing header and looking for "filename=" in the content-disposition tag. This parsing is done in callback provided through CURLOPT_HEADERFUNCTION option. Finally, in a callback for writing data (provided through CURLOPT_WRITEFUNCTION) this remote name is used to create output file.
If file name is missing, its simply figuring it out from URL itself. This is pretty much code copied from lib curl and little modifications of my own to make it simpler and match my requirement.
#define _GNU_SOURCE
#include <stdio.h>
#include <curl/curl.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <stdlib.h>
typedef unsigned long uint64_t;
typedef struct {
char dnld_remote_fname[4096];
char dnld_url[4096];
FILE *dnld_stream;
FILE *dbg_stream;
uint64_t dnld_file_sz;
} dnld_params_t;
static int get_oname_from_cd(char const*const cd, char *oname)
{
char const*const cdtag = "Content-disposition:";
char const*const key = "filename=";
int ret = 0;
char *val = NULL;
/* Example Content-Disposition: filename=name1367; charset=funny; option=strange */
/* If filename is present */
val = strcasestr(cd, key);
if (!val) {
printf("No key-value for \"%s\" in \"%s\"", key, cdtag);
goto bail;
}
/* Move to value */
val += strlen(key);
/* Copy value as oname */
while (*val != '\0' && *val != ';') {
//printf (".... %c\n", *val);
*oname++ = *val++;
}
*oname = '\0';
bail:
return ret;
}
static int get_oname_from_url(char const* url, char *oname)
{
int ret = 0;
char const *u = url;
/* Remove "http(s)://" */
u = strstr(u, "://");
if (u) {
u += strlen("://");
}
u = strrchr(u, '/');
/* Remove last '/' */
u++;
/* Copy value as oname */
while (*u != '\0') {
//printf (".... %c\n", *u);
*oname++ = *u++;
}
*oname = '\0';
return ret;
}
size_t dnld_header_parse(void *hdr, size_t size, size_t nmemb, void *userdata)
{
const size_t cb = size * nmemb;
const char *hdr_str= hdr;
dnld_params_t *dnld_params = (dnld_params_t*)userdata;
char const*const cdtag = "Content-disposition:";
/* Example:
* ...
* Content-Type: text/html
* Content-Disposition: filename=name1367; charset=funny; option=strange
*/
if (strstr(hdr_str, "Content-disposition:")) {
printf ("has c-d: %s\n", hdr_str);
}
if (!strncasecmp(hdr_str, cdtag, strlen(cdtag))) {
printf ("Found c-d: %s\n", hdr_str);
int ret = get_oname_from_cd(hdr_str+strlen(cdtag), dnld_params->dnld_remote_fname);
if (ret) {
printf("ERR: bad remote name");
}
}
return cb;
}
FILE* get_dnld_stream(char const*const fname)
{
char const*const pre = "/tmp/";
char out[4096];
snprintf(out, sizeof(out), "%s/%s", pre, fname);
FILE *fp = fopen(out, "wb");
if (!fp) {
printf ("Could not create file %s\n", out);
}
return fp;
}
size_t write_cb(void *buffer, size_t sz, size_t nmemb, void *userdata)
{
int ret = 0;
dnld_params_t *dnld_params = (dnld_params_t*)userdata;
if (!dnld_params->dnld_remote_fname[0]) {
ret = get_oname_from_url(dnld_params->dnld_url, dnld_params->dnld_remote_fname);
}
if (!dnld_params->dnld_stream) {
dnld_params->dnld_stream = get_dnld_stream(dnld_params->dnld_remote_fname);
}
ret = fwrite(buffer, sz, nmemb, dnld_params->dnld_stream);
if (ret == (sz*nmemb)) {
dnld_params->dnld_file_sz += ret;
}
return ret;
}
int download_url(char const*const url)
{
CURL *curl;
int ret = -1;
CURLcode cerr = CURLE_OK;
dnld_params_t dnld_params;
memset(&dnld_params, 0, sizeof(dnld_params));
strncpy(dnld_params.dnld_url, url, strlen(url));
curl = curl_easy_init();
if (!curl) {
goto bail;
}
cerr = curl_easy_setopt(curl, CURLOPT_URL, url);
if (cerr) { printf ("%s: failed with err %d\n", "URL", cerr); goto bail;}
cerr = curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, dnld_header_parse);
if (cerr) { printf ("%s: failed with err %d\n", "HEADER", cerr); goto bail;}
cerr = curl_easy_setopt(curl, CURLOPT_HEADERDATA, &dnld_params);
if (cerr) { printf ("%s: failed with err %d\n", "HEADER DATA", cerr); goto bail;}
cerr = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
if (cerr) { printf ("%s: failed with err %d\n", "WR CB", cerr); goto bail;}
cerr = curl_easy_setopt(curl, CURLOPT_WRITEDATA, &dnld_params);
if (cerr) { printf ("%s: failed with err %d\n", "WR Data", cerr); goto bail;}
cerr = curl_easy_perform(curl);
if(cerr != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(cerr));
}
printf ("Remote name: %s\n", dnld_params.dnld_remote_fname);
fclose(dnld_params.dnld_stream);
/* always cleanup */
curl_easy_cleanup(curl);
ret = 0;
printf ("file size : %lu\n", dnld_params.dnld_file_sz);
bail:
return ret;
}
int main(int argc, char *argv[])
{
if (argc != 2) {
printf ("Bad args\n");
return -1;
}
return download_url(argv[1]);
}
It's your program, not libcurl that is determining the filename. In your example, you could simple change char *location = "/tmp/test/out"; to char *location = "/tmp/test/vimqrc.pdf"; to get your desired effect.
If you want to derive the download file path programatically given a url and parent directory, you could do something like he following :
int url_to_location(char* location, unsigned int location_length, const char* url, const char* parent_directory)
{
//char location[MAX_PATH];
//const char *url = "http://tnerual.eriogerg.free.fr/vimqrc.pdf";
//const char *parent_directory = "/tmp/test/";
int last_slash_index = -1;
int current_index = (int)strlen(url);
while (current_index >= 0)
{
if (url[current_index] == '/')
{
last_slash_index = current_index;
break;
}
current_index--;
}
unsigned int parent_directory_length = strlen(parent_directory)
if (parent_directory_length <= location_length)
return -1;
strcpy(location, parent_directory);
if (last_slash_index == -1) //no slashes found, use relative url as filename
{
if (parent_directory_length + strlen(url) <= location_length)
return -1;
strcat(location, url);
}
else //use the characters of the url following the last slash as filename
{
if (parent_directory_length + strlen(url + last_slash_index + 1) <= location_length)
return -1;
strcat(location, url + last_slash_index + 1);
}
return strlen(location);
}
I have to trasfer files using FTP protocol and libcurl in c. It works fine, but I have some problems.
1) If a transfer is started, but at a certain point I disconnect from the network, my program remains in the libcurl function, the speed goes to 0 and I can't do anything else. I tried to set timeout (CURLOPT_TIMEOUT), but it's just a timeout on the transfer time.
2) The second problem I have, which is linked to the first one is, how can I know if the trasfer if successfully completed?
My trasfer code is:
struct FtpFile {
const char *filename;
FILE *stream;
};
long int size;
static size_t my_fwrite(void *buffer, size_t size, size_t nmemb, void *stream)
{
struct FtpFile *out=(struct FtpFile *)stream;
if(out && !out->stream) {
/* open file for writing */
out->stream=fopen(out->filename, "ab");
if(!out->stream)
return -1; /* failure, can't open file to write */
}
return fwrite(buffer, size, nmemb, out->stream);
}
int sz;
int main(void)
{
CURL *curl;
CURLcode res;
char prova;
struct stat statbuf;
FILE *stream;
/* apro il file per leggere la dimensione*/
if ((stream = fopen("Pdf.pdf", "rb"))
== NULL)
{
fprintf(stderr, "Nessun file da aprire, il download partirĂ da zero.\n");
}
else
{
/* Ricevo informazioni sul file */
fstat(fileno(stream), &statbuf);
fclose(stream);
size = statbuf.st_size;
printf("Dimensione del file in byte: %ld\n", size);
}
struct FtpFile ftpfile={
"Pdf.pdf", /* name to store the file as if succesful */
NULL
};
curl_global_init(CURL_GLOBAL_DEFAULT);
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "ftp://....");
/* Define our callback to get called when there's data to be written */
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, my_fwrite);
/* Set a pointer to our struct to pass to the callback */
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &ftpfile);
/* Switch on full protocol/debug output */
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
/*Pass a long as parameter. It contains the offset in number of bytes that you want the transfer to start from.
Set this option to 0 to make the transfer start from the beginning (effectively disabling resume). For FTP, set
this option to -1 to make the transfer start from the end of the target file (useful to continue an interrupted
upload).
When doing uploads with FTP, the resume position is where in the local/source file libcurl should try to resume
the upload from and it will then append the source file to the remote target file. */
if(stream == NULL)
{
curl_easy_setopt(curl, CURLOPT_RESUME_FROM, 0);
}
else
{
curl_easy_setopt(curl, CURLOPT_RESUME_FROM, size);
}
/*Used to show file progress*/
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
res = curl_easy_perform(curl);
/* always cleanup */
curl_easy_cleanup(curl);
if(CURLE_OK != res) {
/* we failed */
fprintf(stderr, "curl told us %d\n", res);
}
}
if(ftpfile.stream)
fclose(ftpfile.stream); /* close the local file */
curl_global_cleanup();
return 0;
}
I have recently answered a similar question to this.
1) this is by design. if you want to timeout the connection, try using these instead:
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, dl_lowspeed_bytes);
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, dl_lowspeed_time);
If your download rate falls below your desired threshold, you can check the connectivity & take whatever action you see fit.
NB: Added in 7.25.0: CURLOPT_TCP_KEEPALIVE, CURLOPT_TCP_KEEPIDLE
so these might be another suitable alternative for you.
2) like this:
curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &dl_bytes_remaining);
curl_easy_getinfo(curl, CURLINFO_SIZE_DOWNLOAD, &dl_bytes_received);
if (dl_bytes_remaining == dl_bytes_received)
printf("our work here is done ;)\n");