I am downloading file quite commonly with curl. However, the server does a tricky thing: it return non-200 code and still sends some data. The problem is that I have the HTTP code after the data are written, but I do not want to write anything if it is non-200. Does anyone know a way to do that without storing data on disk or memory?
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlWriteHandler);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, ptr);
res = curl_easy_perform(curl);
if (res == CURLE_OK) {
return 0;
}
long response_code;
curl_easy_getinfo(curl_.get(), CURLINFO_RESPONSE_CODE, &response_code);
if (response_code != 200) {
return 0;
}
size_t curlWriteHandler(char* chars, size_t size, size_t nmemb, void* userp) {
// write to file
return size * nmemb;
}
Setting CURLOPT_FAILONERROR should do it for 4xx and 5xx errors.
When this option is used and an error is detected, it will cause the connection to get closed and CURLE_HTTP_RETURNED_ERROR is returned.
curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L);
Closing connection is not good for me, it is important to reuse one. Can you think about anything else?
Unfortunately I can't find a way to make CURLOPT_FAILONERROR not close the connection.
The other option is to make the write function aware of the response. Unfortunately the curl handle is not passed into the callback.
We could make the curl variable global. Or we can take advantage of the void *userdata option to the write callback and pass in a struct containing both the curl handle and the buffer.
Here's a rough sketch demonstrating how the write callback can get access to the response code and also save the response body.
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <curl/curl.h>
typedef struct {
CURL *curl;
char *buf;
} curl_write_data;
size_t curlWriteHandler(char* chars, size_t size, size_t nmemb, void* userp) {
curl_write_data *curl_data = (curl_write_data*)userp;
long response_code;
curl_easy_getinfo(curl_data->curl, CURLINFO_RESPONSE_CODE, &response_code);
printf("Response: %ld\n", response_code);
// Now we can save if we like.
if( response_code < 300 ) {
curl_data->buf = malloc(size*(nmemb+1));
strcpy(curl_data->buf, chars);
strcat(curl_data->buf, "\0");
return size * nmemb;
}
else {
return 0;
}
}
int main() {
CURL *curl = curl_easy_init();
if(!curl) {
perror("Cant' init curl");
}
curl_write_data curl_data = { .curl = curl, .buf = NULL };
curl_easy_setopt(curl, CURLOPT_URL, "http://example.com/alsdfjalj");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlWriteHandler);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &curl_data);
curl_easy_perform(curl);
if( curl_data.buf ) {
puts(curl_data.buf);
}
}
I'm not sure if this is the best idea, its what I came up with.
Related
I want to play mp3 files on the internet without downloading them. So, I use libcurl to get it as a stream in memory, like this:
static size_t use_data(void *ptr, size_t size, size_t nmemb, FILE *stream) {
/* stream is NULL */
/* What to do with the stream of data ? */
}
CURLcode download_file(const char *url, const char *path, curl_progress_callback progress) {
CURL *curl;
CURLcode res = 0;
FILE *fp;
if ((curl = curl_easy_init())) {
if (progress) {
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progress);
}
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, use_data);
res = curl_easy_perform(curl);
/* always cleanup */
curl_easy_cleanup(curl);
fclose(fp);
}
return res;
}
How can I parse the stream in memory to play sounds ?
The easiest way for you IMHO would be using lightweight MP3 decoding library. For example, minimp3 does it's job and consists of only 2 files.
http://keyj.emphy.de/minimp3
The API is very simple and a usage example can be found here: https://github.com/corporateshark/PortAMP/tree/master/src/Decoders/MP3
I have the following curl code, which make a request to website and retrieve data from it, it works well, but I want to store my data in a string and not in the output window. Any idea?
#include <stdio.h>
#include <curl/curl.h>
int main(void)
{
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "http://api.hostip.info/get_html.php?ip=xxx.xxx.xx.xxx");
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
res = curl_easy_perform(curl);
if(res != CURLE_OK)
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
curl_easy_cleanup(curl);
}
return 0;
}
int http_get_response(void *buffer, size_t size, size_t rxed, char **msg_in)
{
char *c;
if (asprintf(&c, "%s%.*s", *msg_in, size * rxed, buffer) == -1) {
free(*msg_in);
msg_in = NULL;
return -1;
}
free(*msg_in);
*msg_in = c;
return size * rxed;
}
and add the following curl option in your main
char *msg_in = calloc(1,sizeof(char));
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, http_get_response);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &msg_in);
Then you will get the message in the msg_in
EDIT
do not forget to free msg_in when it become uselless in your program
free(msg_in); msg_in = NULL;
A a general (non-curl specific) method, change your standard output (path 1) (or standard error: path 2) path(s) prior to calling curl. Read the man page on dup2 to see how to duplicate a path to a specific descriptor, and the fdopen function to get a FILE * out of it.
The idea is you first dup path 1 for stdout, and/or 2 for stderr, to save copies of them somewhere. You then close the original paths. You create a pipe (man pipe) and then dup2 the second channel of the pipe to path 1 (or 2). You can now read() from the first channel of the pipe to get the output that was placed there.
Try this:
curl_easy_setopt(curl, CURLOPT_RETURNTRANSFER, true);
Here I implemented code for file download from server. its working fine.
Now I want to make my own progress bar function which calculates some data like remaining seconds data Rate per second etc.
So from here I found one way to use curl progress bar option. how we can enable this option.
I completely done with this.
I put my code below. here in this code my_progress_func calls frequently as per curl library time interval. I want to change this interval time and make it to 1 second. is it possible in curl library using to set some options for curl library?
I want to call this my_progress_func function after every 1 second.
Code :
#include <stdio.h>
#include <curl/curl.h>
long test =0;
struct FtpFile {
const char *filename;
FILE *stream;
long iAppend;
};
static size_t my_fwrite(void *buffer, size_t size, size_t nmemb, void *stream)
{
struct FtpFile *out=(struct FtpFile *)stream;
if(out && !out->stream) {
/* open file for writing */
out->stream=fopen(out->filename, out->iAppend ? "ab":"wb");
if(!out->stream)
return -1; /* failure, can't open file to write */
}
out->iAppend += nmemb;
return fwrite(buffer, size, nmemb, out->stream);
}
int my_progress_func(void *bar,
double t, /* dltotal */
double d, /* dlnow */
double ultotal,
double ulnow)
{
printf("%f : %f \n", d, t);
return 0;
}
int main(void)
{
CURL *curl;
CURLcode res;
int c;
struct FtpFile ftpfile={
"dev.zip", /* name to store the file as if succesful */
NULL,
};
curl_global_init(CURL_GLOBAL_DEFAULT);
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL,
"sftp://root:xyz_#192.170.10.1/mnt/xyz.tar");
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 120L);
/* Define our callback to get called when there's data to be written */
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, my_fwrite);
/* Set a pointer to our struct to pass to the callback */
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &ftpfile);
curl_easy_setopt(curl, CURLOPT_FTPPORT, "-");
/* Switch on full protocol/debug output */
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, my_progress_func);
res = curl_easy_perform(curl);
printf("res is %d\n, data get %ld\n", res, ftpfile.iAppend);
///Retry upto 100 times it timeout or connection drop occur
for (c = 0; (res != CURLE_OK) && (c < 100); c++) {
curl_easy_setopt(curl, CURLOPT_RESUME_FROM , ftpfile.iAppend);
res = curl_easy_perform(curl);
if(res == CURLE_OK) c =0;
printf("%d res is %d\n, data get %ld\n",c, res, ftpfile.iAppend);
}
/* always cleanup */
curl_easy_cleanup(curl);
}
if(ftpfile.stream)
fclose(ftpfile.stream); /* close the local file */
curl_global_cleanup();
return 0;
}
According to the curl documentation:
http://curl.haxx.se/libcurl/c/curl_easy_setopt.html
Function pointer that should match the curl_progress_callback
prototype found in . This function gets called by libcurl
instead of its internal equivalent with a frequent interval during
operation (roughly once per second or sooner) no matter if data is
being transfered or not. Unknown/unused argument values passed to the
callback will be set to zero (like if you only download data, the
upload size will remain 0). Returning a non-zero value from this
callback will cause libcurl to abort the transfer and return
CURLE_ABORTED_BY_CALLBACK.
If it's calling too frequently then you can use time() and a static var to limit this, something like this:
static time_t prevtime;
time_t currtime;
double dif;
static int first = 1;
if(first) {
time(&prevtime);
first = 0;
}
time(&currtime);
dif = difftime(currtime, prevtime);
if(dif < 1.0)
return;
prevtime = currtime;
Obviously, you run the risk that curl might not call this function again for fully another second.
I am using the following code to download files from the internet:
size_t write_data(void *ptr, size_t size, size_t nmemb, FILE *stream)
{
size_t written;
written = fwrite(ptr, size, nmemb, stream);
return written;
}
int main(int argc, char** argv)
{
FILE *downloaded_file;
if ( (downloaded_file = fopen (download_path , "w" ) ) != NULL )
{
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl)
{
curl_easy_setopt(curl, CURLOPT_URL, "www.asd.com/files/file_to_download.rar");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, downloaded_file);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
if (res == CURLE_OK)
{
printf("Download complete!\n");
}
}
fclose(downloaded_file);
}
}
How can I measure the current download speed (e.g. every second) and the remaining time to complete the download?
You can use CURLOPT_PROGRESSFUNCTION. curl will pass 5 arguments to your callback function, clientp, dltotal, dlnow, ultotal, and ulnow. clientp is a pointer you provide with CURLOPT_PROGRESSDATA. The total parameters are the total amounts that need to be downloaded; the now ones are the amounts so far. Unknown values are 0.
To use this, you must set CURLOPT_NOPROGRESS to 0.
CURLOPT_PROGRESSFUNCTION has been deprecated since v7.32.0, instead you can use CURLOPT_XFERINFOFUNCTION, the usage and callback structure (all the arguments dltotal, dlnow ...etc) are almost the same as CURLOPT_PROGRESSFUNCTION
I'm currently using this C code:
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL, "http://my-domain.org/");
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
}
It prints the output on the console. How can I get the same output, but read it into, say, a string? (This is a probably a basic question, but I do not yet understand the libcurl API...)
Thanks for any help!
Mike
You need to pass a function and buffer to write it to buffer.
/* setting a callback function to return the data */
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_callback_func);
/* passing the pointer to the response as the callback parameter */
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, &response);
/* the function to invoke as the data recieved */
size_t static write_callback_func(void *buffer,
size_t size,
size_t nmemb,
void *userp)
{
char **response_ptr = (char**)userp;
/* assuming the response is a string */
*response_ptr = strndup(buffer, (size_t)(size *nmemb));
}
Please take a look more info here.
you need a write callback function. I use this kind of function to read the response, error and be able to supply my own headers:
size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream)
{
std::string buf = std::string(static_cast<char *>(ptr), size * nmemb);
std::stringstream *response = static_cast<std::stringstream *>(stream);
response->write(buf.c_str(), (std::streamsize)buf.size());
return size * nmemb;
}
bool CurlGet(
const std::string &url,
const std::vector<std::string> &headers,
std::stringstream &response,
std::string &error)
{
curl_global_init(CURL_GLOBAL_ALL);
curl_slist *headerlist = NULL;
std::vector<std::string>::const_iterator it;
for (it = headers.begin(); it < headers.end(); it++) {
headerlist = curl_slist_append(headerlist, it->c_str());
}
CURL *curl = curl_easy_init();
char ebuf[CURL_ERROR_SIZE];
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, ebuf);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headerlist);
CURLcode res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
curl_slist_free_all(headerlist);
if (res != CURLE_OK)
error = ebuf;
else
error.clear();
return res == CURLE_OK;
}
This can be done using
curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, write_data);
which sets a callback function write_data which is a function with the signature
size_t write_data(void *buffer, size_t size, size_t nmemb, void *userp);
If you want userp be some internal struct you are using in your program, call
curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, &internal_struct);
to get the pointer to internal_struct passed to every call of write_data.
Hi i solve issue of return code 23 from call back function to return size from call back function.
see below code:
/* setting a callback function to return the data */
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_callback_func);
/* passing the pointer to the response as the callback parameter */
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, &response);
/* the function to invoke as the data recieved */
size_t static write_callback_func(void *buffer,
size_t size,
size_t nmemb,
void *userp)
{
char **response_ptr = (char**)userp;
/* assuming the response is a string */
*response_ptr = strndup(buffer, (size_t)(size *nmemb));
return ((size_t)(size *nmemb));
//if you not send return value of size it will show you ERROR CODE 23return curl_easy_perform();
}
None of the other examples worked for me.
Here's what I eventually ended up doing:
size_t static curl_write(void *buffer, size_t size, size_t nmemb, void *userp)
{
userp += strlen(userp); // Skipping to first unpopulated char
memcpy(userp, buffer, nmemb); // Populating it.
return nmemb;
}
int GetCurl()
{
CURL *curl;
CURLcode res;
char *s = (char *) malloc(512);
curl = curl_easy_init();
if (curl)
{
curl_easy_setopt(curl, CURLOPT_URL, "http://www.google.com");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, s);
res = curl_easy_perform(curl);
curl_easy_cleanup(curl);
}
printf("GREAT SUCCESS!! Your string is %s\n", s);
}