I'm using Curl library to create a simple C code with MSVC to download a file from a URL.
The problem is if the connection breaks in the middle of download my code will freeze and the unfinished file hasn't removed from the directory.
What I want is if the download failed the program must retry the connection or remove the unfinished file and then try again. I prefer to use C libraries rather than C++ libs. Here is the code I am using:
//lib for curl
#include <curl/curl.h>
#define CURL_STATICLIB
bool downloader3(string url, string file_path) {
CURL *curl;
FILE *fp;
CURLcode res;
curl = curl_easy_init();
if (curl) {
fp = fopen(file_path.c_str(), "wb");
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
//always cleanup
curl_easy_cleanup(curl);
fclose(fp);
double val;
res = curl_easy_getinfo(curl, CURLINFO_SPEED_DOWNLOAD, &val);
if ((CURLE_OK == res) && (val>0))
printf("Average download speed: %0.3f kbyte/sec.\n", val / 1024);
if ((res == CURLE_OK)) {
printf("Download Successful!\r\n");
return true;
}
else {
printf("Downlaod Failed!\r\n");
remove(file_path.c_str()); //remove the temp file
return false;
}
}
}
EDIT---
Thanks to Ring Ø answer. I modifed the code but I am looking for a resume capability that can resume the download of incomplete file.
bool downloader3(string url, string file_path) {
CURL *curl;
FILE *fp = NULL;
CURLcode res;
int status;
int maxtries = 3;
do {
printf("Doing try # %d\r\n", maxtries);
curl = curl_easy_init();
if (curl) {
fp = fopen(file_path.c_str(), "wb");
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L); // 30 seconds
res = curl_easy_perform(curl);
//always cleanup
curl_easy_cleanup(curl);
fclose(fp);
if ((res == CURLE_OK)) {
printf("Download Successful!\r\n");
break;
//return true;
}
}
} while (--maxtries);
if (maxtries) { // was OK
//curl_easy_cleanup(curl); // clean curl / delete file?
//fclose(fp);
return true;
}
else {
printf("Download Failed!\r\n");
printf("file path is: %s", file_path.c_str());
Sleep(5000);
status = remove(file_path.c_str()); //remove the unfinished file
if (status == 0)
printf("%s file deleted successfully.\n", file_path);
else
{
printf("Unable to delete the file\n");
}
return false;
}
}
You could set a timeout option
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 30L); // 30 seconds
if the operation is not done within 30 seconds, the timeout is triggered. Then check the result value, in a while loop for instance
res = curl_easy_perform( ... );
if (res == CURLE_OK) {
break;
}
// delete file
// keep retrying (add a counter if necessary)
See also the curl page.
Loop example
int maxtries = 5;
do {
curl = curl_easy_init();
if (curl) {
...
res = curl_easy_perform( ... );
if (res == CURLE_OK) {
break;
}
// delete file, curl cleanup...
}
} while ( --maxtries );
if (maxtries) { // was OK
// clean curl / delete file?
}
This is not the ideal solution, as you said, the download may take more or less time. This (should) prevent a never ending program, provided the timeout is big enough.
Curl library was known to have some problems in case of erratic connection - there could be something better nowadays, please try the latest stable build.
If you don't get a better answer within a few days, try to add a "Bounty" of 50 rep to attract more attention.
What you are looking for is the RESUME_FROM feature. To use this you must know which byte you want to start the download from. In this example it is an upload but should be same setopt technique. Here is example usage from curl website:
CURL *curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "ftp://example.com");
/* resume upload at byte index 200 */
curl_easy_setopt(curl, CURLOPT_RESUME_FROM, 200L);
/* ask for upload */
curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
/* set total data amount to expect */
curl_easy_setopt(curl, CURLOPT_INFILESIZE, size_of_file);
/* Perform the request */
curl_easy_perform(curl);
}
source: https://curl.haxx.se/libcurl/c/CURLOPT_RESUME_FROM.html
Related
I am trying to write curl c code to using the http webdav put method to upload a file.
Using wireshark I have tried to capture the packets, there is 301 response from the server.
When I try to put the file from the PC to webserver it works fine
Below is the code:
static size_t read_callback(void *ptr, size_t size, size_t nmemb, void *stream)
{
size_t retcode;
curl_off_t nread;
/* in real-world cases, this would probably get this data differently
as this fread() stuff is exactly what the library already would do
by default internally */
retcode = fread(ptr, size, nmemb, stream);
nread = (curl_off_t)retcode;
fprintf(stderr, "*** We read %" CURL_FORMAT_CURL_OFF_T
" bytes from file\n", nread);
return retcode;
}
int curlApache ()
{
CURL *curl;
CURLcode res;
FILE * hd_src;
struct stat file_info;
char *file;
char *url;
char error;
file = "/bd0/filecreate.txt";
url = "http://10.1.21.14/webdav/test.txt";
curl_slist *slist = NULL;
slist = curl_slist_append(slist, "Accept: text/xml");
slist = curl_slist_append(slist, "Depth: infinity");
slist = curl_slist_append(slist, "Connection: Keep-Alive");
slist = curl_slist_append(slist, "Content-Type: text/xml");
slist = curl_slist_append(slist, "Expect:");
/* get the file size of the local file */
stat(file, &file_info);
hd_src = fopen(file, "a+");
if (hd_src == NULL)
printf("Disc full or no permission\n");
const char *str = "This is the file content";
const char read[24];
if (hd_src != NULL)
if (fputs (str, hd_src) != EOF);
if( fgets (read, 24, hd_src)!=NULL )
{
/* writing content to stdout */
puts(read);
}
/* In windows, this will init the winsock stuff */
curl_global_init(CURL_GLOBAL_ALL);
/* get a curl handle */
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_VERBOSE, 3L);
/* we want to use our own read function */
curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_callback);
/* enable uploading */
curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
/* HTTP PUT please */
curl_easy_setopt(curl, CURLOPT_PUT, 1L);
/* tell libcurl we can use "any" auth, which lets the lib pick one, but it also costs one extra round-trip and possibly sending of all the PUT data twice!!! */
curl_easy_setopt(curl, CURLOPT_HTTPAUTH, (long)CURLAUTH_DIGEST);
curl_easy_setopt(curl, CURLOPT_USERPWD, "admin:nimo0630");
fseek(hd_src, 0L, SEEK_END);
int file_size;
file_size = ftell(hd_src);
Curl_setopt(curl, CURLOPT_INFILE, hd_src);
Curl_setopt(curl, CURLOPT_INFILESIZE, file_size);
/* specify target URL, and note that this URL should include a file
name, not only a directory */
curl_easy_setopt(curl, CURLOPT_URL, url);
/* now specify which file to upload */
curl_easy_setopt(curl, CURLOPT_READDATA, hd_src);
/* provide the size of the upload, we specicially typecast the value
to curl_off_t since we must be sure to use the correct data size */
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, curl_off_t)file_info.st_size);
/* Now run off and do what you've been told! */
res = curl_easy_perform(curl);
/* Check for errors */
if(res != CURLE_OK)
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
if(!res) {
/* extract the available authentication types */
long auth;
res = curl_easy_getinfo(curl, CURLINFO_HTTPAUTH_AVAIL, &auth);
if(!res)
{
if(!auth)
printf("No auth available, perhaps no 401?\n");
else
{
printf("%s%s%s%s\n", \
auth & CURLAUTH_BASIC ? "Basic ":"", \
auth & CURLAUTH_DIGEST ? "Digest ":"", \
auth & CURLAUTH_NEGOTIATE ? "Negotiate ":"", \
auth % CURLAUTH_NTLM ? "NTLM ":"");
}
}
}
/* always cleanup */
curl_easy_cleanup(curl);
}
fclose(hd_src); /* close the local file */
curl_global_cleanup();
return 0;
}
There is status code 301 returned from the server
I believe you want to understand how to debug this issue. If you paste your code, a more specific answer can be provided.
First check if you are able to upload file using command line curl. This will tell you if server is working fine
Check if the options you use are provided in C API
Check the API's are not returning any error
You could use tcpdump / wireshark to capture packets on the client or server to see if packets went out and what was the http content. You may not see any packet if API failed
I have to check the network upload speed as well as upload a file in azure blob storage.
Initially I have tried with some sample HTTP servers to check the network speed and it's working. But when I am trying to send the file to azure blob storage HTTP endpoint, I am not sure how to use the SAS token for the authentication or the username and password will be enough to do the authentication and proceed with further procedure.
I have attached the code I have written , Please give some examples or reliable link for the authentication with SAS token and file upload.
size_t uploadspeed()
{
CURL *curl;
CURLcode res;
FILE *hd_src;
struct stat file_info;
curl_off_t fsize;
size_t speed_upload, total_time1;
hd_src = fopen("bbb.txt", "rb");
curl = curl_easy_init();
if(curl) {
int old_stdout = dup(2);
freopen("/dev/null" , "w+" , stderr);
FILE *f = fopen("target.txt", "wb");
curl_easy_setopt(curl, CURLOPT_WRITEDATA, f);
curl_easy_setopt(curl, CURLOPT_URL,
"AZURE_URL");
curl_easy_setopt(curl, CURLOPT_USERPWD, "A:B");
curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(curl, CURLOPT_READDATA, hd_src);
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE,(curl_off_t)file_info.st_size);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
res = curl_easy_perform(curl);
fclose(stderr);
if(res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
}
else {
curl_easy_getinfo(curl, CURLINFO_SPEED_UPLOAD, &speed_upload);
curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &total_time1);
}
curl_easy_cleanup(curl);
fclose(hd_src);
curl_global_cleanup();
return speed_upload;
}
}
Sumana, sorry for the late response, you should consider looking at this documentation
Here's an example using SAS with block blob using C#:
string sasUri = "https://storagesample.blob.core.windows.net/sample-container/" +
"sampleBlob.txt?sv=2015-07-08&sr=b&sig=39Up9JzHkxhUIhFEjEH9594DJxe7w6cIRCg0V6lCGSo%3D" +
"&se=2016-10-18T21%3A51%3A37Z&sp=rcw";
CloudBlockBlob blob = new CloudBlockBlob(new Uri(sasUri));
// Create operation: Upload a blob with the specified name to the container.
// If the blob does not exist, it will be created. If it does exist, it will be overwritten.
try
{
MemoryStream msWrite = new MemoryStream(Encoding.UTF8.GetBytes(blobContent));
msWrite.Position = 0;
using (msWrite)
{
await blob.UploadFromStreamAsync(msWrite);
}
Console.WriteLine("Create operation succeeded for SAS {0}", sasUri);
Console.WriteLine();
}
catch (StorageException e)
{
if (e.RequestInformation.HttpStatusCode == 403)
{
Console.WriteLine("Create operation failed for SAS {0}", sasUri);
Console.WriteLine("Additional error information: " + e.Message);
Console.WriteLine();
}
else
{
Console.WriteLine(e.Message);
Console.ReadLine();
throw;
}
}
Let me know if this helps.
In a C project I want to reuse curl easy handles. The flow of program is like:
Client --> C Application --> Call URL1, do something, Call URL2, do something, Call URL3...
In short, for each client request, the same set of URLs are called.
I initially create the curl easy handle when the program starts. The main program creates a configurable number of child, so each child gets its own easy handle.
static int child_init(int rank) {
LM_NOTICE("init_child [%d] pid [%d]\n", rank, getpid());
pid = my_pid();
curl_global_init(CURL_GLOBAL_ALL);
// initialize curl handle
curl = curl_easy_init();
if (!curl) {
LM_ERR("Child %d: Curl initialization failed.\n", rank);
return -1;
}
//create some connections before actual requests come.
curl_head(URL);
return 0;
}
I created a C file where the functions are created to handle GET/POST/PUT etc requests:
int curl_head(const char* url) {
if (!url) {
LM_ERR("URL not provided. Returning with error.\n");
return -1;
}
CURLcode res;
int http_code = 0;
struct curl_slist *headers = NULL;
headers = curl_slist_append(headers, "charsets: utf-8");
/* set URL */
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "HEAD");
curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
LM_ERR("curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
}
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
LM_DBG("HTTP return CODE %d\n", http_code);
curl_slist_free_all(headers);
curl_easy_reset(curl);
return http_code;
}
int curl_post(const char* url, char *postdata) {
if (!url) {
LM_ERR("URL not provided. Returning with error.\n");
return -1;
}
CURLcode res;
int http_code = 0;
struct curl_slist *headers = NULL;
headers = curl_slist_append(headers, "Accept: application/json");
headers = curl_slist_append(headers, "Content-Type: application/json");
headers = curl_slist_append(headers, "charsets: utf-8");
/* set URL */
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, postdata);
curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcrp/0.1");
res = curl_easy_perform(curl);
if (res != CURLE_OK) {
LM_ERR("curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
}
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
curl_slist_free_all(headers);
curl_easy_reset(curl);
return http_code;
}
When somewhere in the program I want to call the WS I call the desired function as curl_post(url).
Am I doing this correctly or are there any flaws in this implementation?
I am a beginner in both C programming and libcurl and writing a program to fetch 1000 data values from a website. The website provides a job number and is redirected into another page for the results. Since, the code I have written is almost 500 lines, I am giving a general flow of the program and a short code which I think is the problematic area:
for(row=0;row<1000;row++)
{
------
url = "http://example.com";
curl_global_init(CURL_GLOBAL_ALL);
curlHandle = curl_easy_init();
if(curlHandle)
{
curl_easy_setopt(curlHandle, CURLOPT_TIMEOUT, 1800);
curl_easy_setopt(curlHandle, CURLOPT_ERRORBUFFER, curlErrStr);
curl_easy_setopt(curlHandle, CURLOPT_FOLLOWLOCATION, 1);
curl_easy_setopt(curlHandle, CURLOPT_URL, url);
curl_easy_setopt(curlHandle, CURLOPT_LOW_SPEED_LIMIT, dl_lowspeed_bytes);
curl_easy_setopt(curlHandle, CURLOPT_LOW_SPEED_TIME, dl_lowspeed_time);
curl_easy_setopt(curlHandle, CURLOPT_VERBOSE, 1L);
free(url);
curlErr = curl_easy_perform(curlHandle);
if(curlErr != CURLE_OK)
{
fprintf(stderr, "curl_easy_perform() failed: %s\n",curl_easy_strerror(curlErr));
}
else
{
curlErr = curl_easy_getinfo(curlHandle, CURLINFO_EFFECTIVE_URL, &url_new);
if((CURLE_OK == curlErr) && url_new)
{
sprintf(job,"%.*s\n", 18, url_new + 28);
if((ptr1 = strchr(job, '\n')) != NULL)
*ptr1 = '\0';
init_string(&s);
curl_easy_setopt(curlHandle, CURLOPT_TIMEOUT, 1800 );
curl_easy_setopt(curlHandle, CURLOPT_URL, url_new);
curl_easy_setopt(curlHandle, CURLOPT_WRITEFUNCTION, writefunc);
curl_easy_setopt(curlHandle, CURLOPT_WRITEDATA, &s);
curlErr1 = curl_easy_perform(curlHandle);
printf("###### %lu\t%s\n",strlen(s.ptr),s.ptr);
free(s.ptr);
}
curl_easy_cleanup(curlHandle);
}
}
The functions are:
struct string
{
char *ptr;
size_t len;
};
void init_string(struct string *a)
{
a->len = 0;
a->ptr = malloc(a->len+1);
if (a->ptr == NULL)
{
fprintf(stderr, "malloc() failed\n");
exit(EXIT_FAILURE);
}
a->ptr[0] = '\0';
}
size_t writefunc(void *ptr, size_t size, size_t nmemb, struct string *a)
{
size_t new_len = a->len + size*nmemb;
a->ptr = realloc(a->ptr, new_len+1);
if (a->ptr == NULL)
{
fprintf(stderr, "realloc() failed\n");
exit(EXIT_FAILURE);
}
memcpy(a->ptr+a->len, ptr, size*nmemb);
a->ptr[new_len] = '\0';
a->len = new_len;
return size*nmemb;
}
The program shows no error of any kind. But out of the 1000 data, almost 50% couldn't be fetched due to curl_easy_perform() failed: Timeout was reached; and 20% of them have the output of the line strlen(s.ptr),s.ptr => 0. The rest are fetched correctly.
The verbose option for the zero output gave the following:
Connection #0 to host www.example.com left intact
getaddrinfo(3) failed for :80
Couldn't resolve host ''
Closing connection #1
Couldn't resolve host name
0
Please suggest the possible errors in the program.
Here is how I would fetch data using cURL
static CURL *curl = NULL;
CURL *initCURL(void)
{
curl_global_init(CURL_GLOBAL_DEFAULT);
curl = curl_easy_init();
if(curl)
{
// now set all the desired options
curl_easy_setopt(curl, CURLOPT_URL, "http://example.com");
/* example.com is redirected, so we tell libcurl to follow redirection */
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
// etc
}
else
{ // else cURL object creation failed
// display appropriate error message
}
}
void endCurl(void)
{
// and then when all done with the cURL object,
// cleanup
curl_easy_cleanup(curl);
}
CURLcode execCurl( CURL *curl )
{
CURLcode res;
// Perform this request, for each fetch
res = curl_easy_perform(curl);
// Check for errors
if(res != CURLE_OK)
{
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
}
return( res );
}
Note:
I have had this same problem with the cURL timeout occurring.
The best recovery method I found is:
when a timeout occurs, retry the communication, requesting the same data
I need a program that polls a site every second and responds if the site doesnt respond in 15 seconds. I made the following hack from an example program. On a dry run, it printed 7 times in 15 seconds. Cant I make curl_easy_perform wait for 15 seconds before responding?
int main(void)
{
CURL *curl;
CURLcode res;
char *postthis="moo mooo moo moo";
curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_URL, "http://192.168.1.101");
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, postthis);
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, (long)strlen(postthis));
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 15);
while(1)
{
Sleep(1000);
res = curl_easy_perform(curl);
if(res!= CURLE_OK)
printf("nada \n");
}
Sleep(15000 - timeTakenForCurlInMs);
It was a dumb hack that fails every second. It was based on the POST tutorial. I hacked the GET stuff and it works fine.
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "http://URL_HERE");
**curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);**
while(1)
{
Sleep(1000);
res = curl_easy_perform(curl);
if(res== CURLE_OK) printf("\n\n yeah \n");
else printf("\n\n nada \n");
}
/* always cleanup */
curl_easy_cleanup(curl);
}
return 0;
}