I have to check the network upload speed as well as upload a file in azure blob storage.
Initially I have tried with some sample HTTP servers to check the network speed and it's working. But when I am trying to send the file to azure blob storage HTTP endpoint, I am not sure how to use the SAS token for the authentication or the username and password will be enough to do the authentication and proceed with further procedure.
I have attached the code I have written , Please give some examples or reliable link for the authentication with SAS token and file upload.
size_t uploadspeed()
{
CURL *curl;
CURLcode res;
FILE *hd_src;
struct stat file_info;
curl_off_t fsize;
size_t speed_upload, total_time1;
hd_src = fopen("bbb.txt", "rb");
curl = curl_easy_init();
if(curl) {
int old_stdout = dup(2);
freopen("/dev/null" , "w+" , stderr);
FILE *f = fopen("target.txt", "wb");
curl_easy_setopt(curl, CURLOPT_WRITEDATA, f);
curl_easy_setopt(curl, CURLOPT_URL,
"AZURE_URL");
curl_easy_setopt(curl, CURLOPT_USERPWD, "A:B");
curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(curl, CURLOPT_READDATA, hd_src);
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE,(curl_off_t)file_info.st_size);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
res = curl_easy_perform(curl);
fclose(stderr);
if(res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
}
else {
curl_easy_getinfo(curl, CURLINFO_SPEED_UPLOAD, &speed_upload);
curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &total_time1);
}
curl_easy_cleanup(curl);
fclose(hd_src);
curl_global_cleanup();
return speed_upload;
}
}
Sumana, sorry for the late response, you should consider looking at this documentation
Here's an example using SAS with block blob using C#:
string sasUri = "https://storagesample.blob.core.windows.net/sample-container/" +
"sampleBlob.txt?sv=2015-07-08&sr=b&sig=39Up9JzHkxhUIhFEjEH9594DJxe7w6cIRCg0V6lCGSo%3D" +
"&se=2016-10-18T21%3A51%3A37Z&sp=rcw";
CloudBlockBlob blob = new CloudBlockBlob(new Uri(sasUri));
// Create operation: Upload a blob with the specified name to the container.
// If the blob does not exist, it will be created. If it does exist, it will be overwritten.
try
{
MemoryStream msWrite = new MemoryStream(Encoding.UTF8.GetBytes(blobContent));
msWrite.Position = 0;
using (msWrite)
{
await blob.UploadFromStreamAsync(msWrite);
}
Console.WriteLine("Create operation succeeded for SAS {0}", sasUri);
Console.WriteLine();
}
catch (StorageException e)
{
if (e.RequestInformation.HttpStatusCode == 403)
{
Console.WriteLine("Create operation failed for SAS {0}", sasUri);
Console.WriteLine("Additional error information: " + e.Message);
Console.WriteLine();
}
else
{
Console.WriteLine(e.Message);
Console.ReadLine();
throw;
}
}
Let me know if this helps.
Related
I'm trying to develop an EST client program in C using libcurl, to perform simpleenroll and simplereenroll. As of now, I was able to retrieve the certificate using ./well-known/est/simpleenroll., but I'm have a problem performing ./well-known/est/simplereenroll
url = self.url_prefix + '/simplereenroll'
auth = (self.username, self.password)
headers = {'Content-Type': 'application/pkcs10'}
content = est.request.post(url, csr, auth=auth, headers=headers,
verify=self.implicit_trust_anchor_cert_path,
cert=cert)
pem = self.pkcs7_to_pem(content)
I was referring this python code where I could see the certificate being passed for simplereenroll operation. I don't know how to implement the same using libcurl.
Below mentioned is the portion of libcurl code which I'm using to perform the simpleenroll.
/*SIMPLEENROLL
*
*/
if (res = curl_easy_setopt(curl,CURLOPT_URL,"https://localhost:4443/.well-known/est/simpleenroll")!=CURLE_OK)
{
fprintf(stderr,"curl_easy_setopt() failed: %s\n",curl_easy_strerror(res));
return 1;
}
struct curl_slist* slist = NULL;
slist = curl_slist_append(slist, "Content-Type: application/pkcs10");
curl_easy_setopt(curl,CURLOPT_USERPWD,"estuser:estpwd")
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, CSR);
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, CSR_LEN);
if((res = curl_easy_perform(curl))!=CURLE_OK)
{
fprintf(stderr,"curl_easy_perform() failed: %s\n",curl_easy_strerror(res));
return 1;
}
The above code will return a client certificate from server. Now I need to perform simplereenroll where I need to provide this certificate.
curl_easy_setopt(curl, CURLOPT_SSLKEY, client_key);
curl_easy_setopt(curl, CURLOPT_SSLCERT, client_cert);
curl_easy_setopt(curl, CURLOPT_CERTINFO, 1L);
I have a webpage: 10.1.1.165 that I am trying to get the data off of. I have built this program in linux and it works fine, however, when developing for windows systems I am running into an issue.
if (filename.substr(filename.length() - 4, 4) == "html" || filename.substr(filename.length() - 3, 3) == "htm")
{
CURL *curl;
CURLcode res;
curl_global_init(CURL_GLOBAL_DEFAULT);
curl = curl_easy_init();
string s;
if (curl)
{
curl_easy_setopt(curl, CURLOPT_URL, filename.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback);
if (gLogger->isDebug()) curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout/1000);
//curl_easy_setopt (curl, CURLOPT_VERBOSE, 1L); //remove this to disable verbose output
// Perform the request, res will get the return code
res = curl_easy_perform(curl);
// Check for errors
if (res != CURLE_OK)
{
while (res == CURLE_OPERATION_TIMEDOUT)
{
res = curl_easy_perform(curl);
}
if (res != CURLE_OK)
{
gLogger->error("curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
curl_easy_cleanup(curl);
return -1;
}
}
curl_easy_cleanup(curl);
On my linux system, this works perfectly. On my Windows system I get: curl_easy_perform() failed: Unsupported protocol. If I go to my webpage 10.1.1.165 I see that it is being hosted with NO security. I have no way of changing how that webpage is being hosted.
Per this resource: https://curl.haxx.se/libcurl/c/CURLOPT_PROTOCOLS.html
By default libcurl will accept all protocols
Does anyone know why curl doesn't like me grabbing this webpage on windows? And if so, what settings might I have to adjust?
I have solved my issue by switching to curl-7.55.1. Previously I was using curl-7.68.0. I'm not sure if the functionality was removed or if using it is more involved.
I'm using Curl library to create a simple C code with MSVC to download a file from a URL.
The problem is if the connection breaks in the middle of download my code will freeze and the unfinished file hasn't removed from the directory.
What I want is if the download failed the program must retry the connection or remove the unfinished file and then try again. I prefer to use C libraries rather than C++ libs. Here is the code I am using:
//lib for curl
#include <curl/curl.h>
#define CURL_STATICLIB
bool downloader3(string url, string file_path) {
CURL *curl;
FILE *fp;
CURLcode res;
curl = curl_easy_init();
if (curl) {
fp = fopen(file_path.c_str(), "wb");
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
//always cleanup
curl_easy_cleanup(curl);
fclose(fp);
double val;
res = curl_easy_getinfo(curl, CURLINFO_SPEED_DOWNLOAD, &val);
if ((CURLE_OK == res) && (val>0))
printf("Average download speed: %0.3f kbyte/sec.\n", val / 1024);
if ((res == CURLE_OK)) {
printf("Download Successful!\r\n");
return true;
}
else {
printf("Downlaod Failed!\r\n");
remove(file_path.c_str()); //remove the temp file
return false;
}
}
}
EDIT---
Thanks to Ring Ø answer. I modifed the code but I am looking for a resume capability that can resume the download of incomplete file.
bool downloader3(string url, string file_path) {
CURL *curl;
FILE *fp = NULL;
CURLcode res;
int status;
int maxtries = 3;
do {
printf("Doing try # %d\r\n", maxtries);
curl = curl_easy_init();
if (curl) {
fp = fopen(file_path.c_str(), "wb");
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L); // 30 seconds
res = curl_easy_perform(curl);
//always cleanup
curl_easy_cleanup(curl);
fclose(fp);
if ((res == CURLE_OK)) {
printf("Download Successful!\r\n");
break;
//return true;
}
}
} while (--maxtries);
if (maxtries) { // was OK
//curl_easy_cleanup(curl); // clean curl / delete file?
//fclose(fp);
return true;
}
else {
printf("Download Failed!\r\n");
printf("file path is: %s", file_path.c_str());
Sleep(5000);
status = remove(file_path.c_str()); //remove the unfinished file
if (status == 0)
printf("%s file deleted successfully.\n", file_path);
else
{
printf("Unable to delete the file\n");
}
return false;
}
}
You could set a timeout option
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 30L); // 30 seconds
if the operation is not done within 30 seconds, the timeout is triggered. Then check the result value, in a while loop for instance
res = curl_easy_perform( ... );
if (res == CURLE_OK) {
break;
}
// delete file
// keep retrying (add a counter if necessary)
See also the curl page.
Loop example
int maxtries = 5;
do {
curl = curl_easy_init();
if (curl) {
...
res = curl_easy_perform( ... );
if (res == CURLE_OK) {
break;
}
// delete file, curl cleanup...
}
} while ( --maxtries );
if (maxtries) { // was OK
// clean curl / delete file?
}
This is not the ideal solution, as you said, the download may take more or less time. This (should) prevent a never ending program, provided the timeout is big enough.
Curl library was known to have some problems in case of erratic connection - there could be something better nowadays, please try the latest stable build.
If you don't get a better answer within a few days, try to add a "Bounty" of 50 rep to attract more attention.
What you are looking for is the RESUME_FROM feature. To use this you must know which byte you want to start the download from. In this example it is an upload but should be same setopt technique. Here is example usage from curl website:
CURL *curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "ftp://example.com");
/* resume upload at byte index 200 */
curl_easy_setopt(curl, CURLOPT_RESUME_FROM, 200L);
/* ask for upload */
curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
/* set total data amount to expect */
curl_easy_setopt(curl, CURLOPT_INFILESIZE, size_of_file);
/* Perform the request */
curl_easy_perform(curl);
}
source: https://curl.haxx.se/libcurl/c/CURLOPT_RESUME_FROM.html
With this code I am sending strings to a webserver using libcurl and write the data to a MySQL (is done at the webserver).
My Problem is that for every call of this function the program starts a new key exchange with the webserver. I would like to have a persistant connection to the Server.
I searched here and web already and didnt find any satisfying solutions.
Multi-handler and forced keep alive still open a new connection.
Following is my Code to establish an SSL Connection:
CURL *curl;
CURLcode res;
res = curl_global_init(CURL_GLOBAL_DEFAULT); // Check for errors
if(res != CURLE_OK) {
fprintf(stderr, "curl_global_init() failed: %s\n",
curl_easy_strerror(res));
return 1;
}
// curl handler
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, STRING);
curl_easy_setopt(curl, CURLOPT_URL, "https://IP/something/something.php");
curl_easy_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1L);
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); //verbose output activated
struct curl_slist *headers=NULL;
headers = curl_slist_append(headers, "Content-Type: application/json"); // type JSON
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
// Perform the request, res will get the return code
res = curl_easy_perform(curl);
// Check for errors
if(res != CURLE_OK)
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
// cleanup
curl_slist_free_all(headers);
curl_easy_cleanup(curl);
}
curl_global_cleanup();
is Answered by Daniel Stenberg here, which is a similar/the same question.
Re-use the same curl handle in subsequent requests! Don't call curl_easy_cleanup(curl) and curl_easy_init() again between them.
So the Solution is to call curl_easy_cleanup(curl) and curl_easy_init() only once.
I am a beginner in both C programming and libcurl and writing a program to fetch 1000 data values from a website. The website provides a job number and is redirected into another page for the results. Since, the code I have written is almost 500 lines, I am giving a general flow of the program and a short code which I think is the problematic area:
for(row=0;row<1000;row++)
{
------
url = "http://example.com";
curl_global_init(CURL_GLOBAL_ALL);
curlHandle = curl_easy_init();
if(curlHandle)
{
curl_easy_setopt(curlHandle, CURLOPT_TIMEOUT, 1800);
curl_easy_setopt(curlHandle, CURLOPT_ERRORBUFFER, curlErrStr);
curl_easy_setopt(curlHandle, CURLOPT_FOLLOWLOCATION, 1);
curl_easy_setopt(curlHandle, CURLOPT_URL, url);
curl_easy_setopt(curlHandle, CURLOPT_LOW_SPEED_LIMIT, dl_lowspeed_bytes);
curl_easy_setopt(curlHandle, CURLOPT_LOW_SPEED_TIME, dl_lowspeed_time);
curl_easy_setopt(curlHandle, CURLOPT_VERBOSE, 1L);
free(url);
curlErr = curl_easy_perform(curlHandle);
if(curlErr != CURLE_OK)
{
fprintf(stderr, "curl_easy_perform() failed: %s\n",curl_easy_strerror(curlErr));
}
else
{
curlErr = curl_easy_getinfo(curlHandle, CURLINFO_EFFECTIVE_URL, &url_new);
if((CURLE_OK == curlErr) && url_new)
{
sprintf(job,"%.*s\n", 18, url_new + 28);
if((ptr1 = strchr(job, '\n')) != NULL)
*ptr1 = '\0';
init_string(&s);
curl_easy_setopt(curlHandle, CURLOPT_TIMEOUT, 1800 );
curl_easy_setopt(curlHandle, CURLOPT_URL, url_new);
curl_easy_setopt(curlHandle, CURLOPT_WRITEFUNCTION, writefunc);
curl_easy_setopt(curlHandle, CURLOPT_WRITEDATA, &s);
curlErr1 = curl_easy_perform(curlHandle);
printf("###### %lu\t%s\n",strlen(s.ptr),s.ptr);
free(s.ptr);
}
curl_easy_cleanup(curlHandle);
}
}
The functions are:
struct string
{
char *ptr;
size_t len;
};
void init_string(struct string *a)
{
a->len = 0;
a->ptr = malloc(a->len+1);
if (a->ptr == NULL)
{
fprintf(stderr, "malloc() failed\n");
exit(EXIT_FAILURE);
}
a->ptr[0] = '\0';
}
size_t writefunc(void *ptr, size_t size, size_t nmemb, struct string *a)
{
size_t new_len = a->len + size*nmemb;
a->ptr = realloc(a->ptr, new_len+1);
if (a->ptr == NULL)
{
fprintf(stderr, "realloc() failed\n");
exit(EXIT_FAILURE);
}
memcpy(a->ptr+a->len, ptr, size*nmemb);
a->ptr[new_len] = '\0';
a->len = new_len;
return size*nmemb;
}
The program shows no error of any kind. But out of the 1000 data, almost 50% couldn't be fetched due to curl_easy_perform() failed: Timeout was reached; and 20% of them have the output of the line strlen(s.ptr),s.ptr => 0. The rest are fetched correctly.
The verbose option for the zero output gave the following:
Connection #0 to host www.example.com left intact
getaddrinfo(3) failed for :80
Couldn't resolve host ''
Closing connection #1
Couldn't resolve host name
0
Please suggest the possible errors in the program.
Here is how I would fetch data using cURL
static CURL *curl = NULL;
CURL *initCURL(void)
{
curl_global_init(CURL_GLOBAL_DEFAULT);
curl = curl_easy_init();
if(curl)
{
// now set all the desired options
curl_easy_setopt(curl, CURLOPT_URL, "http://example.com");
/* example.com is redirected, so we tell libcurl to follow redirection */
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
// etc
}
else
{ // else cURL object creation failed
// display appropriate error message
}
}
void endCurl(void)
{
// and then when all done with the cURL object,
// cleanup
curl_easy_cleanup(curl);
}
CURLcode execCurl( CURL *curl )
{
CURLcode res;
// Perform this request, for each fetch
res = curl_easy_perform(curl);
// Check for errors
if(res != CURLE_OK)
{
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
}
return( res );
}
Note:
I have had this same problem with the cURL timeout occurring.
The best recovery method I found is:
when a timeout occurs, retry the communication, requesting the same data