This question already has an answer here:
Closed 11 years ago.
Possible Duplicate:
Trying to Access Twitter Streaming API with C
I am not sure what I am doing wrong in my code but it seems there's a problem on authentication. Every time I execute the code there is no output.
My goal here is I wanted to receive a stream of Tweets from Twitter API. The problem might be something else. But I am not sure. Please help.
This is the C code:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <curl/curl.h>
struct string {
char *ptr;
size_t len;
};
void init_string(struct string *s) {
s->len = 0;
s->ptr = malloc(s->len+1);
if (s->ptr == NULL) {
fprintf(stderr, "malloc() failed\n");
exit(EXIT_FAILURE);
}
s->ptr[0] = '\0';
}
size_t writefunc(void *ptr, size_t size, size_t nmemb, struct string *s)
{
size_t new_len = s->len + size*nmemb;
s->ptr = realloc(s->ptr, new_len+1);
if (s->ptr == NULL) {
fprintf(stderr, "realloc() failed\n");
exit(EXIT_FAILURE);
}
memcpy(s->ptr+s->len, ptr, size*nmemb);
s->ptr[new_len] = '\0';
s->len = new_len;
return size*nmemb;
}
int main(void)
{
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
struct string s;
init_string(&s);
curl_easy_setopt(curl, CURLOPT_URL, "http://stream.twitter.com/1/statuses/sample.json");
curl_easy_setopt(curl, CURLOPT_USERPWD, "neilmarion:password_here");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writefunc);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s);
res = curl_easy_perform(curl);
printf("%s\n", s.ptr);
free(s.ptr);
/* always cleanup */
curl_easy_cleanup(curl);
}
return 0;
}
Use ssl.
You could use curl_easy_strerror() to get a human readable error message:
printf("curl error %s", curl_easy_strerror(res));
You'll run out of memory after a while (at my end the stream is ~250Kb/s). Save interesting info to a persistent storage and discard the rest.
Related
I use libcurl library to get all email in sent mail box from url : imaps://imap.gmail.com:993/[Gmail]/Sent Mail .But it not run and show error :
curl_easy_perform() failed: URL using bad/illegal format or missing URL
Please help me to check it . Thanks in advance. Toan Nguyen
struct string {
char *ptr;
size_t len;
};
void init_string(struct string *s) {
s->len = 0;
s->ptr = malloc(s->len+1);
if (s->ptr == NULL) {
fprintf(stderr, "malloc() failed\n");
exit(EXIT_FAILURE);
}
s->ptr[0] = '\0';
}
size_t writefunc(void *ptr, size_t size, size_t nmemb, struct string *s)
{
size_t new_len = s->len + size*nmemb;
s->ptr = realloc(s->ptr, new_len+1);
if (s->ptr == NULL) {
fprintf(stderr, "realloc() failed\n");
exit(EXIT_FAILURE);
}
memcpy(s->ptr+s->len, ptr, size*nmemb);
s->ptr[new_len] = '\0';
s->len = new_len;
return size*nmemb;
}
int main(int argc,char *argv[])
{
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
struct string s;
init_string(&s);
curl_easy_setopt(curl,CURLOPT_USERNAME,argv[1]);
curl_easy_setopt(curl,CURLOPT_PASSWORD,argv[2]);
curl_easy_setopt(curl, CURLOPT_URL, "imaps://imap.gmail.com:993/%5BGmail%5D%2FSent%20Mail;UID=*");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writefunc);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s);
res = curl_easy_perform(curl);
printf("%s\n", s.ptr);
free(s.ptr);
/* Check for errors */
if(res != CURLE_OK)
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
/* always cleanup */
curl_easy_cleanup(curl);
}
return 0;
}
The problem is in your IMAPS URI. According to RFC3501, mailbox names with spaces must be enclosed between double quotes " when used with SELECT or any other IMAP command.
The following should work:
curl_easy_setopt(curl, CURLOPT_URL, "imaps://imap.gmail.com:993/\"[Gmail]/All Mail\"");
I have homework where I need somehow to compare two HTTP responses. I am writing it on C and I use libcurl to make things easier. I am calling the function that uses libcurl to do a HTTP request and response from another function, and I want to return the HTTP response as a char *. Here is my code so far (it crashes):
#include <stdio.h>
#include <curl/curl.h>
#include <string.h>
size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream) {
size_t written;
written = fwrite(ptr, size, nmemb, stream);
return written;
}
char *handle_url(void) {
CURL *curl;
char *fp;
CURLcode res;
char *url = "http://www.yahoo.com";
curl = curl_easy_init();
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
res = curl_easy_perform(curl);
if(res != CURLE_OK)
fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
curl_easy_cleanup(curl);
//printf("\n%s", fp);
}
return fp;
}
This solution C libcurl get output into a string works, but not in my case because I just want to return the string to the calling function.
Any ideas?
Fixed it for you. You need to handle the case where the write_data() function is called multiple times, and pass it the right kind of parameter. You also need to keep track of how big a structure you've got, so you can allocate enough memory.
I left in a debug printf in the write_data function to help you understand how it works.
#include <stdio.h>
#include <curl/curl.h>
#include <string.h>
#include <stdlib.h>
struct url_data {
size_t size;
char* data;
};
size_t write_data(void *ptr, size_t size, size_t nmemb, struct url_data *data) {
size_t index = data->size;
size_t n = (size * nmemb);
char* tmp;
data->size += (size * nmemb);
#ifdef DEBUG
fprintf(stderr, "data at %p size=%ld nmemb=%ld\n", ptr, size, nmemb);
#endif
tmp = realloc(data->data, data->size + 1); /* +1 for '\0' */
if(tmp) {
data->data = tmp;
} else {
if(data->data) {
free(data->data);
}
fprintf(stderr, "Failed to allocate memory.\n");
return 0;
}
memcpy((data->data + index), ptr, n);
data->data[data->size] = '\0';
return size * nmemb;
}
char *handle_url(char* url) {
CURL *curl;
struct url_data data;
data.size = 0;
data.data = malloc(4096); /* reasonable size initial buffer */
if(NULL == data.data) {
fprintf(stderr, "Failed to allocate memory.\n");
return NULL;
}
data.data[0] = '\0';
CURLcode res;
curl = curl_easy_init();
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &data);
res = curl_easy_perform(curl);
if(res != CURLE_OK) {
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
}
curl_easy_cleanup(curl);
}
return data.data;
}
int main(int argc, char* argv[]) {
char* data;
if(argc < 2) {
fprintf(stderr, "Must provide URL to fetch.\n");
return 1;
}
data = handle_url(argv[1]);
if(data) {
printf("%s\n", data);
free(data);
}
return 0;
}
Note: compile with gcc -o test test.c -lcurl (assuming you pasted into test.c). Use gcc -o test test.c -lcurl -DDEBUG to see the test printf() calls.
Disclaimer: this is ugly, quick-and-dirty code. There may be bugs. Please see the more robust, better commented example here.
This question already has an answer here:
Closed 11 years ago.
Possible Duplicate:
Using HTTP authentication with libcurl in C for Twitter Streaming
I was able to write a C code that receives a stream of Tweets from streaming API. But the stream is not being put out (no output) by the code below. The code works when CURLOPT_URL is set to google.com or 9gag.com. I guess the problem has something to do with the steady stream of tweets with tremendous received data. The write_func callback function, which purpose is to print the response (stream), might not be working that is why there is no output? I am thinking that the callback function probably is being overwhelmed by the tremendous stream sent by Twitter API. Then if this is the case, how should I write the proper write callback function?
You might ask. I verified that the reception of stream is working because I watch my System Monitor's network history rise on received bytes whenever I execute the code.
Thanks!
The code:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <curl/curl.h>
struct string {
char *ptr;
size_t len;
};
void init_string(struct string *s) {
s->len = 0;
s->ptr = malloc(s->len+1);
if (s->ptr == NULL) {
fprintf(stderr, "malloc() failed\n");
exit(EXIT_FAILURE);
}
s->ptr[0] = '\0';
}
size_t writefunc(void *ptr, size_t size, size_t nmemb, struct string *s)
{
size_t new_len = s->len + size*nmemb;
s->ptr = realloc(s->ptr, new_len+1);
if (s->ptr == NULL) {
fprintf(stderr, "realloc() failed\n");
exit(EXIT_FAILURE);
}
memcpy(s->ptr+s->len, ptr, size*nmemb);
s->ptr[new_len] = '\0';
s->len = new_len;
return size*nmemb;
}
int main(void)
{
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
struct string s;
init_string(&s);
curl_easy_setopt(curl, CURLOPT_URL, "https://stream.twitter.com/1/statuses/sample.json");
curl_easy_setopt(curl, CURLOPT_USERPWD, "neilmarion:password_here");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writefunc);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s);
res = curl_easy_perform(curl);
printf("%s\n", s.ptr);
free(s.ptr);
/* always cleanup */
curl_easy_cleanup(curl);
}
return 0;
}
OK, I did some testing and it appears that URL just keeps sending data, it doesn't seem to complete. I killed it off after 15M. But if you put print statements in your callback or use strace you can see its working properly. Your string s just keeps growing and growing.
So one solution would be to change you callback to print and re-initialse s once it reaches a certain size. Otherwise it looks like the program will eventually run out of memory. So change your callback to be
size_t max_buffer = 10240; // 10K
size_t writefunc(void *ptr, size_t size, size_t nmemb, struct string *s)
{
size_t new_len = s->len + size*nmemb;
s->ptr = realloc(s->ptr, new_len+1);
if (s->ptr == NULL) {
fprintf(stderr, "realloc() failed\n");
exit(EXIT_FAILURE);
}
memcpy(s->ptr+s->len, ptr, size*nmemb);
s->ptr[new_len] = '\0';
s->len = new_len;
// Begin newly added code
if( s->len >= max_buffer )
{
printf("%s", s->ptr);
fflush( stdout );
free(s->ptr);
initString( s );
}
// End newly added code
return size*nmemb;
}
And still keep the print at the end. To dump the last bit and the trailing newline. Now that you have a buffered solution you could look at a more efficient implementation which doesn't need dynamically added memory.
I got the code from C libcurl get output into a string. I modified it and I want to use it to access the Twitter Stream. I added curl_easy_setopt(curl, CURLOPT_URL, "http://stream.twitter.com/1/statuses/sample.json"); and curl_easy_setopt(curl, CURLOPT_USERPWD, "neilmarion:my_password"); in the code. But the problem is whenever I execute it, there is no output. What must be the problem? Thanks.
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <curl/curl.h>
struct string {
char *ptr;
size_t len;
};
void init_string(struct string *s) {
s->len = 0;
s->ptr = malloc(s->len+1);
if (s->ptr == NULL) {
fprintf(stderr, "malloc() failed\n");
exit(EXIT_FAILURE);
}
s->ptr[0] = '\0';
}
size_t writefunc(void *ptr, size_t size, size_t nmemb, struct string *s)
{
size_t new_len = s->len + size*nmemb;
s->ptr = realloc(s->ptr, new_len+1);
if (s->ptr == NULL) {
fprintf(stderr, "realloc() failed\n");
exit(EXIT_FAILURE);
}
memcpy(s->ptr+s->len, ptr, size*nmemb);
s->ptr[new_len] = '\0';
s->len = new_len;
return size*nmemb;
}
int main(void)
{
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
struct string s;
init_string(&s);
curl_easy_setopt(curl, CURLOPT_URL, "http://stream.twitter.com/1/statuses/sample.json");
curl_easy_setopt(curl, CURLOPT_USERPWD, "neilmarion:my_password");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writefunc);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s);
res = curl_easy_perform(curl);
printf("%s\n", s.ptr);
free(s.ptr);
/* always cleanup */
curl_easy_cleanup(curl);
}
return 0;
}
The quickest next step is probably to set CURLOPT_HEADER, to include headers in the body output. Most likely, I would guess it is failing on security, and you'll see the details in the headers.
I want to store the result of this curl function in a variable, how can I do so?
#include <stdio.h>
#include <curl/curl.h>
int main(void)
{
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "curl.haxx.se");
res = curl_easy_perform(curl);
/* always cleanup */
curl_easy_cleanup(curl);
}
return 0;
}
thanks, I solved it like this:
#include <stdio.h>
#include <stdlib.h>
#include <curl/curl.h>
function_pt(void *ptr, size_t size, size_t nmemb, void *stream){
printf("%d", atoi(ptr));
}
int main(void)
{
CURL *curl;
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, "curl.haxx.se");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, function_pt);
curl_easy_perform(curl);
curl_easy_cleanup(curl);
}
system("pause");
return 0;
}
You can set a callback function to receive incoming data chunks using curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, myfunc);
The callback will take a user defined argument that you can set using curl_easy_setopt(curl, CURLOPT_WRITEDATA, p)
Here's a snippet of code that passes a buffer struct string {*ptr; len} to the callback function and grows that buffer on each call using realloc().
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <curl/curl.h>
struct string {
char *ptr;
size_t len;
};
void init_string(struct string *s) {
s->len = 0;
s->ptr = malloc(s->len+1);
if (s->ptr == NULL) {
fprintf(stderr, "malloc() failed\n");
exit(EXIT_FAILURE);
}
s->ptr[0] = '\0';
}
size_t writefunc(void *ptr, size_t size, size_t nmemb, struct string *s)
{
size_t new_len = s->len + size*nmemb;
s->ptr = realloc(s->ptr, new_len+1);
if (s->ptr == NULL) {
fprintf(stderr, "realloc() failed\n");
exit(EXIT_FAILURE);
}
memcpy(s->ptr+s->len, ptr, size*nmemb);
s->ptr[new_len] = '\0';
s->len = new_len;
return size*nmemb;
}
int main(void)
{
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if(curl) {
struct string s;
init_string(&s);
curl_easy_setopt(curl, CURLOPT_URL, "curl.haxx.se");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writefunc);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s);
res = curl_easy_perform(curl);
printf("%s\n", s.ptr);
free(s.ptr);
/* always cleanup */
curl_easy_cleanup(curl);
}
return 0;
}
The following answer is the C++ way to do it, with std::string, instead of null-terminated string. It still uses a callback function (there's no way around it), but also handles allocation error using try/catch.
#include <iostream>
#include <string>
#include <curl/curl.h>
size_t CurlWrite_CallbackFunc_StdString(void *contents, size_t size, size_t nmemb, std::string *s)
{
size_t newLength = size*nmemb;
try
{
s->append((char*)contents, newLength);
}
catch(std::bad_alloc &e)
{
//handle memory problem
return 0;
}
return newLength;
}
int main()
{
CURL *curl;
CURLcode res;
curl_global_init(CURL_GLOBAL_DEFAULT);
curl = curl_easy_init();
std::string s;
if(curl)
{
curl_easy_setopt(curl, CURLOPT_URL, "curl.haxx.se");
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); //only for https
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); //only for https
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlWrite_CallbackFunc_StdString);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s);
curl_easy_setopt (curl, CURLOPT_VERBOSE, 1L); //remove this to disable verbose output
/* Perform the request, res will get the return code */
res = curl_easy_perform(curl);
/* Check for errors */
if(res != CURLE_OK)
{
fprintf(stderr, "curl_easy_perform() failed: %s\n",
curl_easy_strerror(res));
}
/* always cleanup */
curl_easy_cleanup(curl);
}
std::cout<<s<<std::endl;
std::cout<< "Program finished!" << std::endl;
}
From reading the manual here: http://curl.haxx.se/libcurl/c/curl_easy_setopt.html I think you need several calls to CURL_SETOPT, the first being the URL you want to process, the second being something like:
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, function_ptr);
Where function_ptr matches this signature:
size_t function( void *ptr, size_t size, size_t nmemb, void *stream)
What happens here is you denote a callback function which libcurl will call when it has some output to write from whatever transfer you've invoked. You can get it to automatically write to a file, or pass it a pointer to a function which will handle the output itself. Using this function you should be able to assemble the various output strings into one piece and then use them in your program.
I'm not sure what other options you may have to set / what else affects how you want your app to behave, so have a good look through that page.
Here's a C++ flavor of the accepted answer from alex-jasmin
#include <iostream>
#include <string>
#include <curl/curl.h>
size_t writefunc(void *ptr, size_t size, size_t nmemb, std::string *s)
{
s->append(static_cast<char *>(ptr), size*nmemb);
return size*nmemb;
}
int main(void)
{
CURL *curl = curl_easy_init();
if (curl)
{
std::string s;
curl_easy_setopt(curl, CURLOPT_URL, "curl.haxx.se");
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writefunc);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s);
CURLcode res = curl_easy_perform(curl);
std::cout << s << std::endl;
/* always cleanup */
curl_easy_cleanup(curl);
}
return 0;
}