tpf80
10th May 2007, 20:55
I am using LibCURL to pull in data from various web sites. I have made the callback function work in the following way (which works properly but has horrible performance):
struct MemoryStruct {
char *memory;
size_t size;
};
static size_t WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data) {
size_t realsize = size * nmemb;
struct MemoryStruct *mem = (struct MemoryStruct *)data;
//increase the size of "memory" by the size of the bytes that we have read
mem->memory = (char *)realloc(mem->memory, mem->size + realsize + 1);
if (mem->memory) {
//copy to the end of the memory chunk, the piece of data that we read from curl (pointed to by ptr, size of realsize):
memcpy(&(mem->memory[mem->size]), ptr, realsize);
//update the size of the memory chunk:
mem->size += realsize;
//put a 0 at the end of the memory chunk
mem->memory[mem->size] = 0;
}
return realsize;
}
void FabwareMain::curl_init() {
//init the memory to hold what we gather with CURL
struct MemoryStruct chunk;
chunk.memory=NULL; /* we expect realloc(NULL, size) to work */
chunk.size = 0; /* no data at this point */
//init curl:
curl_global_init( CURL_GLOBAL_ALL ) ;
//create a curl handle:
CURL *curl_handle = curl_easy_init() ;
//the login page:
//set curl options:
curl_easy_setopt(curl_handle, CURLOPT_COOKIEFILE, "");
curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1); //when redirected, follow the redirections
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.3) Gecko/20060601 Firefox/2.0.0.3 (Ubuntu-edgy)");
curl_easy_setopt(curl_handle, CURLOPT_POST, 1);
curl_easy_setopt(curl_handle, CURLOPT_URL, "http://www.somewhere.com");
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, "");
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *)&chunk);
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
//exec curl
curl_easy_perform(curl_handle);
//convert the memory chunk to a qstring
resData = QString::fromUtf8 (chunk.memory);
ui.textEdit->append (resData);
//parse the qstring
//clean up after ourselves:
curl_easy_cleanup(curl_handle);
curl_global_cleanup();
//clean up our memory chunk
if(chunk.memory) {
free(chunk.memory);
}
}
The problem with this method, is that although it works, it consumes a ton of memory and time due to copying the data many times. First CURL copies the data into a buffer, then the callback function copies it into the data chunk, then we convert it into a Qstring, then we copy that into the text edit box.
I have found that I can modify the callback function to put the data from CURL into a QString as it comes in, and parse out what I need on the fly. I can not however figure out how to make the data that is in the callback function QString be available outside of the function. I can't return the data as CURL calls the function, so I will need to store it in a way that can be accessed from the callback function and the rest of the Qt program.
Heres the current callback function that I am modifying:
struct MemoryStruct {
char *memory;
size_t size;
};
static size_t WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data) {
size_t realsize = size * nmemb;
struct MemoryStruct *mem = (struct MemoryStruct *)data;
//increase the size of "memory" by the size of the bytes that we have read
mem->memory = (char *)realloc(mem->memory, mem->size + realsize + 1);
if (mem->memory) {
QString qmemData = QString::fromUtf8 ((char *)ptr);
//parse out the stuff that I want
//How do I get this QString (qmemData) to be accessable outside of this function
}
return realsize;
}
How do I get the QString qmemData to be accessable outside of this function? Im sure its something simple I am missing somewhere.
struct MemoryStruct {
char *memory;
size_t size;
};
static size_t WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data) {
size_t realsize = size * nmemb;
struct MemoryStruct *mem = (struct MemoryStruct *)data;
//increase the size of "memory" by the size of the bytes that we have read
mem->memory = (char *)realloc(mem->memory, mem->size + realsize + 1);
if (mem->memory) {
//copy to the end of the memory chunk, the piece of data that we read from curl (pointed to by ptr, size of realsize):
memcpy(&(mem->memory[mem->size]), ptr, realsize);
//update the size of the memory chunk:
mem->size += realsize;
//put a 0 at the end of the memory chunk
mem->memory[mem->size] = 0;
}
return realsize;
}
void FabwareMain::curl_init() {
//init the memory to hold what we gather with CURL
struct MemoryStruct chunk;
chunk.memory=NULL; /* we expect realloc(NULL, size) to work */
chunk.size = 0; /* no data at this point */
//init curl:
curl_global_init( CURL_GLOBAL_ALL ) ;
//create a curl handle:
CURL *curl_handle = curl_easy_init() ;
//the login page:
//set curl options:
curl_easy_setopt(curl_handle, CURLOPT_COOKIEFILE, "");
curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1); //when redirected, follow the redirections
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.3) Gecko/20060601 Firefox/2.0.0.3 (Ubuntu-edgy)");
curl_easy_setopt(curl_handle, CURLOPT_POST, 1);
curl_easy_setopt(curl_handle, CURLOPT_URL, "http://www.somewhere.com");
curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, "");
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *)&chunk);
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
//exec curl
curl_easy_perform(curl_handle);
//convert the memory chunk to a qstring
resData = QString::fromUtf8 (chunk.memory);
ui.textEdit->append (resData);
//parse the qstring
//clean up after ourselves:
curl_easy_cleanup(curl_handle);
curl_global_cleanup();
//clean up our memory chunk
if(chunk.memory) {
free(chunk.memory);
}
}
The problem with this method, is that although it works, it consumes a ton of memory and time due to copying the data many times. First CURL copies the data into a buffer, then the callback function copies it into the data chunk, then we convert it into a Qstring, then we copy that into the text edit box.
I have found that I can modify the callback function to put the data from CURL into a QString as it comes in, and parse out what I need on the fly. I can not however figure out how to make the data that is in the callback function QString be available outside of the function. I can't return the data as CURL calls the function, so I will need to store it in a way that can be accessed from the callback function and the rest of the Qt program.
Heres the current callback function that I am modifying:
struct MemoryStruct {
char *memory;
size_t size;
};
static size_t WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data) {
size_t realsize = size * nmemb;
struct MemoryStruct *mem = (struct MemoryStruct *)data;
//increase the size of "memory" by the size of the bytes that we have read
mem->memory = (char *)realloc(mem->memory, mem->size + realsize + 1);
if (mem->memory) {
QString qmemData = QString::fromUtf8 ((char *)ptr);
//parse out the stuff that I want
//How do I get this QString (qmemData) to be accessable outside of this function
}
return realsize;
}
How do I get the QString qmemData to be accessable outside of this function? Im sure its something simple I am missing somewhere.