mirrors
/
netsurf
mirror of https://github.com/netsurf-browser/netsurf
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rename register_curl_fetchers() to fetch_curl_register(). Add declarations for static functions and reorder functions. 

svn path=/trunk/netsurf/; revision=3379
This commit is contained in:
James Bursa 2007-07-04 18:05:16 +00:00
parent 5ef99ad521
commit 960c11de1a
3 changed files with 477 additions and 400 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

470
content/fetch.c
View File

@ -48,16 +48,16 @@ bool fetch_active; /**< Fetches in progress, please call fetch_poll(). */
/** Information about a fetcher for a given scheme. */
typedef struct scheme_fetcher_s {
char *scheme_name; /**< The scheme. */
fetcher_setup_fetch setup_fetch; /**< Set up a fetch. */
fetcher_start_fetch start_fetch; /**< Start a fetch. */
fetcher_abort_fetch abort_fetch; /**< Abort a fetch. */
fetcher_free_fetch free_fetch; /**< Free a fetch. */
fetcher_poll_fetcher poll_fetcher; /**< Poll this fetcher. */
fetcher_finalise finaliser; /**< Clean up this fetcher. */
int refcount; /**< When zero, clean up the fetcher. */
struct scheme_fetcher_s *next_fetcher; /**< Next fetcher in the list. */
struct scheme_fetcher_s *prev_fetcher; /**< Prev fetcher in the list. */
char *scheme_name; /**< The scheme. */
fetcher_setup_fetch setup_fetch; /**< Set up a fetch. */
fetcher_start_fetch start_fetch; /**< Start a fetch. */
fetcher_abort_fetch abort_fetch; /**< Abort a fetch. */
fetcher_free_fetch free_fetch; /**< Free a fetch. */
fetcher_poll_fetcher poll_fetcher; /**< Poll this fetcher. */
fetcher_finalise finaliser; /**< Clean up this fetcher. */
int refcount; /**< When zero, clean up the fetcher. */
struct scheme_fetcher_s *next_fetcher; /**< Next fetcher in the list. */
struct scheme_fetcher_s *prev_fetcher; /**< Prev fetcher in the list. */
} scheme_fetcher;
static scheme_fetcher *fetchers = NULL;
@ -67,15 +67,16 @@ struct fetch {
fetch_callback callback;/**< Callback function. */
bool abort; /**< Abort requested. */
bool stopped; /**< Download stopped on purpose. */
char *url; /**< URL. */
char *referer; /**< Referer URL. */
char *url; /**< URL. */
char *referer; /**< Referer URL. */
bool send_referer; /**< Valid to send the referer */
void *p; /**< Private data for callback. */
char *host; /**< Host part of URL. */
long http_code; /**< HTTP response code, or 0. */
scheme_fetcher *ops; /**< Fetcher operations for this fetch. NULL if not set. */
void *fetcher_handle; /**< The handle for the fetcher. */
bool fetch_is_active; /**< This fetch is active. */
scheme_fetcher *ops; /**< Fetcher operations for this fetch,
NULL if not set. */
void *fetcher_handle; /**< The handle for the fetcher. */
bool fetch_is_active; /**< This fetch is active. */
struct fetch *r_prev; /**< Previous active fetch in ::fetch_ring. */
struct fetch *r_next; /**< Next active fetch in ::fetch_ring. */
};
@ -83,65 +84,13 @@ struct fetch {
static struct fetch *fetch_ring = 0; /**< Ring of active fetches. */
static struct fetch *queue_ring = 0; /**< Ring of queued fetches */
static void fetch_free(struct fetch *f);
#define fetch_ref_fetcher(F) F->refcount++
static void fetch_unref_fetcher(scheme_fetcher *fetcher);
static void fetch_dispatch_jobs(void);
static bool fetch_choose_and_dispatch(void);
static bool fetch_dispatch_job(struct fetch *fetch);
static void fetch_free(struct fetch *f);
#define fetch_ref_fetcher(F) F->refcount++;
static void fetch_unref_fetcher(scheme_fetcher *fetcher)
{
if (--fetcher->refcount == 0) {
fetcher->finaliser(fetcher->scheme_name);
free(fetcher->scheme_name);
if (fetcher == fetchers) {
fetchers = fetcher->next_fetcher;
if (fetchers)
fetchers->prev_fetcher = NULL;
} else {
fetcher->prev_fetcher->next_fetcher = fetcher->next_fetcher;
if (fetcher->next_fetcher != NULL)
fetcher->next_fetcher->prev_fetcher = fetcher->prev_fetcher;
}
free(fetcher);
}
}
bool
fetch_add_fetcher(const char *scheme,
fetcher_initialise initialiser,
fetcher_setup_fetch setup_fetch,
fetcher_start_fetch start_fetch,
fetcher_abort_fetch abort_fetch,
fetcher_free_fetch free_fetch,
fetcher_poll_fetcher poll_fetcher,
fetcher_finalise finaliser)
{
scheme_fetcher *new_fetcher;
if (!initialiser(scheme))
return false;
new_fetcher = malloc(sizeof(scheme_fetcher));
if (new_fetcher == NULL) {
finaliser(scheme);
return false;
}
new_fetcher->scheme_name = strdup(scheme);
if (new_fetcher->scheme_name == NULL) {
free(new_fetcher);
finaliser(scheme);
return false;
}
new_fetcher->refcount = 0;
new_fetcher->setup_fetch = setup_fetch;
new_fetcher->start_fetch = start_fetch;
new_fetcher->abort_fetch = abort_fetch;
new_fetcher->free_fetch = free_fetch;
new_fetcher->poll_fetcher = poll_fetcher;
new_fetcher->finaliser = finaliser;
new_fetcher->next_fetcher = fetchers;
fetchers = new_fetcher;
fetch_ref_fetcher(new_fetcher);
return true;
}
/**
* Initialise the fetcher.
@ -151,8 +100,8 @@ fetch_add_fetcher(const char *scheme,
void fetch_init(void)
{
register_curl_fetchers();
fetch_active = false;
fetch_curl_register();
fetch_active = false;
}
@ -164,14 +113,73 @@ void fetch_init(void)
void fetch_quit(void)
{
while (fetchers != NULL) {
if (fetchers->refcount != 1) {
LOG(("Fetcher for scheme %s still active?!", fetchers->scheme_name));
/* We shouldn't do this, but... */
fetchers->refcount = 1;
}
fetch_unref_fetcher(fetchers);
}
while (fetchers != NULL) {
if (fetchers->refcount != 1) {
LOG(("Fetcher for scheme %s still active?!",
fetchers->scheme_name));
/* We shouldn't do this, but... */
fetchers->refcount = 1;
}
fetch_unref_fetcher(fetchers);
}
}
bool fetch_add_fetcher(const char *scheme,
fetcher_initialise initialiser,
fetcher_setup_fetch setup_fetch,
fetcher_start_fetch start_fetch,
fetcher_abort_fetch abort_fetch,
fetcher_free_fetch free_fetch,
fetcher_poll_fetcher poll_fetcher,
fetcher_finalise finaliser)
{
scheme_fetcher *new_fetcher;
if (!initialiser(scheme))
return false;
new_fetcher = malloc(sizeof(scheme_fetcher));
if (new_fetcher == NULL) {
finaliser(scheme);
return false;
}
new_fetcher->scheme_name = strdup(scheme);
if (new_fetcher->scheme_name == NULL) {
free(new_fetcher);
finaliser(scheme);
return false;
}
new_fetcher->refcount = 0;
new_fetcher->setup_fetch = setup_fetch;
new_fetcher->start_fetch = start_fetch;
new_fetcher->abort_fetch = abort_fetch;
new_fetcher->free_fetch = free_fetch;
new_fetcher->poll_fetcher = poll_fetcher;
new_fetcher->finaliser = finaliser;
new_fetcher->next_fetcher = fetchers;
fetchers = new_fetcher;
fetch_ref_fetcher(new_fetcher);
return true;
}
void fetch_unref_fetcher(scheme_fetcher *fetcher)
{
if (--fetcher->refcount == 0) {
fetcher->finaliser(fetcher->scheme_name);
free(fetcher->scheme_name);
if (fetcher == fetchers) {
fetchers = fetcher->next_fetcher;
if (fetchers)
fetchers->prev_fetcher = NULL;
} else {
fetcher->prev_fetcher->next_fetcher =
fetcher->next_fetcher;
if (fetcher->next_fetcher != NULL)
fetcher->next_fetcher->prev_fetcher =
fetcher->prev_fetcher;
}
free(fetcher);
}
}
@ -196,16 +204,17 @@ void fetch_quit(void)
*/
struct fetch * fetch_start(const char *url, const char *referer,
fetch_callback callback,
void *p, bool only_2xx, const char *post_urlenc,
struct form_successful_control *post_multipart,
bool verifiable, const char *parent_url, char *headers[])
fetch_callback callback,
void *p, bool only_2xx, const char *post_urlenc,
struct form_successful_control *post_multipart,
bool verifiable, const char *parent_url,
char *headers[])
{
char *host;
struct fetch *fetch;
url_func_result res;
char *ref1 = 0, *ref2 = 0;
scheme_fetcher *fetcher = fetchers;
scheme_fetcher *fetcher = fetchers;
fetch = malloc(sizeof (*fetch));
if (!fetch)
@ -252,45 +261,45 @@ struct fetch * fetch_start(const char *url, const char *referer,
fetch->http_code = 0;
fetch->r_prev = 0;
fetch->r_next = 0;
fetch->referer = 0;
fetch->ops = 0;
fetch->fetch_is_active = false;
fetch->referer = 0;
fetch->ops = 0;
fetch->fetch_is_active = false;
if (referer != NULL) {
fetch->referer = strdup(referer);
if (fetch->referer == NULL)
goto failed;
if (referer != NULL) {
fetch->referer = strdup(referer);
if (fetch->referer == NULL)
goto failed;
if (option_send_referer && ref1 && ref2 &&
strcasecmp(ref1, ref2) == 0)
fetch->send_referer = true;
}
}
if (!fetch->url)
goto failed;
/* Pick the scheme ops */
while (fetcher) {
if (strcmp(fetcher->scheme_name, ref1) == 0) {
fetch->ops = fetcher;
break;
}
fetcher = fetcher->next_fetcher;
}
/* Pick the scheme ops */
while (fetcher) {
if (strcmp(fetcher->scheme_name, ref1) == 0) {
fetch->ops = fetcher;
break;
}
fetcher = fetcher->next_fetcher;
}
if (fetch->ops == NULL)
goto failed;
if (fetch->ops == NULL)
goto failed;
/* Got a scheme fetcher, try and set up the fetch */
fetch->fetcher_handle =
fetch->ops->setup_fetch(fetch, url, only_2xx, post_urlenc,
post_multipart, verifiable, parent_url,
(const char **)headers);
/* Got a scheme fetcher, try and set up the fetch */
fetch->fetcher_handle =
fetch->ops->setup_fetch(fetch, url, only_2xx, post_urlenc,
post_multipart, verifiable, parent_url,
(const char **)headers);
if (fetch->fetcher_handle == NULL)
goto failed;
if (fetch->fetcher_handle == NULL)
goto failed;
/* Rah, got it, so ref the fetcher. */
fetch_ref_fetcher(fetch->ops);
/* Rah, got it, so ref the fetcher. */
fetch_ref_fetcher(fetch->ops);
/* these aren't needed past here */
if (ref1) {
@ -298,12 +307,12 @@ struct fetch * fetch_start(const char *url, const char *referer,
ref1 = 0;
}
if (ref2) {
free(ref2);
ref2 = 0;
}
if (ref2) {
free(ref2);
ref2 = 0;
}
/* Dump us in the queue and ask the queue to run. */
/* Dump us in the queue and ask the queue to run. */
RING_INSERT(queue_ring, fetch);
fetch_dispatch_jobs();
return fetch;
@ -313,59 +322,17 @@ failed:
if (ref1)
free(ref1);
free(fetch->url);
if (fetch->referer)
free(fetch->referer);
free(fetch);
if (fetch->referer)
free(fetch->referer);
free(fetch);
return 0;
}
/**
* Dispatch a single job
*/
static bool fetch_dispatch_job(struct fetch *fetch)
{
RING_REMOVE(queue_ring, fetch);
LOG(("Attempting to start fetch %p, fetcher %p, url %s", fetch,
fetch->fetcher_handle, fetch->url));
if (!fetch->ops->start_fetch(fetch->fetcher_handle)) {
RING_INSERT(queue_ring, fetch); /* Put it back on the end of the queue */
return false;
} else {
RING_INSERT(fetch_ring, fetch);
fetch->fetch_is_active = true;
return true;
}
}
/**
* Choose and dispatch a single job. Return false if we failed to dispatch anything.
*
* We don't check the overall dispatch size here because we're not called unless
* there is room in the fetch queue for us.
*/
static bool fetch_choose_and_dispatch(void)
{
struct fetch *queueitem;
queueitem = queue_ring;
do {
/* We can dispatch the selected item if there is room in the
* fetch ring
*/
int countbyhost;
RING_COUNTBYHOST(struct fetch, fetch_ring, countbyhost, queueitem->host);
if (countbyhost < option_max_fetchers_per_host) {
/* We can dispatch this item in theory */
return fetch_dispatch_job(queueitem);
}
queueitem = queueitem->r_next;
} while (queueitem != queue_ring);
return false;
}
/**
* Dispatch as many jobs as we have room to dispatch.
*/
static void fetch_dispatch_jobs(void)
void fetch_dispatch_jobs(void)
{
int all_active, all_queued;
@ -373,6 +340,24 @@ static void fetch_dispatch_jobs(void)
return; /* Nothing to do, the queue is empty */
RING_GETSIZE(struct fetch, queue_ring, all_queued);
RING_GETSIZE(struct fetch, fetch_ring, all_active);
LOG(("queue_ring %i, fetch_ring %i", all_queued, all_active));
struct fetch *q = queue_ring;
if (q) {
do {
LOG(("queue_ring: %s", q->url));
q = q->r_next;
} while (q != queue_ring);
}
struct fetch *f = fetch_ring;
if (f) {
do {
LOG(("fetch_ring: %s", f->url));
f = f->r_next;
} while (f != fetch_ring);
}
while ( all_queued && all_active < option_max_fetchers ) {
/*LOG(("%d queued, %d fetching", all_queued, all_active));*/
if (fetch_choose_and_dispatch()) {
@ -383,11 +368,59 @@ static void fetch_dispatch_jobs(void)
break;
}
}
fetch_active = (all_active > 0);
LOG(("Fetch ring is now %d elements.", all_active));
LOG(("Queue ring is now %d elements.", all_queued));
fetch_active = (all_active > 0);
LOG(("Fetch ring is now %d elements.", all_active));
LOG(("Queue ring is now %d elements.", all_queued));
}
/**
* Choose and dispatch a single job. Return false if we failed to dispatch
* anything.
*
* We don't check the overall dispatch size here because we're not called unless
* there is room in the fetch queue for us.
*/
bool fetch_choose_and_dispatch(void)
{
struct fetch *queueitem;
queueitem = queue_ring;
do {
/* We can dispatch the selected item if there is room in the
* fetch ring
*/
int countbyhost;
RING_COUNTBYHOST(struct fetch, fetch_ring, countbyhost,
queueitem->host);
if (countbyhost < option_max_fetchers_per_host) {
/* We can dispatch this item in theory */
return fetch_dispatch_job(queueitem);
}
queueitem = queueitem->r_next;
} while (queueitem != queue_ring);
return false;
}
/**
* Dispatch a single job
*/
bool fetch_dispatch_job(struct fetch *fetch)
{
RING_REMOVE(queue_ring, fetch);
LOG(("Attempting to start fetch %p, fetcher %p, url %s", fetch,
fetch->fetcher_handle, fetch->url));
if (!fetch->ops->start_fetch(fetch->fetcher_handle)) {
RING_INSERT(queue_ring, fetch); /* Put it back on the end of the queue */
return false;
} else {
RING_INSERT(fetch_ring, fetch);
fetch->fetch_is_active = true;
return true;
}
}
/**
* Abort a fetch.
*/
@ -396,7 +429,7 @@ void fetch_abort(struct fetch *f)
{
assert(f);
LOG(("fetch %p, fetcher %p, url '%s'", f, f->fetcher_handle, f->url));
f->ops->abort_fetch(f->fetcher_handle);
f->ops->abort_fetch(f->fetcher_handle);
}
@ -406,13 +439,13 @@ void fetch_abort(struct fetch *f)
void fetch_free(struct fetch *f)
{
LOG(("Freeing fetch %p, fetcher %p", f, f->fetcher_handle));
f->ops->free_fetch(f->fetcher_handle);
fetch_unref_fetcher(f->ops);
LOG(("Freeing fetch %p, fetcher %p", f, f->fetcher_handle));
f->ops->free_fetch(f->fetcher_handle);
fetch_unref_fetcher(f->ops);
free(f->url);
free(f->host);
if (f->referer)
free(f->referer);
if (f->referer)
free(f->referer);
free(f);
}
@ -425,14 +458,17 @@ void fetch_free(struct fetch *f)
void fetch_poll(void)
{
scheme_fetcher *fetcher = fetchers;
if (!fetch_active)
return; /* No point polling, there's no fetch active. */
while (fetcher != NULL) {
/* LOG(("Polling fetcher for %s", fetcher->scheme_name)); */
fetcher->poll_fetcher(fetcher->scheme_name);
fetcher = fetcher->next_fetcher;
}
scheme_fetcher *fetcher = fetchers;
fetch_dispatch_jobs();
if (!fetch_active)
return; /* No point polling, there's no fetch active. */
while (fetcher != NULL) {
/* LOG(("Polling fetcher for %s", fetcher->scheme_name)); */
fetcher->poll_fetcher(fetcher->scheme_name);
fetcher = fetcher->next_fetcher;
}
}
@ -447,18 +483,18 @@ bool fetch_can_fetch(const char *url)
{
const char *semi;
size_t len;
scheme_fetcher *fetcher = fetchers;
scheme_fetcher *fetcher = fetchers;
if ((semi = strchr(url, ':')) == NULL)
return false;
len = semi - url;
while (fetcher != NULL) {
if (strlen(fetcher->scheme_name) == len &&
strncmp(fetcher->scheme_name, url, len) == 0)
return true;
fetcher = fetcher->next_fetcher;
}
while (fetcher != NULL) {
if (strlen(fetcher->scheme_name) == len &&
strncmp(fetcher->scheme_name, url, len) == 0)
return true;
fetcher = fetcher->next_fetcher;
}
return false;
}
@ -469,8 +505,8 @@ bool fetch_can_fetch(const char *url)
*/
void fetch_change_callback(struct fetch *fetch,
fetch_callback callback,
void *p)
fetch_callback callback,
void *p)
{
assert(fetch);
fetch->callback = callback;
@ -503,49 +539,47 @@ void
fetch_send_callback(fetch_msg msg, struct fetch *fetch, const void *data,
unsigned long size)
{
LOG(("Fetcher sending callback. Fetch %p, fetcher %p data %p size %lu",
fetch, fetch->fetcher_handle, data, size));
fetch->callback(msg, fetch->p, data, size);
LOG(("Fetcher sending callback. Fetch %p, fetcher %p data %p size %lu",
fetch, fetch->fetcher_handle, data, size));
fetch->callback(msg, fetch->p, data, size);
}
void
fetch_can_be_freed(struct fetch *fetch)
{
int all_active, all_queued;
/* Go ahead and free the fetch properly now */
LOG(("Fetch %p, fetcher %p can be freed", fetch, fetch->fetcher_handle));
if (fetch->fetch_is_active) {
RING_REMOVE(fetch_ring, fetch);
} else {
RING_REMOVE(queue_ring, fetch);
}
int all_active, all_queued;
/* Go ahead and free the fetch properly now */
LOG(("Fetch %p, fetcher %p can be freed", fetch, fetch->fetcher_handle));
if (fetch->fetch_is_active) {
RING_REMOVE(fetch_ring, fetch);
} else {
RING_REMOVE(queue_ring, fetch);
}
RING_GETSIZE(struct fetch, fetch_ring, all_active);
RING_GETSIZE(struct fetch, queue_ring, all_queued);
fetch_active = (all_active > 0);
LOG(("Fetch ring is now %d elements.", all_active));
LOG(("Queue ring is now %d elements.", all_queued));
fetch_free(fetch);
fetch_dispatch_jobs();
fetch_active = (all_active > 0);
LOG(("Fetch ring is now %d elements.", all_active));
LOG(("Queue ring is now %d elements.", all_queued));
fetch_free(fetch);
}
void
fetch_set_http_code(struct fetch *fetch, long http_code)
{
LOG(("Setting HTTP code to %ld", http_code));
fetch->http_code = http_code;
LOG(("Setting HTTP code to %ld", http_code));
fetch->http_code = http_code;
}
const char *
fetch_get_referer_to_send(struct fetch *fetch)
{
if (fetch->send_referer)
return fetch->referer;
return NULL;
if (fetch->send_referer)
return fetch->referer;
return NULL;
}

405
content/fetchers/fetch_curl.c
View File

@ -57,15 +57,15 @@ struct cert_info {
/** Information for a single fetch. */
struct curl_fetch_info {
struct fetch *fetch_handle; /**< The fetch handle we're parented by. */
struct fetch *fetch_handle; /**< The fetch handle we're parented by. */
CURL * curl_handle; /**< cURL handle if being fetched, or 0. */
bool had_headers; /**< Headers have been processed. */
bool abort; /**< Abort requested. */
bool stopped; /**< Download stopped on purpose. */
bool only_2xx; /**< Only HTTP 2xx responses acceptable. */
bool verifiable; /**< Transaction is verifiable */
char *url; /**< URL of this fetch. */
char *host; /**< The hostname of this fetch. */
char *url; /**< URL of this fetch. */
char *host; /**< The hostname of this fetch. */
char *parent_fetch_url; /**< URL of parent fetch (not necessarily
* the same as the referer) */
struct curl_slist *headers; /**< List of request headers. */
@ -74,7 +74,7 @@ struct curl_fetch_info {
char *cookie_string; /**< Cookie string for this fetch */
char *realm; /**< HTTP Auth Realm */
char *post_urlenc; /**< Url encoded POST string, or 0. */
unsigned long http_code; /**< HTTP result code from cURL. */
unsigned long http_code; /**< HTTP result code from cURL. */
struct curl_httppost *post_multipart; /**< Multipart post data, or 0. */
struct cache_data cachedata; /**< Cache control data */
time_t last_modified; /**< If-Modified-Since time */
@ -97,20 +97,36 @@ CURLM *fetch_curl_multi; /**< Global cURL multi handle. */
/** Curl handle with default options set; not used for transfers. */
static CURL *fetch_blank_curl;
static struct cache_handle *curl_handle_ring = 0; /**< Ring of cached handles */
static int curl_fetchers_registered = 0;
static char fetch_error_buffer[CURL_ERROR_SIZE]; /**< Error buffer for cURL. */
static char fetch_progress_buffer[256]; /**< Progress buffer for cURL */
static char fetch_proxy_userpwd[100]; /**< Proxy authentication details. */
static bool fetch_curl_initialise(const char *scheme);
static void fetch_curl_finalise(const char *scheme);
static void * fetch_curl_setup(struct fetch *parent_fetch, const char *url,
bool only_2xx, const char *post_urlenc,
struct form_successful_control *post_multipart,
bool verifiable, const char *parent_url, const char **headers);
static bool fetch_curl_start(void *vfetch);
static bool fetch_curl_initiate_fetch(struct curl_fetch_info *fetch,
CURL *handle);
static CURL *fetch_curl_get_handle(char *host);
static void fetch_curl_cache_handle(CURL *handle, char *hostname);
static CURLcode fetch_curl_set_options(struct curl_fetch_info *f);
#ifdef WITH_SSL
static CURLcode fetch_curl_sslctxfun(CURL *curl_handle, SSL_CTX *sslctx, void *p);
static CURLcode fetch_curl_sslctxfun(CURL *curl_handle, SSL_CTX *sslctx,
void *p);
#endif
static void fetch_curl_free(void *f);
static void fetch_curl_abort(void *vf);
static void fetch_curl_stop(struct curl_fetch_info *f);
static void fetch_curl_free(void *f);
static void fetch_curl_poll(const char *scheme_ignored);
static void fetch_curl_done(CURL *curl_handle, CURLcode result);
static int fetch_curl_progress(void *clientp, double dltotal, double dlnow,
double ultotal, double ulnow);
static int fetch_curl_ignore(void);
static size_t fetch_curl_data(void *data, size_t size, size_t nmemb,
struct curl_fetch_info *f);
static size_t fetch_curl_header(char *data, size_t size, size_t nmemb,
@ -119,11 +135,133 @@ static bool fetch_curl_process_headers(struct curl_fetch_info *f);
static struct curl_httppost *fetch_curl_post_convert(
struct form_successful_control *control);
#ifdef WITH_SSL
static int fetch_curl_verify_callback(int preverify_ok, X509_STORE_CTX *x509_ctx);
static int fetch_curl_cert_verify_callback(X509_STORE_CTX *x509_ctx, void *parm);
static int fetch_curl_verify_callback(int preverify_ok,
X509_STORE_CTX *x509_ctx);
static int fetch_curl_cert_verify_callback(X509_STORE_CTX *x509_ctx,
void *parm);
#endif
static void fetch_curl_cache_handle(CURL *handle, char *hostname);
/**
* Initialise the fetcher.
*
* Must be called once before any other function.
*/
void fetch_curl_register(void)
{
CURLcode code;
curl_version_info_data *data;
int i;
LOG(("curl_version %s", curl_version()));
code = curl_global_init(CURL_GLOBAL_ALL);
if (code != CURLE_OK)
die("Failed to initialise the fetch module "
"(curl_global_init failed).");
fetch_curl_multi = curl_multi_init();
if (!fetch_curl_multi)
die("Failed to initialise the fetch module "
"(curl_multi_init failed).");
/* Create a curl easy handle with the options that are common to all
fetches. */
fetch_blank_curl = curl_easy_init();
if (!fetch_blank_curl)
die("Failed to initialise the fetch module "
"(curl_easy_init failed).");
#undef SETOPT
#define SETOPT(option, value) \
code = curl_easy_setopt(fetch_blank_curl, option, value); \
if (code != CURLE_OK) \
goto curl_easy_setopt_failed;
if (verbose_log) {
SETOPT(CURLOPT_VERBOSE, 1);
} else {
SETOPT(CURLOPT_VERBOSE, 0);
}
SETOPT(CURLOPT_ERRORBUFFER, fetch_error_buffer);
if (option_suppress_curl_debug)
SETOPT(CURLOPT_DEBUGFUNCTION, fetch_curl_ignore);
SETOPT(CURLOPT_WRITEFUNCTION, fetch_curl_data);
SETOPT(CURLOPT_HEADERFUNCTION, fetch_curl_header);
SETOPT(CURLOPT_PROGRESSFUNCTION, fetch_curl_progress);
SETOPT(CURLOPT_NOPROGRESS, 0);
SETOPT(CURLOPT_USERAGENT, user_agent_string());
SETOPT(CURLOPT_ENCODING, "gzip");
SETOPT(CURLOPT_LOW_SPEED_LIMIT, 1L);
SETOPT(CURLOPT_LOW_SPEED_TIME, 180L);
SETOPT(CURLOPT_NOSIGNAL, 1L);
SETOPT(CURLOPT_CONNECTTIMEOUT, 30L);
if (option_ca_bundle && strcmp(option_ca_bundle, ""))
SETOPT(CURLOPT_CAINFO, option_ca_bundle);
if (option_ca_path && strcmp(option_ca_path, ""))
SETOPT(CURLOPT_CAPATH, option_ca_path);
/* cURL initialised okay, register the fetchers */
data = curl_version_info(CURLVERSION_NOW);
for (i = 0; data->protocols[i]; i++)
if (!fetch_add_fetcher(data->protocols[i],
fetch_curl_initialise,
fetch_curl_setup,
fetch_curl_start,
fetch_curl_abort,
fetch_curl_free,
fetch_curl_poll,
fetch_curl_finalise)) {
LOG(("Unable to register cURL fetcher for %s",
data->protocols[i]));
}
return;
curl_easy_setopt_failed:
die("Failed to initialise the fetch module "
"(curl_easy_setopt failed).");
}
/**
* Initialise a cURL fetcher.
*/
bool fetch_curl_initialise(const char *scheme)
{
LOG(("Initialise cURL fetcher for %s", scheme));
curl_fetchers_registered++;
return true; /* Always succeeds */
}
/**
* Finalise a cURL fetcher
*/
void fetch_curl_finalise(const char *scheme)
{
curl_fetchers_registered--;
LOG(("Finalise cURL fetcher %s", scheme));
if (curl_fetchers_registered == 0) {
/* All the fetchers have been finalised. */
LOG(("All cURL fetchers finalised, closing down cURL"));
CURLMcode codem;
curl_easy_cleanup(fetch_blank_curl);
codem = curl_multi_cleanup(fetch_curl_multi);
if (codem != CURLM_OK)
LOG(("curl_multi_cleanup failed: ignoring"));
curl_global_cleanup();
}
}
/**
* Start fetching data for the given URL.
@ -147,11 +285,10 @@ static void fetch_curl_cache_handle(CURL *handle, char *hostname);
* callbacks will contain this.
*/
static void *
fetch_curl_setup(struct fetch *parent_fetch, const char *url,
bool only_2xx, const char *post_urlenc,
struct form_successful_control *post_multipart,
bool verifiable, const char *parent_url, const char **headers)
void * fetch_curl_setup(struct fetch *parent_fetch, const char *url,
bool only_2xx, const char *post_urlenc,
struct form_successful_control *post_multipart,
bool verifiable, const char *parent_url, const char **headers)
{
char *host;
struct curl_fetch_info *fetch;
@ -163,7 +300,7 @@ fetch_curl_setup(struct fetch *parent_fetch, const char *url,
if (!fetch)
return 0;
fetch->fetch_handle = parent_fetch;
fetch->fetch_handle = parent_fetch;
res = url_host(url, &host);
if (res != URL_FUNC_OK) {
@ -176,7 +313,7 @@ fetch_curl_setup(struct fetch *parent_fetch, const char *url,
goto failed;
}
LOG(("fetch %p, url '%s'", fetch, url));
LOG(("fetch %p, url '%s'", fetch, url));
/* construct a new fetch structure */
fetch->curl_handle = 0;
@ -211,15 +348,15 @@ fetch_curl_setup(struct fetch *parent_fetch, const char *url,
fetch->cachedata.last_modified = 0;
fetch->last_modified = 0;
fetch->file_etag = 0;
fetch->http_code = 0;
fetch->http_code = 0;
#ifdef WITH_SSL
memset(fetch->cert_data, 0, sizeof(fetch->cert_data));
#endif
if (!fetch->url ||
(parent_url && !fetch->parent_fetch_url) ||
(post_urlenc && !fetch->post_urlenc) ||
(post_multipart && !fetch->post_multipart))
(parent_url && !fetch->parent_fetch_url) ||
(post_urlenc && !fetch->post_urlenc) ||
(post_multipart && !fetch->post_multipart))
goto failed;
#define APPEND(list, value) \
@ -284,14 +421,28 @@ failed:
return 0;
}
/**
* Dispatch a single job
*/
bool fetch_curl_start(void *vfetch)
{
struct curl_fetch_info *fetch = (struct curl_fetch_info*)vfetch;
return fetch_curl_initiate_fetch(fetch,
fetch_curl_get_handle(fetch->host));
}
/**
* Initiate a fetch from the queue.
*
* Called with a fetch structure and a CURL handle to be used to fetch the content.
* Called with a fetch structure and a CURL handle to be used to fetch the
* content.
*
* This will return whether or not the fetch was successfully initiated.
*/
static bool fetch_curl_initiate_fetch(struct curl_fetch_info *fetch, CURL *handle)
bool fetch_curl_initiate_fetch(struct curl_fetch_info *fetch, CURL *handle)
{
CURLcode code;
CURLMcode codem;
@ -312,10 +463,12 @@ static bool fetch_curl_initiate_fetch(struct curl_fetch_info *fetch, CURL *handl
return true;
}
/**
* Find a CURL handle to use to dispatch a job
*/
static CURL *fetch_curl_get_handle(char *host)
CURL *fetch_curl_get_handle(char *host)
{
struct cache_handle *h;
CURL *ret;
@ -331,20 +484,12 @@ static CURL *fetch_curl_get_handle(char *host)
return ret;
}
/**
* Dispatch a single job
*/
static bool fetch_curl_start(void *vfetch)
{
struct curl_fetch_info *fetch = (struct curl_fetch_info*)vfetch;
return fetch_curl_initiate_fetch(fetch, fetch_curl_get_handle(fetch->host));
}
/**
* Cache a CURL handle for the provided host (if wanted)
*
*/
static void fetch_curl_cache_handle(CURL *handle, char *host)
void fetch_curl_cache_handle(CURL *handle, char *host)
{
struct cache_handle *h = 0;
int c;
@ -377,11 +522,12 @@ static void fetch_curl_cache_handle(CURL *handle, char *host)
RING_INSERT(curl_handle_ring, h);
}
/**
* Set options specific for a fetch.
*/
static CURLcode
CURLcode
fetch_curl_set_options(struct curl_fetch_info *f)
{
CURLcode code;
@ -475,12 +621,12 @@ fetch_curl_set_options(struct curl_fetch_info *f)
* cURL SSL setup callback
*/
static CURLcode
CURLcode
fetch_curl_sslctxfun(CURL *curl_handle, SSL_CTX *sslctx, void *parm)
{
SSL_CTX_set_verify(sslctx, SSL_VERIFY_PEER, fetch_curl_verify_callback);
SSL_CTX_set_cert_verify_callback(sslctx, fetch_curl_cert_verify_callback,
parm);
parm);
return CURLE_OK;
}
#endif
@ -490,15 +636,15 @@ fetch_curl_sslctxfun(CURL *curl_handle, SSL_CTX *sslctx, void *parm)
* Abort a fetch.
*/
static void fetch_curl_abort(void *vf)
void fetch_curl_abort(void *vf)
{
struct curl_fetch_info *f = (struct curl_fetch_info *)vf;
struct curl_fetch_info *f = (struct curl_fetch_info *)vf;
assert(f);
LOG(("fetch %p, url '%s'", f, f->url));
if (f->curl_handle) {
f->abort = true;
} else {
fetch_can_be_freed(f->fetch_handle);
fetch_can_be_freed(f->fetch_handle);
}
}
@ -509,7 +655,7 @@ static void fetch_curl_abort(void *vf)
* Will prod the queue afterwards to allow pending requests to be initiated.
*/
static void fetch_curl_stop(struct curl_fetch_info *f)
void fetch_curl_stop(struct curl_fetch_info *f)
{
CURLMcode codem;
@ -534,9 +680,9 @@ static void fetch_curl_stop(struct curl_fetch_info *f)
* Free a fetch structure and associated resources.
*/
static void fetch_curl_free(void *vf)
void fetch_curl_free(void *vf)
{
struct curl_fetch_info *f = (struct curl_fetch_info *)vf;
struct curl_fetch_info *f = (struct curl_fetch_info *)vf;
#ifdef WITH_SSL
int i;
#endif
@ -574,7 +720,7 @@ static void fetch_curl_free(void *vf)
* Must be called regularly to make progress on fetches.
*/
static void fetch_curl_poll(const char *scheme_ignored)
void fetch_curl_poll(const char *scheme_ignored)
{
int running, queue;
CURLMcode codem;
@ -608,7 +754,7 @@ static void fetch_curl_poll(const char *scheme_ignored)
* \param curl_handle curl easy handle of fetch
*/
static void fetch_curl_done(CURL *curl_handle, CURLcode result)
void fetch_curl_done(CURL *curl_handle, CURLcode result)
{
bool finished = false;
bool error = false;
@ -629,6 +775,7 @@ static void fetch_curl_done(CURL *curl_handle, CURLcode result)
assert(code == CURLE_OK);
abort = f->abort;
LOG(("done %s", f->url));
if (!abort && result == CURLE_OK) {
/* fetch completed normally */
@ -667,11 +814,11 @@ static void fetch_curl_done(CURL *curl_handle, CURLcode result)
f->cachedata.etag = 0;
}
/* postponed until after stop so that queue fetches are started */
if (abort)
; /* fetch was aborted: no callback */
else if (finished) {
fetch_send_callback(FETCH_FINISHED, f->fetch_handle, &cachedata, 0);
fetch_send_callback(FETCH_FINISHED, f->fetch_handle,
&cachedata, 0);
free(cachedata.etag);
}
#ifdef WITH_SSL
@ -752,14 +899,16 @@ static void fetch_curl_done(CURL *curl_handle, CURLcode result)
X509_free(certs[i].cert);
}
fetch_send_callback(FETCH_CERT_ERR, f->fetch_handle, &ssl_certs, i);
fetch_send_callback(FETCH_CERT_ERR, f->fetch_handle,
&ssl_certs, i);
}
#endif
else if (error)
fetch_send_callback(FETCH_ERROR, f->fetch_handle, fetch_error_buffer, 0);
fetch_send_callback(FETCH_ERROR, f->fetch_handle,
fetch_error_buffer, 0);
fetch_curl_stop(f);
fetch_curl_stop(f);
}
@ -768,7 +917,7 @@ static void fetch_curl_done(CURL *curl_handle, CURLcode result)
*/
int fetch_curl_progress(void *clientp, double dltotal, double dlnow,
double ultotal, double ulnow)
double ultotal, double ulnow)
{
struct curl_fetch_info *f = (struct curl_fetch_info *) clientp;
double percent;
@ -783,26 +932,39 @@ int fetch_curl_progress(void *clientp, double dltotal, double dlnow,
human_friendly_bytesize(dlnow),
human_friendly_bytesize(dltotal));
fetch_send_callback(FETCH_PROGRESS, f->fetch_handle,
fetch_progress_buffer,
(unsigned long) percent);
fetch_progress_buffer,
(unsigned long) percent);
} else {
snprintf(fetch_progress_buffer, 255,
messages_get("ProgressU"),
human_friendly_bytesize(dlnow));
fetch_send_callback(FETCH_PROGRESS, f->fetch_handle,
fetch_progress_buffer, 0);
fetch_progress_buffer, 0);
}
return 0;
}
/**
* Ignore everything given to it.
*
* Used to ignore cURL debug.
*/
int fetch_curl_ignore(void)
{
return 0;
}
/**
* Callback function for cURL.
*/
size_t fetch_curl_data(void *data, size_t size, size_t nmemb,
struct curl_fetch_info *f)
struct curl_fetch_info *f)
{
CURLcode code;
@ -810,8 +972,8 @@ size_t fetch_curl_data(void *data, size_t size, size_t nmemb,
if (!f->http_code)
{
code = curl_easy_getinfo(f->curl_handle, CURLINFO_HTTP_CODE,
&f->http_code);
fetch_set_http_code(f->fetch_handle, f->http_code);
&f->http_code);
fetch_set_http_code(f->fetch_handle, f->http_code);
assert(code == CURLE_OK);
}
@ -848,7 +1010,7 @@ size_t fetch_curl_data(void *data, size_t size, size_t nmemb,
*/
size_t fetch_curl_header(char *data, size_t size, size_t nmemb,
struct curl_fetch_info *f)
struct curl_fetch_info *f)
{
int i;
size *= nmemb;
@ -1024,8 +1186,8 @@ bool fetch_curl_process_headers(struct curl_fetch_info *f)
if (!f->http_code)
{
code = curl_easy_getinfo(f->curl_handle, CURLINFO_HTTP_CODE,
&f->http_code);
fetch_set_http_code(f->fetch_handle, f->http_code);
&f->http_code);
fetch_set_http_code(f->fetch_handle, f->http_code);
assert(code == CURLE_OK);
}
http_code = f->http_code;
@ -1034,7 +1196,7 @@ bool fetch_curl_process_headers(struct curl_fetch_info *f)
if (http_code == 304 && !f->post_urlenc && !f->post_multipart) {
/* Not Modified && GET request */
fetch_send_callback(FETCH_NOTMODIFIED, f->fetch_handle,
(const char *)&f->cachedata, 0);
(const char *)&f->cachedata, 0);
return true;
}
@ -1057,7 +1219,7 @@ bool fetch_curl_process_headers(struct curl_fetch_info *f)
if (f->only_2xx && strncmp(f->url, "http", 4) == 0 &&
(http_code < 200 || 299 < http_code)) {
fetch_send_callback(FETCH_ERROR, f->fetch_handle,
messages_get("Not2xx"), 0);
messages_get("Not2xx"), 0);
return true;
}
@ -1086,7 +1248,7 @@ bool fetch_curl_process_headers(struct curl_fetch_info *f)
f->last_modified > s.st_mtime &&
f->file_etag == s.st_mtime) {
fetch_send_callback(FETCH_NOTMODIFIED, f->fetch_handle,
(const char *)&f->cachedata, 0);
(const char *)&f->cachedata, 0);
curl_free(url_path);
return true;
}
@ -1257,122 +1419,3 @@ int fetch_curl_cert_verify_callback(X509_STORE_CTX *x509_ctx, void *parm)
}
#endif
static int curl_fetchers_registered = 0;
/** Initialise a cURL fetcher */
static bool
fetch_curl_initialise(const char *scheme)
{
LOG(("Initialise cURL fetcher for %s", scheme));
curl_fetchers_registered++;
return true; /* Always succeeds */
}
/** Finalise a cURL fetcher */
static void
fetch_curl_finalise(const char *scheme)
{
curl_fetchers_registered--;
LOG(("Finalise cURL fetcher %s", scheme));
if (curl_fetchers_registered == 0) {
/* All the fetchers have been finalised. */
LOG(("All cURL fetchers finalised, closing down cURL"));
CURLMcode codem;
curl_easy_cleanup(fetch_blank_curl);
codem = curl_multi_cleanup(fetch_curl_multi);
if (codem != CURLM_OK)
LOG(("curl_multi_cleanup failed: ignoring"));
curl_global_cleanup();
}
}
/** Ignore everything given to it.
*
* Used to ignore cURL debug.
*/
int fetch_curl_ignore(void) { return 0; }
/**
* Initialise the fetcher.
*
* Must be called once before any other function.
*/
void register_curl_fetchers(void)
{
CURLcode code;
curl_version_info_data *data;
int i;
LOG(("curl_version %s", curl_version()));
code = curl_global_init(CURL_GLOBAL_ALL);
if (code != CURLE_OK)
die("Failed to initialise the fetch module "
"(curl_global_init failed).");
fetch_curl_multi = curl_multi_init();
if (!fetch_curl_multi)
die("Failed to initialise the fetch module "
"(curl_multi_init failed).");
/* Create a curl easy handle with the options that are common to all
fetches. */
fetch_blank_curl = curl_easy_init();
if (!fetch_blank_curl)
die("Failed to initialise the fetch module "
"(curl_easy_init failed).");
#undef SETOPT
#define SETOPT(option, value) \
code = curl_easy_setopt(fetch_blank_curl, option, value); \
if (code != CURLE_OK) \
goto curl_easy_setopt_failed;
if (verbose_log) {
SETOPT(CURLOPT_VERBOSE, 1);
} else {
SETOPT(CURLOPT_VERBOSE, 0);
}
SETOPT(CURLOPT_ERRORBUFFER, fetch_error_buffer);
if (option_suppress_curl_debug)
SETOPT(CURLOPT_DEBUGFUNCTION, fetch_curl_ignore);
SETOPT(CURLOPT_WRITEFUNCTION, fetch_curl_data);
SETOPT(CURLOPT_HEADERFUNCTION, fetch_curl_header);
SETOPT(CURLOPT_PROGRESSFUNCTION, fetch_curl_progress);
SETOPT(CURLOPT_NOPROGRESS, 0);
SETOPT(CURLOPT_USERAGENT, user_agent_string());
SETOPT(CURLOPT_ENCODING, "gzip");
SETOPT(CURLOPT_LOW_SPEED_LIMIT, 1L);
SETOPT(CURLOPT_LOW_SPEED_TIME, 180L);
SETOPT(CURLOPT_NOSIGNAL, 1L);
SETOPT(CURLOPT_CONNECTTIMEOUT, 30L);
if (option_ca_bundle && strcmp(option_ca_bundle, ""))
SETOPT(CURLOPT_CAINFO, option_ca_bundle);
if (option_ca_path && strcmp(option_ca_path, ""))
SETOPT(CURLOPT_CAPATH, option_ca_path);
/* cURL initialised okay, register the fetchers */
data = curl_version_info(CURLVERSION_NOW);
for (i = 0; data->protocols[i]; i++)
if (!fetch_add_fetcher(data->protocols[i],
fetch_curl_initialise,
fetch_curl_setup,
fetch_curl_start,
fetch_curl_abort,
fetch_curl_free,
fetch_curl_poll,
fetch_curl_finalise)) {
LOG(("Unable to register cURL fetcher for %s", data->protocols[i]));
}
return;
curl_easy_setopt_failed:
die("Failed to initialise the fetch module "
"(curl_easy_setopt failed).");
}

2
content/fetchers/fetch_curl.h
View File

@ -14,7 +14,7 @@
#include <curl/curl.h>
void register_curl_fetchers(void);
void fetch_curl_register(void);
/** Global cURL multi handle. */
extern CURLM *fetch_curl_multi;