diff --git a/content/fetch.c b/content/fetch.c index 2c3f63700..b69f50be1 100644 --- a/content/fetch.c +++ b/content/fetch.c @@ -48,16 +48,16 @@ bool fetch_active; /**< Fetches in progress, please call fetch_poll(). */ /** Information about a fetcher for a given scheme. */ typedef struct scheme_fetcher_s { - char *scheme_name; /**< The scheme. */ - fetcher_setup_fetch setup_fetch; /**< Set up a fetch. */ - fetcher_start_fetch start_fetch; /**< Start a fetch. */ - fetcher_abort_fetch abort_fetch; /**< Abort a fetch. */ - fetcher_free_fetch free_fetch; /**< Free a fetch. */ - fetcher_poll_fetcher poll_fetcher; /**< Poll this fetcher. */ - fetcher_finalise finaliser; /**< Clean up this fetcher. */ - int refcount; /**< When zero, clean up the fetcher. */ - struct scheme_fetcher_s *next_fetcher; /**< Next fetcher in the list. */ - struct scheme_fetcher_s *prev_fetcher; /**< Prev fetcher in the list. */ + char *scheme_name; /**< The scheme. */ + fetcher_setup_fetch setup_fetch; /**< Set up a fetch. */ + fetcher_start_fetch start_fetch; /**< Start a fetch. */ + fetcher_abort_fetch abort_fetch; /**< Abort a fetch. */ + fetcher_free_fetch free_fetch; /**< Free a fetch. */ + fetcher_poll_fetcher poll_fetcher; /**< Poll this fetcher. */ + fetcher_finalise finaliser; /**< Clean up this fetcher. */ + int refcount; /**< When zero, clean up the fetcher. */ + struct scheme_fetcher_s *next_fetcher; /**< Next fetcher in the list. */ + struct scheme_fetcher_s *prev_fetcher; /**< Prev fetcher in the list. */ } scheme_fetcher; static scheme_fetcher *fetchers = NULL; @@ -67,15 +67,16 @@ struct fetch { fetch_callback callback;/**< Callback function. */ bool abort; /**< Abort requested. */ bool stopped; /**< Download stopped on purpose. */ - char *url; /**< URL. */ - char *referer; /**< Referer URL. */ + char *url; /**< URL. */ + char *referer; /**< Referer URL. */ bool send_referer; /**< Valid to send the referer */ void *p; /**< Private data for callback. */ char *host; /**< Host part of URL. */ long http_code; /**< HTTP response code, or 0. */ - scheme_fetcher *ops; /**< Fetcher operations for this fetch. NULL if not set. */ - void *fetcher_handle; /**< The handle for the fetcher. */ - bool fetch_is_active; /**< This fetch is active. */ + scheme_fetcher *ops; /**< Fetcher operations for this fetch, + NULL if not set. */ + void *fetcher_handle; /**< The handle for the fetcher. */ + bool fetch_is_active; /**< This fetch is active. */ struct fetch *r_prev; /**< Previous active fetch in ::fetch_ring. */ struct fetch *r_next; /**< Next active fetch in ::fetch_ring. */ }; @@ -83,65 +84,13 @@ struct fetch { static struct fetch *fetch_ring = 0; /**< Ring of active fetches. */ static struct fetch *queue_ring = 0; /**< Ring of queued fetches */ -static void fetch_free(struct fetch *f); +#define fetch_ref_fetcher(F) F->refcount++ +static void fetch_unref_fetcher(scheme_fetcher *fetcher); static void fetch_dispatch_jobs(void); +static bool fetch_choose_and_dispatch(void); +static bool fetch_dispatch_job(struct fetch *fetch); +static void fetch_free(struct fetch *f); -#define fetch_ref_fetcher(F) F->refcount++; - -static void fetch_unref_fetcher(scheme_fetcher *fetcher) -{ - if (--fetcher->refcount == 0) { - fetcher->finaliser(fetcher->scheme_name); - free(fetcher->scheme_name); - if (fetcher == fetchers) { - fetchers = fetcher->next_fetcher; - if (fetchers) - fetchers->prev_fetcher = NULL; - } else { - fetcher->prev_fetcher->next_fetcher = fetcher->next_fetcher; - if (fetcher->next_fetcher != NULL) - fetcher->next_fetcher->prev_fetcher = fetcher->prev_fetcher; - } - free(fetcher); - } -} - -bool -fetch_add_fetcher(const char *scheme, - fetcher_initialise initialiser, - fetcher_setup_fetch setup_fetch, - fetcher_start_fetch start_fetch, - fetcher_abort_fetch abort_fetch, - fetcher_free_fetch free_fetch, - fetcher_poll_fetcher poll_fetcher, - fetcher_finalise finaliser) -{ - scheme_fetcher *new_fetcher; - if (!initialiser(scheme)) - return false; - new_fetcher = malloc(sizeof(scheme_fetcher)); - if (new_fetcher == NULL) { - finaliser(scheme); - return false; - } - new_fetcher->scheme_name = strdup(scheme); - if (new_fetcher->scheme_name == NULL) { - free(new_fetcher); - finaliser(scheme); - return false; - } - new_fetcher->refcount = 0; - new_fetcher->setup_fetch = setup_fetch; - new_fetcher->start_fetch = start_fetch; - new_fetcher->abort_fetch = abort_fetch; - new_fetcher->free_fetch = free_fetch; - new_fetcher->poll_fetcher = poll_fetcher; - new_fetcher->finaliser = finaliser; - new_fetcher->next_fetcher = fetchers; - fetchers = new_fetcher; - fetch_ref_fetcher(new_fetcher); - return true; -} /** * Initialise the fetcher. @@ -151,8 +100,8 @@ fetch_add_fetcher(const char *scheme, void fetch_init(void) { - register_curl_fetchers(); - fetch_active = false; + fetch_curl_register(); + fetch_active = false; } @@ -164,14 +113,73 @@ void fetch_init(void) void fetch_quit(void) { - while (fetchers != NULL) { - if (fetchers->refcount != 1) { - LOG(("Fetcher for scheme %s still active?!", fetchers->scheme_name)); - /* We shouldn't do this, but... */ - fetchers->refcount = 1; - } - fetch_unref_fetcher(fetchers); - } + while (fetchers != NULL) { + if (fetchers->refcount != 1) { + LOG(("Fetcher for scheme %s still active?!", + fetchers->scheme_name)); + /* We shouldn't do this, but... */ + fetchers->refcount = 1; + } + fetch_unref_fetcher(fetchers); + } +} + + +bool fetch_add_fetcher(const char *scheme, + fetcher_initialise initialiser, + fetcher_setup_fetch setup_fetch, + fetcher_start_fetch start_fetch, + fetcher_abort_fetch abort_fetch, + fetcher_free_fetch free_fetch, + fetcher_poll_fetcher poll_fetcher, + fetcher_finalise finaliser) +{ + scheme_fetcher *new_fetcher; + if (!initialiser(scheme)) + return false; + new_fetcher = malloc(sizeof(scheme_fetcher)); + if (new_fetcher == NULL) { + finaliser(scheme); + return false; + } + new_fetcher->scheme_name = strdup(scheme); + if (new_fetcher->scheme_name == NULL) { + free(new_fetcher); + finaliser(scheme); + return false; + } + new_fetcher->refcount = 0; + new_fetcher->setup_fetch = setup_fetch; + new_fetcher->start_fetch = start_fetch; + new_fetcher->abort_fetch = abort_fetch; + new_fetcher->free_fetch = free_fetch; + new_fetcher->poll_fetcher = poll_fetcher; + new_fetcher->finaliser = finaliser; + new_fetcher->next_fetcher = fetchers; + fetchers = new_fetcher; + fetch_ref_fetcher(new_fetcher); + return true; +} + + +void fetch_unref_fetcher(scheme_fetcher *fetcher) +{ + if (--fetcher->refcount == 0) { + fetcher->finaliser(fetcher->scheme_name); + free(fetcher->scheme_name); + if (fetcher == fetchers) { + fetchers = fetcher->next_fetcher; + if (fetchers) + fetchers->prev_fetcher = NULL; + } else { + fetcher->prev_fetcher->next_fetcher = + fetcher->next_fetcher; + if (fetcher->next_fetcher != NULL) + fetcher->next_fetcher->prev_fetcher = + fetcher->prev_fetcher; + } + free(fetcher); + } } @@ -196,16 +204,17 @@ void fetch_quit(void) */ struct fetch * fetch_start(const char *url, const char *referer, - fetch_callback callback, - void *p, bool only_2xx, const char *post_urlenc, - struct form_successful_control *post_multipart, - bool verifiable, const char *parent_url, char *headers[]) + fetch_callback callback, + void *p, bool only_2xx, const char *post_urlenc, + struct form_successful_control *post_multipart, + bool verifiable, const char *parent_url, + char *headers[]) { char *host; struct fetch *fetch; url_func_result res; char *ref1 = 0, *ref2 = 0; - scheme_fetcher *fetcher = fetchers; + scheme_fetcher *fetcher = fetchers; fetch = malloc(sizeof (*fetch)); if (!fetch) @@ -252,45 +261,45 @@ struct fetch * fetch_start(const char *url, const char *referer, fetch->http_code = 0; fetch->r_prev = 0; fetch->r_next = 0; - fetch->referer = 0; - fetch->ops = 0; - fetch->fetch_is_active = false; + fetch->referer = 0; + fetch->ops = 0; + fetch->fetch_is_active = false; - if (referer != NULL) { - fetch->referer = strdup(referer); - if (fetch->referer == NULL) - goto failed; + if (referer != NULL) { + fetch->referer = strdup(referer); + if (fetch->referer == NULL) + goto failed; if (option_send_referer && ref1 && ref2 && strcasecmp(ref1, ref2) == 0) fetch->send_referer = true; - } + } if (!fetch->url) goto failed; - /* Pick the scheme ops */ - while (fetcher) { - if (strcmp(fetcher->scheme_name, ref1) == 0) { - fetch->ops = fetcher; - break; - } - fetcher = fetcher->next_fetcher; - } + /* Pick the scheme ops */ + while (fetcher) { + if (strcmp(fetcher->scheme_name, ref1) == 0) { + fetch->ops = fetcher; + break; + } + fetcher = fetcher->next_fetcher; + } - if (fetch->ops == NULL) - goto failed; + if (fetch->ops == NULL) + goto failed; - /* Got a scheme fetcher, try and set up the fetch */ - fetch->fetcher_handle = - fetch->ops->setup_fetch(fetch, url, only_2xx, post_urlenc, - post_multipart, verifiable, parent_url, - (const char **)headers); + /* Got a scheme fetcher, try and set up the fetch */ + fetch->fetcher_handle = + fetch->ops->setup_fetch(fetch, url, only_2xx, post_urlenc, + post_multipart, verifiable, parent_url, + (const char **)headers); - if (fetch->fetcher_handle == NULL) - goto failed; + if (fetch->fetcher_handle == NULL) + goto failed; - /* Rah, got it, so ref the fetcher. */ - fetch_ref_fetcher(fetch->ops); + /* Rah, got it, so ref the fetcher. */ + fetch_ref_fetcher(fetch->ops); /* these aren't needed past here */ if (ref1) { @@ -298,12 +307,12 @@ struct fetch * fetch_start(const char *url, const char *referer, ref1 = 0; } - if (ref2) { - free(ref2); - ref2 = 0; - } + if (ref2) { + free(ref2); + ref2 = 0; + } - /* Dump us in the queue and ask the queue to run. */ + /* Dump us in the queue and ask the queue to run. */ RING_INSERT(queue_ring, fetch); fetch_dispatch_jobs(); return fetch; @@ -313,59 +322,17 @@ failed: if (ref1) free(ref1); free(fetch->url); - if (fetch->referer) - free(fetch->referer); - free(fetch); + if (fetch->referer) + free(fetch->referer); + free(fetch); return 0; } -/** - * Dispatch a single job - */ -static bool fetch_dispatch_job(struct fetch *fetch) -{ - RING_REMOVE(queue_ring, fetch); - LOG(("Attempting to start fetch %p, fetcher %p, url %s", fetch, - fetch->fetcher_handle, fetch->url)); - if (!fetch->ops->start_fetch(fetch->fetcher_handle)) { - RING_INSERT(queue_ring, fetch); /* Put it back on the end of the queue */ - return false; - } else { - RING_INSERT(fetch_ring, fetch); - fetch->fetch_is_active = true; - return true; - } -} - -/** - * Choose and dispatch a single job. Return false if we failed to dispatch anything. - * - * We don't check the overall dispatch size here because we're not called unless - * there is room in the fetch queue for us. - */ -static bool fetch_choose_and_dispatch(void) -{ - struct fetch *queueitem; - queueitem = queue_ring; - do { - /* We can dispatch the selected item if there is room in the - * fetch ring - */ - int countbyhost; - RING_COUNTBYHOST(struct fetch, fetch_ring, countbyhost, queueitem->host); - if (countbyhost < option_max_fetchers_per_host) { - /* We can dispatch this item in theory */ - return fetch_dispatch_job(queueitem); - } - queueitem = queueitem->r_next; - } while (queueitem != queue_ring); - return false; -} /** * Dispatch as many jobs as we have room to dispatch. */ -static void fetch_dispatch_jobs(void) +void fetch_dispatch_jobs(void) { int all_active, all_queued; @@ -373,6 +340,24 @@ static void fetch_dispatch_jobs(void) return; /* Nothing to do, the queue is empty */ RING_GETSIZE(struct fetch, queue_ring, all_queued); RING_GETSIZE(struct fetch, fetch_ring, all_active); + + LOG(("queue_ring %i, fetch_ring %i", all_queued, all_active)); + + struct fetch *q = queue_ring; + if (q) { + do { + LOG(("queue_ring: %s", q->url)); + q = q->r_next; + } while (q != queue_ring); + } + struct fetch *f = fetch_ring; + if (f) { + do { + LOG(("fetch_ring: %s", f->url)); + f = f->r_next; + } while (f != fetch_ring); + } + while ( all_queued && all_active < option_max_fetchers ) { /*LOG(("%d queued, %d fetching", all_queued, all_active));*/ if (fetch_choose_and_dispatch()) { @@ -383,11 +368,59 @@ static void fetch_dispatch_jobs(void) break; } } - fetch_active = (all_active > 0); - LOG(("Fetch ring is now %d elements.", all_active)); - LOG(("Queue ring is now %d elements.", all_queued)); + fetch_active = (all_active > 0); + LOG(("Fetch ring is now %d elements.", all_active)); + LOG(("Queue ring is now %d elements.", all_queued)); } + +/** + * Choose and dispatch a single job. Return false if we failed to dispatch + * anything. + * + * We don't check the overall dispatch size here because we're not called unless + * there is room in the fetch queue for us. + */ +bool fetch_choose_and_dispatch(void) +{ + struct fetch *queueitem; + queueitem = queue_ring; + do { + /* We can dispatch the selected item if there is room in the + * fetch ring + */ + int countbyhost; + RING_COUNTBYHOST(struct fetch, fetch_ring, countbyhost, + queueitem->host); + if (countbyhost < option_max_fetchers_per_host) { + /* We can dispatch this item in theory */ + return fetch_dispatch_job(queueitem); + } + queueitem = queueitem->r_next; + } while (queueitem != queue_ring); + return false; +} + + +/** + * Dispatch a single job + */ +bool fetch_dispatch_job(struct fetch *fetch) +{ + RING_REMOVE(queue_ring, fetch); + LOG(("Attempting to start fetch %p, fetcher %p, url %s", fetch, + fetch->fetcher_handle, fetch->url)); + if (!fetch->ops->start_fetch(fetch->fetcher_handle)) { + RING_INSERT(queue_ring, fetch); /* Put it back on the end of the queue */ + return false; + } else { + RING_INSERT(fetch_ring, fetch); + fetch->fetch_is_active = true; + return true; + } +} + + /** * Abort a fetch. */ @@ -396,7 +429,7 @@ void fetch_abort(struct fetch *f) { assert(f); LOG(("fetch %p, fetcher %p, url '%s'", f, f->fetcher_handle, f->url)); - f->ops->abort_fetch(f->fetcher_handle); + f->ops->abort_fetch(f->fetcher_handle); } @@ -406,13 +439,13 @@ void fetch_abort(struct fetch *f) void fetch_free(struct fetch *f) { - LOG(("Freeing fetch %p, fetcher %p", f, f->fetcher_handle)); - f->ops->free_fetch(f->fetcher_handle); - fetch_unref_fetcher(f->ops); + LOG(("Freeing fetch %p, fetcher %p", f, f->fetcher_handle)); + f->ops->free_fetch(f->fetcher_handle); + fetch_unref_fetcher(f->ops); free(f->url); free(f->host); - if (f->referer) - free(f->referer); + if (f->referer) + free(f->referer); free(f); } @@ -425,14 +458,17 @@ void fetch_free(struct fetch *f) void fetch_poll(void) { - scheme_fetcher *fetcher = fetchers; - if (!fetch_active) - return; /* No point polling, there's no fetch active. */ - while (fetcher != NULL) { - /* LOG(("Polling fetcher for %s", fetcher->scheme_name)); */ - fetcher->poll_fetcher(fetcher->scheme_name); - fetcher = fetcher->next_fetcher; - } + scheme_fetcher *fetcher = fetchers; + + fetch_dispatch_jobs(); + + if (!fetch_active) + return; /* No point polling, there's no fetch active. */ + while (fetcher != NULL) { + /* LOG(("Polling fetcher for %s", fetcher->scheme_name)); */ + fetcher->poll_fetcher(fetcher->scheme_name); + fetcher = fetcher->next_fetcher; + } } @@ -447,18 +483,18 @@ bool fetch_can_fetch(const char *url) { const char *semi; size_t len; - scheme_fetcher *fetcher = fetchers; + scheme_fetcher *fetcher = fetchers; if ((semi = strchr(url, ':')) == NULL) return false; len = semi - url; - while (fetcher != NULL) { - if (strlen(fetcher->scheme_name) == len && - strncmp(fetcher->scheme_name, url, len) == 0) - return true; - fetcher = fetcher->next_fetcher; - } + while (fetcher != NULL) { + if (strlen(fetcher->scheme_name) == len && + strncmp(fetcher->scheme_name, url, len) == 0) + return true; + fetcher = fetcher->next_fetcher; + } return false; } @@ -469,8 +505,8 @@ bool fetch_can_fetch(const char *url) */ void fetch_change_callback(struct fetch *fetch, - fetch_callback callback, - void *p) + fetch_callback callback, + void *p) { assert(fetch); fetch->callback = callback; @@ -503,49 +539,47 @@ void fetch_send_callback(fetch_msg msg, struct fetch *fetch, const void *data, unsigned long size) { - LOG(("Fetcher sending callback. Fetch %p, fetcher %p data %p size %lu", - fetch, fetch->fetcher_handle, data, size)); - fetch->callback(msg, fetch->p, data, size); + LOG(("Fetcher sending callback. Fetch %p, fetcher %p data %p size %lu", + fetch, fetch->fetcher_handle, data, size)); + fetch->callback(msg, fetch->p, data, size); } void fetch_can_be_freed(struct fetch *fetch) { - int all_active, all_queued; - - /* Go ahead and free the fetch properly now */ - LOG(("Fetch %p, fetcher %p can be freed", fetch, fetch->fetcher_handle)); - - if (fetch->fetch_is_active) { - RING_REMOVE(fetch_ring, fetch); - } else { - RING_REMOVE(queue_ring, fetch); - } - + int all_active, all_queued; + + /* Go ahead and free the fetch properly now */ + LOG(("Fetch %p, fetcher %p can be freed", fetch, fetch->fetcher_handle)); + + if (fetch->fetch_is_active) { + RING_REMOVE(fetch_ring, fetch); + } else { + RING_REMOVE(queue_ring, fetch); + } + RING_GETSIZE(struct fetch, fetch_ring, all_active); RING_GETSIZE(struct fetch, queue_ring, all_queued); - - fetch_active = (all_active > 0); - - LOG(("Fetch ring is now %d elements.", all_active)); - LOG(("Queue ring is now %d elements.", all_queued)); - fetch_free(fetch); - - fetch_dispatch_jobs(); + fetch_active = (all_active > 0); + + LOG(("Fetch ring is now %d elements.", all_active)); + LOG(("Queue ring is now %d elements.", all_queued)); + + fetch_free(fetch); } void fetch_set_http_code(struct fetch *fetch, long http_code) { - LOG(("Setting HTTP code to %ld", http_code)); - fetch->http_code = http_code; + LOG(("Setting HTTP code to %ld", http_code)); + fetch->http_code = http_code; } const char * fetch_get_referer_to_send(struct fetch *fetch) { - if (fetch->send_referer) - return fetch->referer; - return NULL; + if (fetch->send_referer) + return fetch->referer; + return NULL; } diff --git a/content/fetchers/fetch_curl.c b/content/fetchers/fetch_curl.c index 209f9e062..41ebc3fea 100644 --- a/content/fetchers/fetch_curl.c +++ b/content/fetchers/fetch_curl.c @@ -57,15 +57,15 @@ struct cert_info { /** Information for a single fetch. */ struct curl_fetch_info { - struct fetch *fetch_handle; /**< The fetch handle we're parented by. */ + struct fetch *fetch_handle; /**< The fetch handle we're parented by. */ CURL * curl_handle; /**< cURL handle if being fetched, or 0. */ bool had_headers; /**< Headers have been processed. */ bool abort; /**< Abort requested. */ bool stopped; /**< Download stopped on purpose. */ bool only_2xx; /**< Only HTTP 2xx responses acceptable. */ bool verifiable; /**< Transaction is verifiable */ - char *url; /**< URL of this fetch. */ - char *host; /**< The hostname of this fetch. */ + char *url; /**< URL of this fetch. */ + char *host; /**< The hostname of this fetch. */ char *parent_fetch_url; /**< URL of parent fetch (not necessarily * the same as the referer) */ struct curl_slist *headers; /**< List of request headers. */ @@ -74,7 +74,7 @@ struct curl_fetch_info { char *cookie_string; /**< Cookie string for this fetch */ char *realm; /**< HTTP Auth Realm */ char *post_urlenc; /**< Url encoded POST string, or 0. */ - unsigned long http_code; /**< HTTP result code from cURL. */ + unsigned long http_code; /**< HTTP result code from cURL. */ struct curl_httppost *post_multipart; /**< Multipart post data, or 0. */ struct cache_data cachedata; /**< Cache control data */ time_t last_modified; /**< If-Modified-Since time */ @@ -97,20 +97,36 @@ CURLM *fetch_curl_multi; /**< Global cURL multi handle. */ /** Curl handle with default options set; not used for transfers. */ static CURL *fetch_blank_curl; static struct cache_handle *curl_handle_ring = 0; /**< Ring of cached handles */ +static int curl_fetchers_registered = 0; static char fetch_error_buffer[CURL_ERROR_SIZE]; /**< Error buffer for cURL. */ static char fetch_progress_buffer[256]; /**< Progress buffer for cURL */ static char fetch_proxy_userpwd[100]; /**< Proxy authentication details. */ +static bool fetch_curl_initialise(const char *scheme); +static void fetch_curl_finalise(const char *scheme); +static void * fetch_curl_setup(struct fetch *parent_fetch, const char *url, + bool only_2xx, const char *post_urlenc, + struct form_successful_control *post_multipart, + bool verifiable, const char *parent_url, const char **headers); +static bool fetch_curl_start(void *vfetch); +static bool fetch_curl_initiate_fetch(struct curl_fetch_info *fetch, + CURL *handle); +static CURL *fetch_curl_get_handle(char *host); +static void fetch_curl_cache_handle(CURL *handle, char *hostname); static CURLcode fetch_curl_set_options(struct curl_fetch_info *f); #ifdef WITH_SSL -static CURLcode fetch_curl_sslctxfun(CURL *curl_handle, SSL_CTX *sslctx, void *p); +static CURLcode fetch_curl_sslctxfun(CURL *curl_handle, SSL_CTX *sslctx, + void *p); #endif -static void fetch_curl_free(void *f); +static void fetch_curl_abort(void *vf); static void fetch_curl_stop(struct curl_fetch_info *f); +static void fetch_curl_free(void *f); +static void fetch_curl_poll(const char *scheme_ignored); static void fetch_curl_done(CURL *curl_handle, CURLcode result); static int fetch_curl_progress(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow); +static int fetch_curl_ignore(void); static size_t fetch_curl_data(void *data, size_t size, size_t nmemb, struct curl_fetch_info *f); static size_t fetch_curl_header(char *data, size_t size, size_t nmemb, @@ -119,11 +135,133 @@ static bool fetch_curl_process_headers(struct curl_fetch_info *f); static struct curl_httppost *fetch_curl_post_convert( struct form_successful_control *control); #ifdef WITH_SSL -static int fetch_curl_verify_callback(int preverify_ok, X509_STORE_CTX *x509_ctx); -static int fetch_curl_cert_verify_callback(X509_STORE_CTX *x509_ctx, void *parm); +static int fetch_curl_verify_callback(int preverify_ok, + X509_STORE_CTX *x509_ctx); +static int fetch_curl_cert_verify_callback(X509_STORE_CTX *x509_ctx, + void *parm); #endif -static void fetch_curl_cache_handle(CURL *handle, char *hostname); + +/** + * Initialise the fetcher. + * + * Must be called once before any other function. + */ + +void fetch_curl_register(void) +{ + CURLcode code; + curl_version_info_data *data; + int i; + + LOG(("curl_version %s", curl_version())); + + code = curl_global_init(CURL_GLOBAL_ALL); + if (code != CURLE_OK) + die("Failed to initialise the fetch module " + "(curl_global_init failed)."); + + fetch_curl_multi = curl_multi_init(); + if (!fetch_curl_multi) + die("Failed to initialise the fetch module " + "(curl_multi_init failed)."); + + /* Create a curl easy handle with the options that are common to all + fetches. */ + fetch_blank_curl = curl_easy_init(); + if (!fetch_blank_curl) + die("Failed to initialise the fetch module " + "(curl_easy_init failed)."); + +#undef SETOPT +#define SETOPT(option, value) \ + code = curl_easy_setopt(fetch_blank_curl, option, value); \ + if (code != CURLE_OK) \ + goto curl_easy_setopt_failed; + + if (verbose_log) { + SETOPT(CURLOPT_VERBOSE, 1); + } else { + SETOPT(CURLOPT_VERBOSE, 0); + } + SETOPT(CURLOPT_ERRORBUFFER, fetch_error_buffer); + if (option_suppress_curl_debug) + SETOPT(CURLOPT_DEBUGFUNCTION, fetch_curl_ignore); + SETOPT(CURLOPT_WRITEFUNCTION, fetch_curl_data); + SETOPT(CURLOPT_HEADERFUNCTION, fetch_curl_header); + SETOPT(CURLOPT_PROGRESSFUNCTION, fetch_curl_progress); + SETOPT(CURLOPT_NOPROGRESS, 0); + SETOPT(CURLOPT_USERAGENT, user_agent_string()); + SETOPT(CURLOPT_ENCODING, "gzip"); + SETOPT(CURLOPT_LOW_SPEED_LIMIT, 1L); + SETOPT(CURLOPT_LOW_SPEED_TIME, 180L); + SETOPT(CURLOPT_NOSIGNAL, 1L); + SETOPT(CURLOPT_CONNECTTIMEOUT, 30L); + + if (option_ca_bundle && strcmp(option_ca_bundle, "")) + SETOPT(CURLOPT_CAINFO, option_ca_bundle); + if (option_ca_path && strcmp(option_ca_path, "")) + SETOPT(CURLOPT_CAPATH, option_ca_path); + + /* cURL initialised okay, register the fetchers */ + + data = curl_version_info(CURLVERSION_NOW); + + for (i = 0; data->protocols[i]; i++) + if (!fetch_add_fetcher(data->protocols[i], + fetch_curl_initialise, + fetch_curl_setup, + fetch_curl_start, + fetch_curl_abort, + fetch_curl_free, + fetch_curl_poll, + fetch_curl_finalise)) { + LOG(("Unable to register cURL fetcher for %s", + data->protocols[i])); + } + return; + +curl_easy_setopt_failed: + die("Failed to initialise the fetch module " + "(curl_easy_setopt failed)."); +} + + +/** + * Initialise a cURL fetcher. + */ + +bool fetch_curl_initialise(const char *scheme) +{ + LOG(("Initialise cURL fetcher for %s", scheme)); + curl_fetchers_registered++; + return true; /* Always succeeds */ +} + + +/** + * Finalise a cURL fetcher + */ + +void fetch_curl_finalise(const char *scheme) +{ + curl_fetchers_registered--; + LOG(("Finalise cURL fetcher %s", scheme)); + if (curl_fetchers_registered == 0) { + /* All the fetchers have been finalised. */ + LOG(("All cURL fetchers finalised, closing down cURL")); + CURLMcode codem; + + curl_easy_cleanup(fetch_blank_curl); + + codem = curl_multi_cleanup(fetch_curl_multi); + if (codem != CURLM_OK) + LOG(("curl_multi_cleanup failed: ignoring")); + + curl_global_cleanup(); + } +} + /** * Start fetching data for the given URL. @@ -147,11 +285,10 @@ static void fetch_curl_cache_handle(CURL *handle, char *hostname); * callbacks will contain this. */ -static void * -fetch_curl_setup(struct fetch *parent_fetch, const char *url, - bool only_2xx, const char *post_urlenc, - struct form_successful_control *post_multipart, - bool verifiable, const char *parent_url, const char **headers) +void * fetch_curl_setup(struct fetch *parent_fetch, const char *url, + bool only_2xx, const char *post_urlenc, + struct form_successful_control *post_multipart, + bool verifiable, const char *parent_url, const char **headers) { char *host; struct curl_fetch_info *fetch; @@ -163,7 +300,7 @@ fetch_curl_setup(struct fetch *parent_fetch, const char *url, if (!fetch) return 0; - fetch->fetch_handle = parent_fetch; + fetch->fetch_handle = parent_fetch; res = url_host(url, &host); if (res != URL_FUNC_OK) { @@ -176,7 +313,7 @@ fetch_curl_setup(struct fetch *parent_fetch, const char *url, goto failed; } - LOG(("fetch %p, url '%s'", fetch, url)); + LOG(("fetch %p, url '%s'", fetch, url)); /* construct a new fetch structure */ fetch->curl_handle = 0; @@ -211,15 +348,15 @@ fetch_curl_setup(struct fetch *parent_fetch, const char *url, fetch->cachedata.last_modified = 0; fetch->last_modified = 0; fetch->file_etag = 0; - fetch->http_code = 0; + fetch->http_code = 0; #ifdef WITH_SSL memset(fetch->cert_data, 0, sizeof(fetch->cert_data)); #endif if (!fetch->url || - (parent_url && !fetch->parent_fetch_url) || - (post_urlenc && !fetch->post_urlenc) || - (post_multipart && !fetch->post_multipart)) + (parent_url && !fetch->parent_fetch_url) || + (post_urlenc && !fetch->post_urlenc) || + (post_multipart && !fetch->post_multipart)) goto failed; #define APPEND(list, value) \ @@ -284,14 +421,28 @@ failed: return 0; } + +/** + * Dispatch a single job + */ +bool fetch_curl_start(void *vfetch) +{ + struct curl_fetch_info *fetch = (struct curl_fetch_info*)vfetch; + return fetch_curl_initiate_fetch(fetch, + fetch_curl_get_handle(fetch->host)); +} + + /** * Initiate a fetch from the queue. * - * Called with a fetch structure and a CURL handle to be used to fetch the content. + * Called with a fetch structure and a CURL handle to be used to fetch the + * content. * * This will return whether or not the fetch was successfully initiated. */ -static bool fetch_curl_initiate_fetch(struct curl_fetch_info *fetch, CURL *handle) + +bool fetch_curl_initiate_fetch(struct curl_fetch_info *fetch, CURL *handle) { CURLcode code; CURLMcode codem; @@ -312,10 +463,12 @@ static bool fetch_curl_initiate_fetch(struct curl_fetch_info *fetch, CURL *handl return true; } + /** * Find a CURL handle to use to dispatch a job */ -static CURL *fetch_curl_get_handle(char *host) + +CURL *fetch_curl_get_handle(char *host) { struct cache_handle *h; CURL *ret; @@ -331,20 +484,12 @@ static CURL *fetch_curl_get_handle(char *host) return ret; } -/** - * Dispatch a single job - */ -static bool fetch_curl_start(void *vfetch) -{ - struct curl_fetch_info *fetch = (struct curl_fetch_info*)vfetch; - return fetch_curl_initiate_fetch(fetch, fetch_curl_get_handle(fetch->host)); -} /** * Cache a CURL handle for the provided host (if wanted) - * */ -static void fetch_curl_cache_handle(CURL *handle, char *host) + +void fetch_curl_cache_handle(CURL *handle, char *host) { struct cache_handle *h = 0; int c; @@ -377,11 +522,12 @@ static void fetch_curl_cache_handle(CURL *handle, char *host) RING_INSERT(curl_handle_ring, h); } + /** * Set options specific for a fetch. */ -static CURLcode +CURLcode fetch_curl_set_options(struct curl_fetch_info *f) { CURLcode code; @@ -475,12 +621,12 @@ fetch_curl_set_options(struct curl_fetch_info *f) * cURL SSL setup callback */ -static CURLcode +CURLcode fetch_curl_sslctxfun(CURL *curl_handle, SSL_CTX *sslctx, void *parm) { SSL_CTX_set_verify(sslctx, SSL_VERIFY_PEER, fetch_curl_verify_callback); SSL_CTX_set_cert_verify_callback(sslctx, fetch_curl_cert_verify_callback, - parm); + parm); return CURLE_OK; } #endif @@ -490,15 +636,15 @@ fetch_curl_sslctxfun(CURL *curl_handle, SSL_CTX *sslctx, void *parm) * Abort a fetch. */ -static void fetch_curl_abort(void *vf) +void fetch_curl_abort(void *vf) { - struct curl_fetch_info *f = (struct curl_fetch_info *)vf; + struct curl_fetch_info *f = (struct curl_fetch_info *)vf; assert(f); LOG(("fetch %p, url '%s'", f, f->url)); if (f->curl_handle) { f->abort = true; } else { - fetch_can_be_freed(f->fetch_handle); + fetch_can_be_freed(f->fetch_handle); } } @@ -509,7 +655,7 @@ static void fetch_curl_abort(void *vf) * Will prod the queue afterwards to allow pending requests to be initiated. */ -static void fetch_curl_stop(struct curl_fetch_info *f) +void fetch_curl_stop(struct curl_fetch_info *f) { CURLMcode codem; @@ -534,9 +680,9 @@ static void fetch_curl_stop(struct curl_fetch_info *f) * Free a fetch structure and associated resources. */ -static void fetch_curl_free(void *vf) +void fetch_curl_free(void *vf) { - struct curl_fetch_info *f = (struct curl_fetch_info *)vf; + struct curl_fetch_info *f = (struct curl_fetch_info *)vf; #ifdef WITH_SSL int i; #endif @@ -574,7 +720,7 @@ static void fetch_curl_free(void *vf) * Must be called regularly to make progress on fetches. */ -static void fetch_curl_poll(const char *scheme_ignored) +void fetch_curl_poll(const char *scheme_ignored) { int running, queue; CURLMcode codem; @@ -608,7 +754,7 @@ static void fetch_curl_poll(const char *scheme_ignored) * \param curl_handle curl easy handle of fetch */ -static void fetch_curl_done(CURL *curl_handle, CURLcode result) +void fetch_curl_done(CURL *curl_handle, CURLcode result) { bool finished = false; bool error = false; @@ -629,6 +775,7 @@ static void fetch_curl_done(CURL *curl_handle, CURLcode result) assert(code == CURLE_OK); abort = f->abort; + LOG(("done %s", f->url)); if (!abort && result == CURLE_OK) { /* fetch completed normally */ @@ -667,11 +814,11 @@ static void fetch_curl_done(CURL *curl_handle, CURLcode result) f->cachedata.etag = 0; } - /* postponed until after stop so that queue fetches are started */ if (abort) ; /* fetch was aborted: no callback */ else if (finished) { - fetch_send_callback(FETCH_FINISHED, f->fetch_handle, &cachedata, 0); + fetch_send_callback(FETCH_FINISHED, f->fetch_handle, + &cachedata, 0); free(cachedata.etag); } #ifdef WITH_SSL @@ -752,14 +899,16 @@ static void fetch_curl_done(CURL *curl_handle, CURLcode result) X509_free(certs[i].cert); } - fetch_send_callback(FETCH_CERT_ERR, f->fetch_handle, &ssl_certs, i); + fetch_send_callback(FETCH_CERT_ERR, f->fetch_handle, + &ssl_certs, i); } #endif else if (error) - fetch_send_callback(FETCH_ERROR, f->fetch_handle, fetch_error_buffer, 0); + fetch_send_callback(FETCH_ERROR, f->fetch_handle, + fetch_error_buffer, 0); - fetch_curl_stop(f); + fetch_curl_stop(f); } @@ -768,7 +917,7 @@ static void fetch_curl_done(CURL *curl_handle, CURLcode result) */ int fetch_curl_progress(void *clientp, double dltotal, double dlnow, - double ultotal, double ulnow) + double ultotal, double ulnow) { struct curl_fetch_info *f = (struct curl_fetch_info *) clientp; double percent; @@ -783,26 +932,39 @@ int fetch_curl_progress(void *clientp, double dltotal, double dlnow, human_friendly_bytesize(dlnow), human_friendly_bytesize(dltotal)); fetch_send_callback(FETCH_PROGRESS, f->fetch_handle, - fetch_progress_buffer, - (unsigned long) percent); + fetch_progress_buffer, + (unsigned long) percent); } else { snprintf(fetch_progress_buffer, 255, messages_get("ProgressU"), human_friendly_bytesize(dlnow)); fetch_send_callback(FETCH_PROGRESS, f->fetch_handle, - fetch_progress_buffer, 0); + fetch_progress_buffer, 0); } return 0; } + +/** + * Ignore everything given to it. + * + * Used to ignore cURL debug. + */ + +int fetch_curl_ignore(void) +{ + return 0; +} + + /** * Callback function for cURL. */ size_t fetch_curl_data(void *data, size_t size, size_t nmemb, - struct curl_fetch_info *f) + struct curl_fetch_info *f) { CURLcode code; @@ -810,8 +972,8 @@ size_t fetch_curl_data(void *data, size_t size, size_t nmemb, if (!f->http_code) { code = curl_easy_getinfo(f->curl_handle, CURLINFO_HTTP_CODE, - &f->http_code); - fetch_set_http_code(f->fetch_handle, f->http_code); + &f->http_code); + fetch_set_http_code(f->fetch_handle, f->http_code); assert(code == CURLE_OK); } @@ -848,7 +1010,7 @@ size_t fetch_curl_data(void *data, size_t size, size_t nmemb, */ size_t fetch_curl_header(char *data, size_t size, size_t nmemb, - struct curl_fetch_info *f) + struct curl_fetch_info *f) { int i; size *= nmemb; @@ -1024,8 +1186,8 @@ bool fetch_curl_process_headers(struct curl_fetch_info *f) if (!f->http_code) { code = curl_easy_getinfo(f->curl_handle, CURLINFO_HTTP_CODE, - &f->http_code); - fetch_set_http_code(f->fetch_handle, f->http_code); + &f->http_code); + fetch_set_http_code(f->fetch_handle, f->http_code); assert(code == CURLE_OK); } http_code = f->http_code; @@ -1034,7 +1196,7 @@ bool fetch_curl_process_headers(struct curl_fetch_info *f) if (http_code == 304 && !f->post_urlenc && !f->post_multipart) { /* Not Modified && GET request */ fetch_send_callback(FETCH_NOTMODIFIED, f->fetch_handle, - (const char *)&f->cachedata, 0); + (const char *)&f->cachedata, 0); return true; } @@ -1057,7 +1219,7 @@ bool fetch_curl_process_headers(struct curl_fetch_info *f) if (f->only_2xx && strncmp(f->url, "http", 4) == 0 && (http_code < 200 || 299 < http_code)) { fetch_send_callback(FETCH_ERROR, f->fetch_handle, - messages_get("Not2xx"), 0); + messages_get("Not2xx"), 0); return true; } @@ -1086,7 +1248,7 @@ bool fetch_curl_process_headers(struct curl_fetch_info *f) f->last_modified > s.st_mtime && f->file_etag == s.st_mtime) { fetch_send_callback(FETCH_NOTMODIFIED, f->fetch_handle, - (const char *)&f->cachedata, 0); + (const char *)&f->cachedata, 0); curl_free(url_path); return true; } @@ -1257,122 +1419,3 @@ int fetch_curl_cert_verify_callback(X509_STORE_CTX *x509_ctx, void *parm) } #endif -static int curl_fetchers_registered = 0; -/** Initialise a cURL fetcher */ -static bool -fetch_curl_initialise(const char *scheme) -{ - LOG(("Initialise cURL fetcher for %s", scheme)); - curl_fetchers_registered++; - return true; /* Always succeeds */ -} - -/** Finalise a cURL fetcher */ -static void -fetch_curl_finalise(const char *scheme) -{ - curl_fetchers_registered--; - LOG(("Finalise cURL fetcher %s", scheme)); - if (curl_fetchers_registered == 0) { - /* All the fetchers have been finalised. */ - LOG(("All cURL fetchers finalised, closing down cURL")); - CURLMcode codem; - - curl_easy_cleanup(fetch_blank_curl); - - codem = curl_multi_cleanup(fetch_curl_multi); - if (codem != CURLM_OK) - LOG(("curl_multi_cleanup failed: ignoring")); - - curl_global_cleanup(); - } -} - -/** Ignore everything given to it. - * - * Used to ignore cURL debug. - */ -int fetch_curl_ignore(void) { return 0; } - -/** - * Initialise the fetcher. - * - * Must be called once before any other function. - */ - -void register_curl_fetchers(void) -{ - CURLcode code; - curl_version_info_data *data; - int i; - - LOG(("curl_version %s", curl_version())); - - code = curl_global_init(CURL_GLOBAL_ALL); - if (code != CURLE_OK) - die("Failed to initialise the fetch module " - "(curl_global_init failed)."); - - fetch_curl_multi = curl_multi_init(); - if (!fetch_curl_multi) - die("Failed to initialise the fetch module " - "(curl_multi_init failed)."); - - /* Create a curl easy handle with the options that are common to all - fetches. */ - fetch_blank_curl = curl_easy_init(); - if (!fetch_blank_curl) - die("Failed to initialise the fetch module " - "(curl_easy_init failed)."); - -#undef SETOPT -#define SETOPT(option, value) \ - code = curl_easy_setopt(fetch_blank_curl, option, value); \ - if (code != CURLE_OK) \ - goto curl_easy_setopt_failed; - - if (verbose_log) { - SETOPT(CURLOPT_VERBOSE, 1); - } else { - SETOPT(CURLOPT_VERBOSE, 0); - } - SETOPT(CURLOPT_ERRORBUFFER, fetch_error_buffer); - if (option_suppress_curl_debug) - SETOPT(CURLOPT_DEBUGFUNCTION, fetch_curl_ignore); - SETOPT(CURLOPT_WRITEFUNCTION, fetch_curl_data); - SETOPT(CURLOPT_HEADERFUNCTION, fetch_curl_header); - SETOPT(CURLOPT_PROGRESSFUNCTION, fetch_curl_progress); - SETOPT(CURLOPT_NOPROGRESS, 0); - SETOPT(CURLOPT_USERAGENT, user_agent_string()); - SETOPT(CURLOPT_ENCODING, "gzip"); - SETOPT(CURLOPT_LOW_SPEED_LIMIT, 1L); - SETOPT(CURLOPT_LOW_SPEED_TIME, 180L); - SETOPT(CURLOPT_NOSIGNAL, 1L); - SETOPT(CURLOPT_CONNECTTIMEOUT, 30L); - - if (option_ca_bundle && strcmp(option_ca_bundle, "")) - SETOPT(CURLOPT_CAINFO, option_ca_bundle); - if (option_ca_path && strcmp(option_ca_path, "")) - SETOPT(CURLOPT_CAPATH, option_ca_path); - - /* cURL initialised okay, register the fetchers */ - - data = curl_version_info(CURLVERSION_NOW); - - for (i = 0; data->protocols[i]; i++) - if (!fetch_add_fetcher(data->protocols[i], - fetch_curl_initialise, - fetch_curl_setup, - fetch_curl_start, - fetch_curl_abort, - fetch_curl_free, - fetch_curl_poll, - fetch_curl_finalise)) { - LOG(("Unable to register cURL fetcher for %s", data->protocols[i])); - } - return; - -curl_easy_setopt_failed: - die("Failed to initialise the fetch module " - "(curl_easy_setopt failed)."); -} diff --git a/content/fetchers/fetch_curl.h b/content/fetchers/fetch_curl.h index 6dcba8914..fcfadc723 100644 --- a/content/fetchers/fetch_curl.h +++ b/content/fetchers/fetch_curl.h @@ -14,7 +14,7 @@ #include -void register_curl_fetchers(void); +void fetch_curl_register(void); /** Global cURL multi handle. */ extern CURLM *fetch_curl_multi;