From cc3481a2ff3fcf65317e035cb20ac632e5e0c149 Mon Sep 17 00:00:00 2001 From: John Mark Bell Date: Wed, 30 Jan 2008 19:56:41 +0000 Subject: [PATCH] Make core fetching code handle verifiability of transactions Make core fetching code be responsible for inserting cookies into the urldb Provide accessor to a fetch's parent url (this is defined as being the URL of the verifiable fetch which caused this one to occur) Make fetchcache's 3xx handling use the parent url when spawning new fetches svn path=/trunk/netsurf/; revision=3809 --- content/fetch.c | 45 ++++++++++++++++++++++++++++--- content/fetch.h | 6 +++-- content/fetchcache.c | 50 ++++++++++++++++++++++++++++++++--- content/fetchers/fetch_curl.c | 25 +++--------------- 4 files changed, 95 insertions(+), 31 deletions(-) diff --git a/content/fetch.c b/content/fetch.c index 5813e9c1b..632e04505 100644 --- a/content/fetch.c +++ b/content/fetch.c @@ -81,6 +81,8 @@ struct fetch { char *url; /**< URL. */ char *referer; /**< Referer URL. */ bool send_referer; /**< Valid to send the referer */ + bool verifiable; /**< Transaction is verifiable */ + char *parent_fetch_url; /**< URL of parent fetch */ void *p; /**< Private data for callback. */ char *host; /**< Host part of URL. */ long http_code; /**< HTTP response code, or 0. */ @@ -266,6 +268,8 @@ struct fetch * fetch_start(const char *url, const char *referer, fetch->abort = false; fetch->stopped = false; fetch->url = strdup(url); + fetch->verifiable = verifiable; + fetch->parent_fetch_url = parent_url ? strdup(parent_url) : 0; fetch->p = p; fetch->host = host; fetch->http_code = 0; @@ -286,7 +290,8 @@ struct fetch * fetch_start(const char *url, const char *referer, fetch->send_referer = true; } - if (!fetch->url) + if (!fetch->url || + (parent_url && !fetch->parent_fetch_url)) goto failed; /* Pick the scheme ops */ @@ -304,8 +309,7 @@ struct fetch * fetch_start(const char *url, const char *referer, /* Got a scheme fetcher, try and set up the fetch */ fetch->fetcher_handle = fetch->ops->setup_fetch(fetch, url, only_2xx, post_urlenc, - post_multipart, verifiable, parent_url, - (const char **)headers); + post_multipart, (const char **)headers); if (fetch->fetcher_handle == NULL) goto failed; @@ -333,6 +337,7 @@ failed: free(host); if (ref1) free(ref1); + free(fetch->parent_fetch_url); free(fetch->url); if (fetch->referer) free(fetch->referer); @@ -454,6 +459,7 @@ void fetch_free(struct fetch *f) LOG(("Freeing fetch %p, fetcher %p", f, f->fetcher_handle)); f->ops->free_fetch(f->fetcher_handle); fetch_unref_fetcher(f->ops); + free(f->parent_fetch_url); free(f->url); free(f->host); if (f->referer) @@ -549,6 +555,20 @@ const char *fetch_get_referer(struct fetch *fetch) return fetch->referer; } +/** + * Get the parent URL for this fetch + * + * \param fetch fetch to retrieve parent url from + * \return Pointer to parent url, or NULL if none. + */ +const char *fetch_get_parent_url(struct fetch *fetch) +{ + assert(fetch); + + /* If the fetch is verifiable, then its own URL suffices */ + return fetch->verifiable ? fetch->url : fetch->parent_fetch_url; +} + void fetch_send_callback(fetch_msg msg, struct fetch *fetch, const void *data, unsigned long size) @@ -596,3 +616,22 @@ fetch_get_referer_to_send(struct fetch *fetch) return fetch->referer; return NULL; } + +void +fetch_set_cookie(struct fetch *fetch, const char *data) +{ + assert(fetch && data); + + /* If the fetch is unverifiable and there's no parent fetch + * url, err on the side of caution and do not set the cookie */ + + if (fetch->verifiable || fetch->parent_fetch_url) { + /* If the transaction's verifiable, we don't require + * that the request uri and the parent domain match, + * so don't pass in the parent in this case. */ + urldb_set_cookie(data, fetch->url, + fetch->verifiable ? 0 + : fetch->parent_fetch_url); + } +} + diff --git a/content/fetch.h b/content/fetch.h index d1c388ac9..da846c607 100644 --- a/content/fetch.h +++ b/content/fetch.h @@ -96,14 +96,15 @@ void fetch_change_callback(struct fetch *fetch, void *p); long fetch_http_code(struct fetch *fetch); const char *fetch_get_referer(struct fetch *fetch); +const char *fetch_get_parent_url(struct fetch *fetch); /* API for fetchers themselves */ typedef bool (*fetcher_initialise)(const char *); typedef void* (*fetcher_setup_fetch)(struct fetch *, const char *, bool, const char *, - struct form_successful_control *, bool, - const char *, const char **); + struct form_successful_control *, + const char **); typedef bool (*fetcher_start_fetch)(void *); typedef void (*fetcher_abort_fetch)(void *); typedef void (*fetcher_free_fetch)(void *); @@ -125,4 +126,5 @@ void fetch_remove_from_queues(struct fetch *fetch); void fetch_free(struct fetch *f); void fetch_set_http_code(struct fetch *fetch, long http_code); const char *fetch_get_referer_to_send(struct fetch *fetch); +void fetch_set_cookie(struct fetch *fetch, const char *data); #endif diff --git a/content/fetchcache.c b/content/fetchcache.c index e11c6a765..b6b17a618 100644 --- a/content/fetchcache.c +++ b/content/fetchcache.c @@ -730,7 +730,9 @@ void fetchcache_notmodified(struct content *c, const void *data) /* No cached content, so unconditionally refetch */ struct content_user *u; const char *ref = fetch_get_referer(c->fetch); + const char *parent = fetch_get_parent_url(c->fetch); char *referer = NULL; + char *parent_url = NULL; if (ref) { referer = strdup(ref); @@ -744,6 +746,19 @@ void fetchcache_notmodified(struct content *c, const void *data) } } + if (parent) { + parent_url = strdup(parent); + if (!parent_url) { + c->type = CONTENT_UNKNOWN; + c->status = CONTENT_STATUS_ERROR; + msg_data.error = messages_get("NoMemory"); + content_broadcast(c, CONTENT_MSG_ERROR, + msg_data); + free(referer); + return; + } + } + fetch_abort(c->fetch); c->fetch = 0; @@ -754,9 +769,10 @@ void fetchcache_notmodified(struct content *c, const void *data) for (u = c->user_list->next; u; u = u->next) { fetchcache_go(c, referer, u->callback, u->p1, u->p2, c->width, c->height, 0, 0, - false, ref ? referer : c->url); + false, parent_url); } + free(parent_url); free(referer); } } @@ -769,9 +785,10 @@ void fetchcache_redirect(struct content *c, const void *data, unsigned long size) { char *url, *url1; - char *referer; + char *referer, *parent_url; long http_code = fetch_http_code(c->fetch); const char *ref = fetch_get_referer(c->fetch); + const char *parent = fetch_get_parent_url(c->fetch); union content_msg_data msg_data; url_func_result result; @@ -783,8 +800,15 @@ void fetchcache_redirect(struct content *c, const void *data, /* 304 is handled by fetch_notmodified() */ assert(http_code != 304); - /* Clone referer -- original is destroyed in fetch_abort() */ + /* Extract fetch details */ + http_code = fetch_http_code(c->fetch); + ref = fetch_get_referer(c->fetch); + parent = fetch_get_parent_url(c->fetch); + + /* Clone referer and parent url + * originals are destroyed in fetch_abort() */ referer = ref ? strdup(ref) : NULL; + parent_url = parent ? strdup(parent) : NULL; /* set the status to ERROR so that this content is * destroyed in content_clean() */ @@ -803,6 +827,18 @@ void fetchcache_redirect(struct content *c, const void *data, return; } + /* Ensure parent url cloning succeeded + * _must_ be after content invalidation */ + if (parent && !parent_url) { + LOG(("Failed cloning parent url")); + + msg_data.error = messages_get("BadRedirect"); + content_broadcast(c, CONTENT_MSG_ERROR, msg_data); + + free(referer); + return; + } + /** \todo 300, 305, 307 * More specifically: * + 300 needs to serve up the fetch body to the user @@ -823,6 +859,7 @@ void fetchcache_redirect(struct content *c, const void *data, msg_data.error = messages_get("BadRedirect"); content_broadcast(c, CONTENT_MSG_ERROR, msg_data); + free(parent_url); free(referer); return; } @@ -835,6 +872,7 @@ void fetchcache_redirect(struct content *c, const void *data, msg_data.error = messages_get("BadRedirect"); content_broadcast(c, CONTENT_MSG_ERROR, msg_data); + free(parent_url); free(referer); return; } @@ -848,6 +886,7 @@ void fetchcache_redirect(struct content *c, const void *data, msg_data.error = messages_get("BadRedirect"); content_broadcast(c, CONTENT_MSG_ERROR, msg_data); + free(parent_url); free(referer); return; } @@ -860,6 +899,7 @@ void fetchcache_redirect(struct content *c, const void *data, content_broadcast(c, CONTENT_MSG_ERROR, msg_data); free(url1); + free(parent_url); free(referer); return; } @@ -892,6 +932,7 @@ void fetchcache_redirect(struct content *c, const void *data, content_broadcast(c, CONTENT_MSG_ERROR, msg_data); free(url); + free(parent_url); free(referer); return; } @@ -905,11 +946,12 @@ void fetchcache_redirect(struct content *c, const void *data, /* Start fetching the replacement content */ fetchcache_go(replacement, referer, callback, p1, p2, c->width, c->height, NULL, NULL, - false, referer ? referer : c->url); + false, parent_url); } /* Clean up */ free(url); + free(parent_url); free(referer); } diff --git a/content/fetchers/fetch_curl.c b/content/fetchers/fetch_curl.c index 620ada15e..ecd911326 100644 --- a/content/fetchers/fetch_curl.c +++ b/content/fetchers/fetch_curl.c @@ -74,11 +74,8 @@ struct curl_fetch_info { bool abort; /**< Abort requested. */ bool stopped; /**< Download stopped on purpose. */ bool only_2xx; /**< Only HTTP 2xx responses acceptable. */ - bool verifiable; /**< Transaction is verifiable */ char *url; /**< URL of this fetch. */ char *host; /**< The hostname of this fetch. */ - char *parent_fetch_url; /**< URL of parent fetch (not necessarily - * the same as the referer) */ struct curl_slist *headers; /**< List of request headers. */ char *location; /**< Response Location header, or 0. */ unsigned long content_length; /**< Response Content-Length, or 0. */ @@ -119,7 +116,7 @@ static void fetch_curl_finalise(const char *scheme); static void * fetch_curl_setup(struct fetch *parent_fetch, const char *url, bool only_2xx, const char *post_urlenc, struct form_successful_control *post_multipart, - bool verifiable, const char *parent_url, const char **headers); + const char **headers); static bool fetch_curl_start(void *vfetch); static bool fetch_curl_initiate_fetch(struct curl_fetch_info *fetch, CURL *handle); @@ -299,7 +296,7 @@ void fetch_curl_finalise(const char *scheme) void * fetch_curl_setup(struct fetch *parent_fetch, const char *url, bool only_2xx, const char *post_urlenc, struct form_successful_control *post_multipart, - bool verifiable, const char *parent_url, const char **headers) + const char **headers) { char *host; struct curl_fetch_info *fetch; @@ -332,9 +329,7 @@ void * fetch_curl_setup(struct fetch *parent_fetch, const char *url, fetch->abort = false; fetch->stopped = false; fetch->only_2xx = only_2xx; - fetch->verifiable = verifiable; fetch->url = strdup(url); - fetch->parent_fetch_url = parent_url ? strdup(parent_url) : 0; fetch->headers = 0; fetch->host = host; fetch->location = 0; @@ -365,7 +360,6 @@ void * fetch_curl_setup(struct fetch *parent_fetch, const char *url, #endif if (!fetch->url || - (parent_url && !fetch->parent_fetch_url) || (post_urlenc && !fetch->post_urlenc) || (post_multipart && !fetch->post_multipart)) goto failed; @@ -423,7 +417,6 @@ void * fetch_curl_setup(struct fetch *parent_fetch, const char *url, failed: free(host); free(fetch->url); - free(fetch->parent_fetch_url); free(fetch->post_urlenc); if (fetch->post_multipart) curl_formfree(fetch->post_multipart); @@ -703,7 +696,6 @@ void fetch_curl_free(void *vf) curl_easy_cleanup(f->curl_handle); free(f->url); free(f->host); - free(f->parent_fetch_url); free(f->location); free(f->cookie_string); free(f->realm); @@ -1158,18 +1150,7 @@ size_t fetch_curl_header(char *data, size_t size, size_t nmemb, /* extract Set-Cookie header */ SKIP_ST(11); - /* If the fetch is unverifiable and there's no parent fetch - * url, err on the side of caution and do not set the - cookie */ - - if (f->verifiable || f->parent_fetch_url) { - /* If the transaction's verifiable, we don't require - * that the request uri and the parent domain match, - * so don't pass in the parent in this case. */ - urldb_set_cookie(&data[i], f->url, - f->verifiable ? 0 - : f->parent_fetch_url); - } + fetch_set_cookie(f->fetch_handle, &data[i]); } return size;