mirrors
/
netsurf
mirror of https://github.com/netsurf-browser/netsurf
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Port fetch layer to nsurl. Remove unused fetch_get_referer function. 

svn path=/trunk/netsurf/; revision=12899
This commit is contained in:
Michael Drake 2011-09-27 11:07:32 +00:00
parent a269a82d1f
commit e1f7a37f15
5 changed files with 129 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile.sources
View File

@ -16,7 +16,7 @@ S_RENDER := box.c box_construct.c box_normalise.c \
hubbub_binding.c imagemap.c layout.c list.c search.c table.c \
textinput.c textplain.c
S_UTILS := base64.c filename.c hashtable.c locale.c messages.c \
S_UTILS := base64.c filename.c hashtable.c locale.c messages.c nsurl.c \
talloc.c url.c utf8.c utils.c useragent.c filepath.c log.c
S_HTTP := challenge.c generics.c primitives.c parameter.c \

170
content/fetch.c
View File

@ -48,7 +48,7 @@
#include "desktop/options.h"
#include "utils/log.h"
#include "utils/messages.h"
#include "utils/url.h"
#include "utils/nsurl.h"
#include "utils/utils.h"
#include "utils/ring.h"
@ -76,12 +76,12 @@ static scheme_fetcher *fetchers = NULL;
/** Information for a single fetch. */
struct fetch {
fetch_callback callback;/**< Callback function. */
char *url; /**< URL. */
char *referer; /**< Referer URL. */
nsurl *url; /**< URL. */
nsurl *referer; /**< Referer URL. */
bool send_referer; /**< Valid to send the referer */
bool verifiable; /**< Transaction is verifiable */
void *p; /**< Private data for callback. */
lwc_string *lwc_host; /**< Host part of URL, interned */
lwc_string *host; /**< Host part of URL, interned */
long http_code; /**< HTTP response code, or 0. */
scheme_fetcher *ops; /**< Fetcher operations for this fetch,
NULL if not set. */
@ -100,6 +100,10 @@ static void fetch_dispatch_jobs(void);
static bool fetch_choose_and_dispatch(void);
static bool fetch_dispatch_job(struct fetch *fetch);
/* Static lwc_strings */
static lwc_string *fetch_http_lwc;
static lwc_string *fetch_https_lwc;
/**
* Initialise the fetcher.
@ -115,6 +119,18 @@ void fetch_init(void)
fetch_resource_register();
fetch_about_register();
fetch_active = false;
if (lwc_intern_string("http", SLEN("http"), &fetch_http_lwc) !=
lwc_error_ok) {
die("Failed to initialise the fetch module "
"(couldn't intern \"http\").");
}
if (lwc_intern_string("https", SLEN("https"), &fetch_https_lwc) !=
lwc_error_ok) {
die("Failed to initialise the fetch module "
"(couldn't intern \"https\").");
}
}
@ -135,6 +151,9 @@ void fetch_quit(void)
}
fetch_unref_fetcher(fetchers);
}
lwc_string_unref(fetch_http_lwc);
lwc_string_unref(fetch_https_lwc);
}
@ -211,56 +230,33 @@ void fetch_unref_fetcher(scheme_fetcher *fetcher)
*
*/
struct fetch * fetch_start(const char *url, const char *referer,
struct fetch * fetch_start(nsurl *url, nsurl *referer,
fetch_callback callback,
void *p, bool only_2xx, const char *post_urlenc,
const struct fetch_multipart_data *post_multipart,
bool verifiable, const char *headers[])
{
char *host;
struct fetch *fetch;
url_func_result res;
char *scheme = NULL, *ref_scheme = NULL;
scheme_fetcher *fetcher = fetchers;
lwc_string *scheme;
bool match;
fetch = malloc(sizeof (*fetch));
if (fetch == NULL)
return NULL;
res = url_host(url, &host);
if (res != URL_FUNC_OK) {
/* we only fail memory exhaustion */
if (res == URL_FUNC_NOMEM)
goto failed;
host = strdup("");
if (host == NULL)
goto failed;
}
/* The URL we're fetching must have a scheme */
res = url_scheme(url, &scheme);
if (res != URL_FUNC_OK)
scheme = nsurl_get_component(url, NSURL_SCHEME);
if (scheme == NULL)
goto failed;
if (referer) {
res = url_scheme(referer, &ref_scheme);
if (res != URL_FUNC_OK) {
/* we only fail memory exhaustion */
if (res == URL_FUNC_NOMEM)
goto failed;
ref_scheme = NULL;
}
}
#ifdef DEBUG_FETCH_VERBOSE
LOG(("fetch %p, url '%s'", fetch, url));
LOG(("fetch %p, url '%s'", fetch, nsurl_access(url)));
#endif
/* construct a new fetch structure */
fetch->callback = callback;
fetch->url = strdup(url);
fetch->url = nsurl_ref(url);
fetch->verifiable = verifiable;
fetch->p = p;
fetch->http_code = 0;
@ -271,15 +267,14 @@ struct fetch * fetch_start(const char *url, const char *referer,
fetch->fetcher_handle = NULL;
fetch->ops = NULL;
fetch->fetch_is_active = false;
fetch->lwc_host = NULL;
if (lwc_intern_string(host, strlen(host), &fetch->lwc_host) != lwc_error_ok)
goto failed;
fetch->host = nsurl_get_component(url, NSURL_HOST);
if (referer != NULL) {
fetch->referer = strdup(referer);
if (fetch->referer == NULL)
goto failed;
lwc_string *ref_scheme;
fetch->referer = nsurl_ref(referer);
ref_scheme = nsurl_get_component(referer, NSURL_SCHEME);
/* Not a problem if referer has no scheme */
/* Determine whether to send the Referer header */
if (option_send_referer && ref_scheme != NULL) {
@ -293,11 +288,16 @@ struct fetch * fetch_start(const char *url, const char *referer,
* request from a page served over http. The inverse
* (https -> http) should not send the referer (15.1.3)
*/
if (strcasecmp(scheme, ref_scheme) == 0 ||
(strcasecmp(scheme, "https") == 0 &&
strcasecmp(ref_scheme, "http") == 0))
bool match1;
bool match2;
lwc_string_isequal(scheme, ref_scheme, &match);
lwc_string_isequal(scheme, fetch_https_lwc, &match1);
lwc_string_isequal(ref_scheme, fetch_http_lwc, &match2);
if (match == true || (match1 == true && match2 == true))
fetch->send_referer = true;
}
if (ref_scheme != NULL)
lwc_string_unref(ref_scheme);
}
if (fetch->url == NULL)
@ -305,8 +305,8 @@ struct fetch * fetch_start(const char *url, const char *referer,
/* Pick the scheme ops */
while (fetcher) {
if (strcmp(lwc_string_data(fetcher->scheme_name),
scheme) == 0) {
lwc_string_isequal(fetcher->scheme_name, scheme, &match);
if (match == true) {
fetch->ops = fetcher;
break;
}
@ -318,7 +318,8 @@ struct fetch * fetch_start(const char *url, const char *referer,
/* Got a scheme fetcher, try and set up the fetch */
fetch->fetcher_handle =
fetch->ops->setup_fetch(fetch, url, only_2xx, post_urlenc,
fetch->ops->setup_fetch(fetch, nsurl_access(url),
only_2xx, post_urlenc,
post_multipart, headers);
if (fetch->fetcher_handle == NULL)
@ -328,9 +329,7 @@ struct fetch * fetch_start(const char *url, const char *referer,
fetch_ref_fetcher(fetch->ops);
/* these aren't needed past here */
free(scheme);
free(host);
free(ref_scheme);
lwc_string_unref(scheme);
/* Dump us in the queue and ask the queue to run. */
RING_INSERT(queue_ring, fetch);
@ -339,13 +338,16 @@ struct fetch * fetch_start(const char *url, const char *referer,
return fetch;
failed:
free(host);
free(ref_scheme);
free(scheme);
free(fetch->url);
free(fetch->referer);
if (fetch->lwc_host != NULL)
lwc_string_unref(fetch->lwc_host);
if (scheme != NULL)
lwc_string_unref(scheme);
if (fetch->host != NULL)
lwc_string_unref(fetch->host);
if (fetch->url != NULL)
nsurl_unref(fetch->url);
if (fetch->referer != NULL)
nsurl_unref(fetch->referer);
free(fetch);
return NULL;
@ -422,7 +424,7 @@ bool fetch_choose_and_dispatch(void)
*/
int countbyhost;
RING_COUNTBYLWCHOST(struct fetch, fetch_ring, countbyhost,
queueitem->lwc_host);
queueitem->host);
if (countbyhost < option_max_fetchers_per_host) {
/* We can dispatch this item in theory */
return fetch_dispatch_job(queueitem);
@ -441,7 +443,7 @@ bool fetch_dispatch_job(struct fetch *fetch)
RING_REMOVE(queue_ring, fetch);
#ifdef DEBUG_FETCH_VERBOSE
LOG(("Attempting to start fetch %p, fetcher %p, url %s", fetch,
fetch->fetcher_handle, fetch->url));
fetch->fetcher_handle, nsurl_access(fetch->url)));
#endif
if (!fetch->ops->start_fetch(fetch->fetcher_handle)) {
RING_INSERT(queue_ring, fetch); /* Put it back on the end of the queue */
@ -462,7 +464,8 @@ void fetch_abort(struct fetch *f)
{
assert(f);
#ifdef DEBUG_FETCH_VERBOSE
LOG(("fetch %p, fetcher %p, url '%s'", f, f->fetcher_handle, f->url));
LOG(("fetch %p, fetcher %p, url '%s'", f, f->fetcher_handle,
nsurl_access(f->url)));
#endif
f->ops->abort_fetch(f->fetcher_handle);
}
@ -479,11 +482,11 @@ void fetch_free(struct fetch *f)
#endif
f->ops->free_fetch(f->fetcher_handle);
fetch_unref_fetcher(f->ops);
free(f->url);
if (f->referer)
free(f->referer);
if (f->lwc_host != NULL)
lwc_string_unref(f->lwc_host);
nsurl_unref(f->url);
if (f->referer != NULL)
nsurl_unref(f->referer);
if (f->host != NULL)
lwc_string_unref(f->host);
free(f);
}
@ -520,20 +523,11 @@ void fetch_poll(void)
* \return true if the scheme is supported
*/
bool fetch_can_fetch(const char *url)
bool fetch_can_fetch(nsurl *url)
{
const char *semi;
size_t len;
scheme_fetcher *fetcher = fetchers;
lwc_string *scheme;
bool match;
if ((semi = strchr(url, ':')) == NULL)
return false;
len = semi - url;
if (lwc_intern_string(url, len, &scheme) != lwc_error_ok)
return false;
lwc_string *scheme = nsurl_get_component(url, NSURL_SCHEME);
while (fetcher != NULL) {
lwc_string_isequal(fetcher->scheme_name, scheme, &match);
@ -573,20 +567,6 @@ long fetch_http_code(struct fetch *fetch)
return fetch->http_code;
}
/**
* Get the referer
*
* \param fetch fetch to retrieve referer from
* \return Pointer to referer string, or NULL if none.
*/
const char *fetch_get_referer(struct fetch *fetch)
{
assert(fetch);
return fetch->referer;
}
/**
* Determine if a fetch was verifiable
*
@ -718,11 +698,10 @@ fetch_set_http_code(struct fetch *fetch, long http_code)
fetch->http_code = http_code;
}
const char *
fetch_get_referer_to_send(struct fetch *fetch)
const char *fetch_get_referer_to_send(struct fetch *fetch)
{
if (fetch->send_referer)
return fetch->referer;
return nsurl_access(fetch->referer);
return NULL;
}
@ -738,7 +717,7 @@ fetch_set_cookie(struct fetch *fetch, const char *data)
/* If the transaction's verifiable, we don't require
* that the request uri and the parent domain match,
* so don't pass in any referer/parent in this case. */
urldb_set_cookie(data, fetch->url, NULL);
urldb_set_cookie(data, nsurl_access(fetch->url), NULL);
} else if (fetch->referer != NULL) {
/* Permit the cookie to be set if the fetch is unverifiable
* and the fetch URI domain matches the referer. */
@ -747,7 +726,8 @@ fetch_set_cookie(struct fetch *fetch, const char *data)
* where a nested object requests a fetch, the origin URI
* is the nested object's parent URI, whereas the referer
* for the fetch will be the nested object's URI. */
urldb_set_cookie(data, fetch->url, fetch->referer);
urldb_set_cookie(data, nsurl_access(fetch->url),
nsurl_access(fetch->referer));
}
}

6
content/fetch.h
View File

@ -28,6 +28,7 @@
#include <libwapcaplet/libwapcaplet.h>
#include "utils/config.h"
#include "utils/nsurl.h"
typedef enum {
FETCH_PROGRESS,
@ -84,7 +85,7 @@ typedef void (*fetch_callback)(fetch_msg msg, void *p, const void *data,
void fetch_init(void);
struct fetch * fetch_start(const char *url, const char *referer,
struct fetch * fetch_start(nsurl *url, nsurl *referer,
fetch_callback callback,
void *p, bool only_2xx, const char *post_urlenc,
const struct fetch_multipart_data *post_multipart,
@ -95,12 +96,11 @@ void fetch_poll(void);
void fetch_quit(void);
const char *fetch_filetype(const char *unix_path);
char *fetch_mimetype(const char *ro_path);
bool fetch_can_fetch(const char *url);
bool fetch_can_fetch(nsurl *url);
void fetch_change_callback(struct fetch *fetch,
fetch_callback callback,
void *p);
long fetch_http_code(struct fetch *fetch);
const char *fetch_get_referer(struct fetch *fetch);
bool fetch_get_verifiable(struct fetch *fetch);
void fetch_multipart_data_destroy(struct fetch_multipart_data *list);

50
content/llcache.c
View File

@ -351,11 +351,20 @@ nserror llcache_handle_retrieve(const char *url, uint32_t flags,
nserror error;
llcache_object_user *user;
llcache_object *object;
nsurl *nsurl;
/* make a nsurl */
error = nsurl_create(url, &nsurl);
if (error != NSERROR_OK)
return error;
/* Can we fetch this URL at all? */
if (fetch_can_fetch(url) == false)
if (fetch_can_fetch(nsurl) == false)
return NSERROR_NO_FETCH_HANDLER;
/* done with nsurl */
nsurl_unref(nsurl);
/* Create a new object user */
error = llcache_object_user_new(cb, pw, &user);
if (error != NSERROR_OK)
@ -1051,6 +1060,8 @@ nserror llcache_object_refetch(llcache_object *object)
struct fetch_multipart_data *multipart = NULL;
char **headers = NULL;
int header_idx = 0;
nsurl*nsurl, *nsurl_ref;
nserror error;
if (object->fetch.post != NULL) {
if (object->fetch.post->type == LLCACHE_POST_URL_ENCODED)
@ -1109,14 +1120,35 @@ nserror llcache_object_refetch(llcache_object *object)
LOG(("Refetching %p", object));
#endif
/* make nsurls */
error = nsurl_create(object->url, &nsurl);
if (error != NSERROR_OK) {
free(headers);
return error;
}
if (object->fetch.referer != NULL) {
error = nsurl_create(object->fetch.referer, &nsurl_ref);
if (error != NSERROR_OK) {
nsurl_unref(nsurl);
free(headers);
return error;
}
} else {
nsurl_ref = NULL;
}
/* Kick off fetch */
object->fetch.fetch = fetch_start(object->url, object->fetch.referer,
object->fetch.fetch = fetch_start(nsurl, nsurl_ref,
llcache_fetch_callback, object,
object->fetch.flags & LLCACHE_RETRIEVE_NO_ERROR_PAGES,
urlenc, multipart,
object->fetch.flags & LLCACHE_RETRIEVE_VERIFIABLE,
(const char **) headers);
nsurl_unref(nsurl);
if (nsurl_ref != NULL)
nsurl_unref(nsurl_ref);
/* Clean up cache-control headers */
while (--header_idx >= 0)
free(headers[header_idx]);
@ -2000,6 +2032,7 @@ nserror llcache_fetch_redirect(llcache_object *object, const char *target,
url_func_result result;
/* Extract HTTP response code from the fetch object */
long http_code = fetch_http_code(object->fetch.fetch);
nsurl *nsurl;
/* Abort fetch for this object */
fetch_abort(object->fetch.fetch);
@ -2074,11 +2107,20 @@ nserror llcache_fetch_redirect(llcache_object *object, const char *target,
free(scheme);
free(object_scheme);
/* Bail out if we've no way of handling this URL */
if (fetch_can_fetch(url) == false) {
/* make a nsurl */
error = nsurl_create(url, &nsurl);
if (error != NSERROR_OK) {
free(url);
return error;
}
/* Bail out if we've no way of handling this URL */
if (fetch_can_fetch(nsurl) == false) {
free(url);
nsurl_unref(nsurl);
return NSERROR_OK;
}
nsurl_unref(nsurl);
if (http_code == 301 || http_code == 302 || http_code == 303) {
/* 301, 302, 303 redirects are all unconditional GET requests */

8
utils/ring.h
View File

@ -113,10 +113,10 @@
ringtype *p = ring; \
sizevar = 0; \
do { \
bool matches = false; \
if (lwc_string_caseless_isequal(p->lwc_host, \
lwc_hostname, \
&matches) == lwc_error_ok) \
bool matches = false; \
/* nsurl guarantees lowercase host */ \
if (lwc_string_isequal(p->host, lwc_hostname, \
&matches) == lwc_error_ok) \
if (matches) \
sizevar++; \
p = p->r_next; \