From 6f86862edbcf4b71823e556824636dbaff752990 Mon Sep 17 00:00:00 2001 From: Vincent Sanders Date: Sat, 28 Apr 2012 13:24:09 +0000 Subject: [PATCH] cleanup and reorganise low level cache source ready for extending with disc cache svn path=/trunk/netsurf/; revision=13893 --- content/llcache.c | 4006 ++++++++++++++++++++++----------------------- 1 file changed, 1975 insertions(+), 2031 deletions(-) diff --git a/content/llcache.c b/content/llcache.c index 503fea40c..29f1a557c 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -144,6 +144,7 @@ struct llcache_object { struct llcache_s { /** Handler for fetch-related queries */ llcache_query_callback query_cb; + /** Data for fetch-related query handler */ void *query_cb_pw; @@ -164,70 +165,389 @@ static lwc_string *llcache_file_lwc; static lwc_string *llcache_about_lwc; static lwc_string *llcache_resource_lwc; - -static nserror llcache_object_user_new(llcache_handle_callback cb, void *pw, - llcache_object_user **user); -static nserror llcache_object_user_destroy(llcache_object_user *user); - -static nserror llcache_object_retrieve(nsurl *url, uint32_t flags, - nsurl *referer, const llcache_post_data *post, - uint32_t redirect_count, llcache_object **result); -static nserror llcache_object_retrieve_from_cache(nsurl *url, - uint32_t flags, nsurl *referer, - const llcache_post_data *post, uint32_t redirect_count, - llcache_object **result); -static bool llcache_object_is_fresh(const llcache_object *object); -static nserror llcache_object_cache_update(llcache_object *object); -static nserror llcache_object_clone_cache_data(llcache_object *source, - llcache_object *destination, bool deep); -static nserror llcache_object_fetch(llcache_object *object, uint32_t flags, - nsurl *referer, const llcache_post_data *post, - uint32_t redirect_count); -static nserror llcache_object_refetch(llcache_object *object); - -static nserror llcache_object_new(nsurl *url, llcache_object **result); -static nserror llcache_object_destroy(llcache_object *object); -static nserror llcache_object_add_user(llcache_object *object, - llcache_object_user *user); -static nserror llcache_object_remove_user(llcache_object *object, - llcache_object_user *user); -static llcache_object_user *llcache_object_find_user( - const llcache_handle *handle); - -static nserror llcache_object_add_to_list(llcache_object *object, - llcache_object **list); -static nserror llcache_object_remove_from_list(llcache_object *object, - llcache_object **list); -static bool llcache_object_in_list(const llcache_object *object, - const llcache_object *list); - -static nserror llcache_object_notify_users(llcache_object *object); - -static nserror llcache_object_snapshot(llcache_object *object, - llcache_object **snapshot); - -static nserror llcache_post_data_clone(const llcache_post_data *orig, - llcache_post_data **clone); - -static nserror llcache_query_handle_response(bool proceed, void *cbpw); - +/* forward referenced callback function */ static void llcache_fetch_callback(const fetch_msg *msg, void *p); -static nserror llcache_fetch_redirect(llcache_object *object, - const char *target, llcache_object **replacement); -static nserror llcache_fetch_notmodified(llcache_object *object, - llcache_object **replacement); + + +/****************************************************************************** + * Low-level cache internals * + ******************************************************************************/ + +/** + * Create a new object user + * + * \param cb Callback routine + * \param pw Private data for callback + * \param user Pointer to location to receive result + * \return NSERROR_OK on success, appropriate error otherwise + */ +static nserror llcache_object_user_new(llcache_handle_callback cb, void *pw, + llcache_object_user **user) +{ + llcache_handle *h; + llcache_object_user *u; + + h = calloc(1, sizeof(llcache_handle)); + if (h == NULL) + return NSERROR_NOMEM; + + u = calloc(1, sizeof(llcache_object_user)); + if (u == NULL) { + free(h); + return NSERROR_NOMEM; + } + + h->cb = cb; + h->pw = pw; + + u->handle = h; + +#ifdef LLCACHE_TRACE + LOG(("Created user %p (%p, %p, %p)", u, h, (void *) cb, pw)); +#endif + + *user = u; + + return NSERROR_OK; +} + +/** + * Destroy an object user + * + * \param user User to destroy + * \return NSERROR_OK on success, appropriate error otherwise + * + * \pre User is not attached to an object + */ +static nserror llcache_object_user_destroy(llcache_object_user *user) +{ +#ifdef LLCACHE_TRACE + LOG(("Destroyed user %p", user)); +#endif + + assert(user->next == NULL); + assert(user->prev == NULL); + + if (user->handle != NULL) + free(user->handle); + + free(user); + + return NSERROR_OK; +} + +/** + * Remove a user from a low-level cache object + * + * \param object Object to remove user from + * \param user User to remove + * \return NSERROR_OK. + */ +static nserror llcache_object_remove_user(llcache_object *object, + llcache_object_user *user) +{ + assert(user != NULL); + assert(object != NULL); + assert(object->users != NULL); + assert(user->handle == NULL || user->handle->object == object); + assert((user->prev != NULL) || (object->users == user)); + + if (user == object->users) + object->users = user->next; + else + user->prev->next = user->next; + + if (user->next != NULL) + user->next->prev = user->prev; + + user->next = user->prev = NULL; + +#ifdef LLCACHE_TRACE + LOG(("Removing user %p from %p", user, object)); +#endif + + return NSERROR_OK; +} + +/** + * Iterate the users of an object, calling their callbacks. + * + * \param object The object to iterate + * \param event The event to pass to the callback. + * \return NSERROR_OK on success, appropriate error otherwise. + */ +static nserror llcache_send_event_to_users(llcache_object *object, + llcache_event *event) +{ + nserror error = NSERROR_OK; + llcache_object_user *user, *next_user; + + user = object->users; + while (user != NULL) { + user->iterator_target = true; + + error = user->handle->cb(user->handle, event, + user->handle->pw); + + next_user = user->next; + + user->iterator_target = false; + + if (user->queued_for_delete) { + llcache_object_remove_user(object, user); + llcache_object_user_destroy(user); + } + + if (error != NSERROR_OK) + break; + + user = next_user; + } + + return error; +} + +/** + * Create a new low-level cache object + * + * \param url URL of object to create + * \param result Pointer to location to receive result + * \return NSERROR_OK on success, appropriate error otherwise + */ +static nserror llcache_object_new(nsurl *url, llcache_object **result) +{ + llcache_object *obj = calloc(1, sizeof(llcache_object)); + if (obj == NULL) + return NSERROR_NOMEM; + +#ifdef LLCACHE_TRACE + LOG(("Created object %p (%s)", obj, nsurl_access(url))); +#endif + + obj->url = nsurl_ref(url); + + *result = obj; + + return NSERROR_OK; +} + +/** + * Clone a POST data object + * + * \param orig Object to clone + * \param clone Pointer to location to receive clone + * \return NSERROR_OK on success, appropriate error otherwise + */ +static nserror llcache_post_data_clone(const llcache_post_data *orig, + llcache_post_data **clone) +{ + llcache_post_data *post_clone; + + post_clone = calloc(1, sizeof(llcache_post_data)); + if (post_clone == NULL) + return NSERROR_NOMEM; + + post_clone->type = orig->type; + + /* Deep-copy the type-specific data */ + if (orig->type == LLCACHE_POST_URL_ENCODED) { + post_clone->data.urlenc = strdup(orig->data.urlenc); + if (post_clone->data.urlenc == NULL) { + free(post_clone); + + return NSERROR_NOMEM; + } + } else { + post_clone->data.multipart = fetch_multipart_data_clone( + orig->data.multipart); + if (post_clone->data.multipart == NULL) { + free(post_clone); + + return NSERROR_NOMEM; + } + } + + *clone = post_clone; + + return NSERROR_OK; +} + +/** + * Split a fetch header into name and value + * + * \param data Header string + * \param len Byte length of header + * \param name Pointer to location to receive header name + * \param value Pointer to location to receive header value + * \return NSERROR_OK on success, appropriate error otherwise + */ static nserror llcache_fetch_split_header(const uint8_t *data, size_t len, - char **name, char **value); -static nserror llcache_fetch_parse_header(llcache_object *object, - const uint8_t *data, size_t len, char **name, char **value); -static nserror llcache_fetch_process_header(llcache_object *object, - const uint8_t *data, size_t len); -static nserror llcache_fetch_process_data(llcache_object *object, - const uint8_t *data, size_t len); -static nserror llcache_fetch_auth(llcache_object *object, - const char *realm); -static nserror llcache_fetch_cert_error(llcache_object *object, - const struct ssl_cert_info *certs, size_t num); + char **name, char **value) +{ + char *n, *v; + const uint8_t *colon; + + /* Find colon */ + colon = (const uint8_t *) strchr((const char *) data, ':'); + if (colon == NULL) { + /* Failed, assume a key with no value */ + n = strdup((const char *) data); + if (n == NULL) + return NSERROR_NOMEM; + + v = strdup(""); + if (v == NULL) { + free(n); + return NSERROR_NOMEM; + } + } else { + /* Split header into name & value */ + + /* Strip leading whitespace from name */ + while (data[0] == ' ' || data[0] == '\t' || + data[0] == '\r' || data[0] == '\n') { + data++; + } + + /* Strip trailing whitespace from name */ + while (colon > data && (colon[-1] == ' ' || + colon[-1] == '\t' || colon[-1] == '\r' || + colon[-1] == '\n')) + colon--; + + n = strndup((const char *) data, colon - data); + if (n == NULL) + return NSERROR_NOMEM; + + /* Find colon again */ + while (*colon != ':') { + colon++; + } + + /* Skip over colon and any subsequent whitespace */ + do { + colon++; + } while (*colon == ' ' || *colon == '\t' || + *colon == '\r' || *colon == '\n'); + + /* Strip trailing whitespace from value */ + while (len > 0 && (data[len - 1] == ' ' || + data[len - 1] == '\t' || + data[len - 1] == '\r' || + data[len - 1] == '\n')) { + len--; + } + + v = strndup((const char *) colon, len - (colon - data)); + if (v == NULL) { + free(n); + return NSERROR_NOMEM; + } + } + + *name = n; + *value = v; + + return NSERROR_OK; +} + +/** + * Parse a fetch header + * + * \param object Object to parse header for + * \param data Header string + * \param len Byte length of header + * \param name Pointer to location to receive header name + * \param value Pointer to location to receive header value + * \return NSERROR_OK on success, appropriate error otherwise + * + * \note This function also has the side-effect of updating + * the cache control data for the object if an interesting + * header is encountered + */ +static nserror llcache_fetch_parse_header(llcache_object *object, + const uint8_t *data, size_t len, char **name, char **value) +{ + nserror error; + + /* Set fetch response time if not already set */ + if (object->cache.res_time == 0) + object->cache.res_time = time(NULL); + + /* Decompose header into name-value pair */ + error = llcache_fetch_split_header(data, len, name, value); + if (error != NSERROR_OK) + return error; + + /* Parse cache headers to populate cache control data */ +#define SKIP_ST(p) while (*p != '\0' && (*p == ' ' || *p == '\t')) p++ + + if (5 < len && strcasecmp(*name, "Date") == 0) { + /* extract Date header */ + object->cache.date = curl_getdate(*value, NULL); + } else if (4 < len && strcasecmp(*name, "Age") == 0) { + /* extract Age header */ + if ('0' <= **value && **value <= '9') + object->cache.age = atoi(*value); + } else if (8 < len && strcasecmp(*name, "Expires") == 0) { + /* extract Expires header */ + object->cache.expires = curl_getdate(*value, NULL); + } else if (14 < len && strcasecmp(*name, "Cache-Control") == 0) { + /* extract and parse Cache-Control header */ + const char *start = *value; + const char *comma = *value; + + while (*comma != '\0') { + while (*comma != '\0' && *comma != ',') + comma++; + + if (8 < comma - start && (strncasecmp(start, + "no-cache", 8) == 0 || + strncasecmp(start, "no-store", 8) == 0)) + /* When we get a disk cache we should + * distinguish between these two */ + object->cache.no_cache = LLCACHE_VALIDATE_ALWAYS; + else if (7 < comma - start && + strncasecmp(start, "max-age", 7) == 0) { + /* Find '=' */ + while (start < comma && *start != '=') + start++; + + /* Skip over it */ + start++; + + /* Skip whitespace */ + SKIP_ST(start); + + if (start < comma) + object->cache.max_age = atoi(start); + } + + if (*comma != '\0') { + /* Skip past comma */ + comma++; + /* Skip whitespace */ + SKIP_ST(comma); + } + + /* Set start for next token */ + start = comma; + } + } else if (5 < len && strcasecmp(*name, "ETag") == 0) { + /* extract ETag header */ + free(object->cache.etag); + object->cache.etag = strdup(*value); + if (object->cache.etag == NULL) + return NSERROR_NOMEM; + } else if (14 < len && strcasecmp(*name, "Last-Modified") == 0) { + /* extract Last-Modified header */ + object->cache.last_modified = curl_getdate(*value, NULL); + } + +#undef SKIP_ST + + return NSERROR_OK; +} /* Destroy headers */ static inline void llcache_destroy_headers(llcache_object *object) @@ -252,11 +572,1604 @@ static inline void llcache_invalidate_cache_control_data(llcache_object *object) object->cache.max_age = INVALID_AGE; } +/** + * Process a fetch header + * + * \param object Object being fetched + * \param data Header string + * \param len Byte length of header + * \return NSERROR_OK on success, appropriate error otherwise + */ +static nserror llcache_fetch_process_header(llcache_object *object, + const uint8_t *data, size_t len) +{ + nserror error; + char *name, *value; + llcache_header *temp; + + /* The headers for multiple HTTP responses may be delivered to us if + * the fetch layer receives a 401 response for which it has + * authentication credentials. This will result in a silent re-request + * after which we'll receive the actual response headers for the + * object we want to fetch (assuming that the credentials were correct + * of course) + * + * Therefore, if the header is an HTTP response start marker, then we + * must discard any headers we've read so far, reset the cache data + * that we might have computed, and start again. + */ + /** \todo Properly parse the response line */ + if (strncmp((const char *) data, "HTTP/", SLEN("HTTP/")) == 0) { + time_t req_time = object->cache.req_time; + + llcache_invalidate_cache_control_data(object); + + /* Restore request time, so we compute object's age correctly */ + object->cache.req_time = req_time; + + llcache_destroy_headers(object); + } + + error = llcache_fetch_parse_header(object, data, len, &name, &value); + if (error != NSERROR_OK) + return error; + + /* Append header data to the object's headers array */ + temp = realloc(object->headers, (object->num_headers + 1) * + sizeof(llcache_header)); + if (temp == NULL) { + free(name); + free(value); + return NSERROR_NOMEM; + } + + object->headers = temp; + + object->headers[object->num_headers].name = name; + object->headers[object->num_headers].value = value; + + object->num_headers++; + + return NSERROR_OK; +} + +/** + * (Re)fetch an object + * + * \param object Object to refetch + * \return NSERROR_OK on success, appropriate error otherwise + * + * \pre The fetch parameters in object->fetch must be populated + */ +static nserror llcache_object_refetch(llcache_object *object) +{ + const char *urlenc = NULL; + struct fetch_multipart_data *multipart = NULL; + char **headers = NULL; + int header_idx = 0; + + if (object->fetch.post != NULL) { + if (object->fetch.post->type == LLCACHE_POST_URL_ENCODED) + urlenc = object->fetch.post->data.urlenc; + else + multipart = object->fetch.post->data.multipart; + } + + /* Generate cache-control headers */ + headers = malloc(3 * sizeof(char *)); + if (headers == NULL) + return NSERROR_NOMEM; + + if (object->cache.etag != NULL) { + const size_t len = SLEN("If-None-Match: ") + + strlen(object->cache.etag) + 1; + + headers[header_idx] = malloc(len); + if (headers[header_idx] == NULL) { + free(headers); + return NSERROR_NOMEM; + } + + snprintf(headers[header_idx], len, "If-None-Match: %s", + object->cache.etag); + + header_idx++; + } + if (object->cache.date != 0) { + /* Maximum length of an RFC 1123 date is 29 bytes */ + const size_t len = SLEN("If-Modified-Since: ") + 29 + 1; + + headers[header_idx] = malloc(len); + if (headers[header_idx] == NULL) { + while (--header_idx >= 0) + free(headers[header_idx]); + free(headers); + return NSERROR_NOMEM; + } + + snprintf(headers[header_idx], len, "If-Modified-Since: %s", + rfc1123_date(object->cache.date)); + + header_idx++; + } + headers[header_idx] = NULL; + + /* Reset cache control data */ + llcache_invalidate_cache_control_data(object); + object->cache.req_time = time(NULL); + + /* Reset fetch state */ + object->fetch.state = LLCACHE_FETCH_INIT; + +#ifdef LLCACHE_TRACE + LOG(("Refetching %p", object)); +#endif + + /* Kick off fetch */ + object->fetch.fetch = fetch_start(object->url, object->fetch.referer, + llcache_fetch_callback, object, + object->fetch.flags & LLCACHE_RETRIEVE_NO_ERROR_PAGES, + urlenc, multipart, + object->fetch.flags & LLCACHE_RETRIEVE_VERIFIABLE, + (const char **) headers); + + /* Clean up cache-control headers */ + while (--header_idx >= 0) + free(headers[header_idx]); + free(headers); + + /* Did we succeed in creating a fetch? */ + if (object->fetch.fetch == NULL) + return NSERROR_NOMEM; + + return NSERROR_OK; +} + +/** + * Kick-off a fetch for an object + * + * \param object Object to fetch + * \param flags Fetch flags + * \param referer Referring URL, or NULL for none + * \param post POST data, or NULL for GET + * \param redirect_count Number of redirects followed so far + * \return NSERROR_OK on success, appropriate error otherwise + * + * \pre object::url must contain the URL to fetch + * \pre If there is a freshness validation candidate, + * object::candidate and object::cache must be filled in + * \pre There must not be a fetch in progress for \a object + */ +static nserror llcache_object_fetch(llcache_object *object, uint32_t flags, + nsurl *referer, const llcache_post_data *post, + uint32_t redirect_count) +{ + nserror error; + nsurl *referer_clone = NULL; + llcache_post_data *post_clone = NULL; + +#ifdef LLCACHE_TRACE + LOG(("Starting fetch for %p", object)); +#endif + + if (post != NULL) { + error = llcache_post_data_clone(post, &post_clone); + if (error != NSERROR_OK) + return error; + } + + if (referer != NULL) + referer_clone = nsurl_ref(referer); + + object->fetch.flags = flags; + object->fetch.referer = referer_clone; + object->fetch.post = post_clone; + object->fetch.redirect_count = redirect_count; + + return llcache_object_refetch(object); +} + +/** + * Destroy a low-level cache object + * + * \param object Object to destroy + * \return NSERROR_OK on success, appropriate error otherwise + * + * \pre Object is detached from cache list + * \pre Object has no users + * \pre Object is not a candidate (i.e. object::candidate_count == 0) + */ +static nserror llcache_object_destroy(llcache_object *object) +{ + size_t i; + +#ifdef LLCACHE_TRACE + LOG(("Destroying object %p", object)); +#endif + + nsurl_unref(object->url); + free(object->source_data); + + if (object->fetch.fetch != NULL) { + fetch_abort(object->fetch.fetch); + object->fetch.fetch = NULL; + } + + if (object->fetch.referer != NULL) + nsurl_unref(object->fetch.referer); + + if (object->fetch.post != NULL) { + if (object->fetch.post->type == LLCACHE_POST_URL_ENCODED) { + free(object->fetch.post->data.urlenc); + } else { + fetch_multipart_data_destroy( + object->fetch.post->data.multipart); + } + + free(object->fetch.post); + } + + free(object->cache.etag); + + for (i = 0; i < object->num_headers; i++) { + free(object->headers[i].name); + free(object->headers[i].value); + } + free(object->headers); + + free(object); + + return NSERROR_OK; +} + +/** + * Add a low-level cache object to a cache list + * + * \param object Object to add + * \param list List to add to + * \return NSERROR_OK + */ +static nserror llcache_object_add_to_list(llcache_object *object, + llcache_object **list) +{ + object->prev = NULL; + object->next = *list; + + if (*list != NULL) + (*list)->prev = object; + *list = object; + + return NSERROR_OK; +} + +/** + * Determine if an object is still fresh + * + * \param object Object to consider + * \return True if object is still fresh, false otherwise + */ +static bool llcache_object_is_fresh(const llcache_object *object) +{ + const llcache_cache_control *cd = &object->cache; + int current_age, freshness_lifetime; + time_t now = time(NULL); + + /* Calculate staleness of cached object as per RFC 2616 13.2.3/13.2.4 */ + current_age = max(0, (cd->res_time - cd->date)); + current_age = max(current_age, (cd->age == INVALID_AGE) ? 0 : cd->age); + current_age += cd->res_time - cd->req_time + now - cd->res_time; + + /* Determine freshness lifetime of this object */ + if (cd->max_age != INVALID_AGE) + freshness_lifetime = cd->max_age; + else if (cd->expires != 0) + freshness_lifetime = cd->expires - cd->date; + else if (cd->last_modified != 0) + freshness_lifetime = (now - cd->last_modified) / 10; + else + freshness_lifetime = 0; + +#ifdef LLCACHE_TRACE + LOG(("%p: (%d > %d || %d != %d)", object, + freshness_lifetime, current_age, + object->fetch.state, LLCACHE_FETCH_COMPLETE)); +#endif + + /* The object is fresh if: + * + * it was not forbidden from being returned from the cache + * unvalidated (i.e. the response contained a no-cache directive) + * + * and: + * + * its current age is within the freshness lifetime + * or if we're still fetching the object + */ + return (cd->no_cache == LLCACHE_VALIDATE_FRESH && + (freshness_lifetime > current_age || + object->fetch.state != LLCACHE_FETCH_COMPLETE)); +} + +/** + * Clone an object's cache data + * + * \param source Source object containing cache data to clone + * \param destination Destination object to clone cache data into + * \param deep Whether to deep-copy the data or not + * \return NSERROR_OK on success, appropriate error otherwise + * + * \post If \a deep is false, then any pointers in \a source will be set to NULL + */ +static nserror llcache_object_clone_cache_data(llcache_object *source, + llcache_object *destination, bool deep) +{ + /* ETag must be first, as it can fail when deep cloning */ + if (source->cache.etag != NULL) { + char *etag = source->cache.etag; + + if (deep) { + /* Copy the etag */ + etag = strdup(source->cache.etag); + if (etag == NULL) + return NSERROR_NOMEM; + } else { + /* Destination takes ownership */ + source->cache.etag = NULL; + } + + if (destination->cache.etag != NULL) + free(destination->cache.etag); + + destination->cache.etag = etag; + } + + destination->cache.req_time = source->cache.req_time; + destination->cache.res_time = source->cache.res_time; + + if (source->cache.date != 0) + destination->cache.date = source->cache.date; + + if (source->cache.expires != 0) + destination->cache.expires = source->cache.expires; + + if (source->cache.age != INVALID_AGE) + destination->cache.age = source->cache.age; + + if (source->cache.max_age != INVALID_AGE) + destination->cache.max_age = source->cache.max_age; + + if (source->cache.no_cache != LLCACHE_VALIDATE_FRESH) + destination->cache.no_cache = source->cache.no_cache; + + if (source->cache.last_modified != 0) + destination->cache.last_modified = source->cache.last_modified; + + return NSERROR_OK; +} + +/** + * Retrieve a potentially cached object + * + * \param url URL of object to retrieve + * \param flags Fetch flags + * \param referer Referring URL, or NULL if none + * \param post POST data, or NULL for a GET request + * \param redirect_count Number of redirects followed so far + * \param result Pointer to location to recieve retrieved object + * \return NSERROR_OK on success, appropriate error otherwise + */ +static nserror llcache_object_retrieve_from_cache(nsurl *url, uint32_t flags, + nsurl *referer, const llcache_post_data *post, + uint32_t redirect_count, llcache_object **result) +{ + nserror error; + llcache_object *obj, *newest = NULL; + +#ifdef LLCACHE_TRACE + LOG(("Searching cache for %s (%x %s %p)", url, flags, referer, post)); +#endif + + /* Search for the most recently fetched matching object */ + for (obj = llcache->cached_objects; obj != NULL; obj = obj->next) { + + if ((newest == NULL || + obj->cache.req_time > newest->cache.req_time) && + nsurl_compare(obj->url, url, + NSURL_COMPLETE) == true) { + newest = obj; + } + } + + if (newest != NULL && llcache_object_is_fresh(newest)) { + /* Found a suitable object, and it's still fresh, so use it */ + obj = newest; + +#ifdef LLCACHE_TRACE + LOG(("Found fresh %p", obj)); +#endif + + /* The client needs to catch up with the object's state. + * This will occur the next time that llcache_poll is called. + */ + } else if (newest != NULL) { + /* Found a candidate object but it needs freshness validation */ + + /* Create a new object */ + error = llcache_object_new(url, &obj); + if (error != NSERROR_OK) + return error; + +#ifdef LLCACHE_TRACE + LOG(("Found candidate %p (%p)", obj, newest)); +#endif + + /* Clone candidate's cache data */ + error = llcache_object_clone_cache_data(newest, obj, true); + if (error != NSERROR_OK) { + llcache_object_destroy(obj); + return error; + } + + /* Record candidate, so we can fall back if it is still fresh */ + newest->candidate_count++; + obj->candidate = newest; + + /* Attempt to kick-off fetch */ + error = llcache_object_fetch(obj, flags, referer, post, + redirect_count); + if (error != NSERROR_OK) { + newest->candidate_count--; + llcache_object_destroy(obj); + return error; + } + + /* Add new object to cache */ + llcache_object_add_to_list(obj, &llcache->cached_objects); + } else { + /* No object found; create a new one */ + /* Create new object */ + error = llcache_object_new(url, &obj); + if (error != NSERROR_OK) + return error; + +#ifdef LLCACHE_TRACE + LOG(("Not found %p", obj)); +#endif + + /* Attempt to kick-off fetch */ + error = llcache_object_fetch(obj, flags, referer, post, + redirect_count); + if (error != NSERROR_OK) { + llcache_object_destroy(obj); + return error; + } + + /* Add new object to cache */ + llcache_object_add_to_list(obj, &llcache->cached_objects); + } + + *result = obj; + + return NSERROR_OK; +} + +/** + * Retrieve an object from the cache, fetching it if necessary. + * + * \param url URL of object to retrieve + * \param flags Fetch flags + * \param referer Referring URL, or NULL if none + * \param post POST data, or NULL for a GET request + * \param redirect_count Number of redirects followed so far + * \param result Pointer to location to recieve retrieved object + * \return NSERROR_OK on success, appropriate error otherwise + */ +static nserror llcache_object_retrieve(nsurl *url, uint32_t flags, + nsurl *referer, const llcache_post_data *post, + uint32_t redirect_count, llcache_object **result) +{ + nserror error; + llcache_object *obj; + bool has_query; + nsurl *defragmented_url; + +#ifdef LLCACHE_TRACE + LOG(("Retrieve %s (%x, %s, %p)", url, flags, referer, post)); +#endif + + /** + * Caching Rules: + * + * 1) Forced fetches are never cached + * 2) POST requests are never cached + */ + + /* Look for a query segment */ + has_query = nsurl_has_component(url, NSURL_QUERY); + + /* Get rid of any url fragment */ + if (nsurl_has_component(url, NSURL_FRAGMENT)) { + error = nsurl_defragment(url, &defragmented_url); + if (error != NSERROR_OK) + return error; + } else { + defragmented_url = nsurl_ref(url); + } + + if (flags & LLCACHE_RETRIEVE_FORCE_FETCH || post != NULL) { + /* Create new object */ + error = llcache_object_new(defragmented_url, &obj); + if (error != NSERROR_OK) { + nsurl_unref(defragmented_url); + return error; + } + + /* Attempt to kick-off fetch */ + error = llcache_object_fetch(obj, flags, referer, post, + redirect_count); + if (error != NSERROR_OK) { + llcache_object_destroy(obj); + nsurl_unref(defragmented_url); + return error; + } + + /* Add new object to uncached list */ + llcache_object_add_to_list(obj, &llcache->uncached_objects); + } else { + error = llcache_object_retrieve_from_cache(defragmented_url, + flags, referer, post, redirect_count, &obj); + if (error != NSERROR_OK) { + nsurl_unref(defragmented_url); + return error; + } + + /* Returned object is already in the cached list */ + } + + obj->has_query = has_query; + +#ifdef LLCACHE_TRACE + LOG(("Retrieved %p", obj)); +#endif + + *result = obj; + + nsurl_unref(defragmented_url); + + return NSERROR_OK; +} + +/** + * Add a user to a low-level cache object + * + * \param object Object to add user to + * \param user User to add + * \return NSERROR_OK. + */ +static nserror llcache_object_add_user(llcache_object *object, + llcache_object_user *user) +{ + assert(user->next == NULL); + assert(user->prev == NULL); + + user->handle->object = object; + + user->prev = NULL; + user->next = object->users; + + if (object->users != NULL) + object->users->prev = user; + object->users = user; + +#ifdef LLCACHE_TRACE + LOG(("Adding user %p to %p", user, object)); +#endif + + return NSERROR_OK; +} + +/** + * Handle FETCH_REDIRECT event + * + * \param object Object being redirected + * \param target Target of redirect (may be relative) + * \param replacement Pointer to location to receive replacement object + * \return NSERROR_OK on success, appropriate error otherwise + */ +static nserror llcache_fetch_redirect(llcache_object *object, const char *target, + llcache_object **replacement) +{ + nserror error; + llcache_object *dest; + llcache_object_user *user, *next; + const llcache_post_data *post = object->fetch.post; + nsurl *url; + lwc_string *scheme; + lwc_string *object_scheme; + bool match; + /* Extract HTTP response code from the fetch object */ + long http_code = fetch_http_code(object->fetch.fetch); + + /* Abort fetch for this object */ + fetch_abort(object->fetch.fetch); + object->fetch.fetch = NULL; + + /* Invalidate the cache control data */ + llcache_invalidate_cache_control_data(object); + + /* And mark it complete */ + object->fetch.state = LLCACHE_FETCH_COMPLETE; + + /* Forcibly stop redirecting if we've followed too many redirects */ +#define REDIRECT_LIMIT 10 + if (object->fetch.redirect_count > REDIRECT_LIMIT) { + llcache_event event; + + LOG(("Too many nested redirects")); + + event.type = LLCACHE_EVENT_ERROR; + event.data.msg = messages_get("BadRedirect"); + + return llcache_send_event_to_users(object, &event); + } +#undef REDIRECT_LIMIT + + /* Make target absolute */ + error = nsurl_join(object->url, target, &url); + if (error != NSERROR_OK) + return error; + + /* Reject attempts to redirect from unvalidated to validated schemes + * A "validated" scheme is one over which we have some guarantee that + * the source is trustworthy. */ + object_scheme = nsurl_get_component(object->url, NSURL_SCHEME); + scheme = nsurl_get_component(url, NSURL_SCHEME); + + /* resource: and about: are allowed to redirect anywhere */ + if ((lwc_string_isequal(object_scheme, llcache_resource_lwc, + &match) == lwc_error_ok && match == false) && + (lwc_string_isequal(object_scheme, llcache_about_lwc, + &match) == lwc_error_ok && match == false)) { + /* file, about and resource are not valid redirect targets */ + if ((lwc_string_isequal(object_scheme, llcache_file_lwc, + &match) == lwc_error_ok && match == true) || + (lwc_string_isequal(object_scheme, llcache_about_lwc, + &match) == lwc_error_ok && match == true) || + (lwc_string_isequal(object_scheme, llcache_resource_lwc, + &match) == lwc_error_ok && match == true)) { + lwc_string_unref(object_scheme); + lwc_string_unref(scheme); + nsurl_unref(url); + return NSERROR_OK; + } + } + + lwc_string_unref(scheme); + lwc_string_unref(object_scheme); + + /* Bail out if we've no way of handling this URL */ + if (fetch_can_fetch(url) == false) { + nsurl_unref(url); + return NSERROR_OK; + } + + if (http_code == 301 || http_code == 302 || http_code == 303) { + /* 301, 302, 303 redirects are all unconditional GET requests */ + post = NULL; + } else if (http_code != 307 || post != NULL) { + /** \todo 300, 305, 307 with POST */ + nsurl_unref(url); + return NSERROR_OK; + } + + /* Attempt to fetch target URL */ + error = llcache_object_retrieve(url, object->fetch.flags, + object->fetch.referer, post, + object->fetch.redirect_count + 1, &dest); + + /* No longer require url */ + nsurl_unref(url); + + if (error != NSERROR_OK) + return error; + + /* Move user(s) to replacement object */ + for (user = object->users; user != NULL; user = next) { + next = user->next; + + llcache_object_remove_user(object, user); + llcache_object_add_user(dest, user); + } + + /* Dest is now our object */ + *replacement = dest; + + return NSERROR_OK; +} + +/** + * Update an object's cache state + * + * \param object Object to update cache for + * \return NSERROR_OK. + */ +static nserror llcache_object_cache_update(llcache_object *object) +{ + if (object->cache.date == 0) + object->cache.date = time(NULL); + + return NSERROR_OK; +} + +/** + * Handle FETCH_NOTMODIFIED event + * + * \param object Object to process + * \param replacement Pointer to location to receive replacement object + * \return NSERROR_OK. + */ +static nserror llcache_fetch_notmodified(llcache_object *object, + llcache_object **replacement) +{ + /* There may be no candidate if the server erroneously responded + * to an unconditional request with a 304 Not Modified response. + * In this case, we simply retain the initial object, having + * invalidated it and marked it as complete. + */ + if (object->candidate != NULL) { + llcache_object_user *user, *next; + + /* Move user(s) to candidate content */ + for (user = object->users; user != NULL; user = next) { + next = user->next; + + llcache_object_remove_user(object, user); + llcache_object_add_user(object->candidate, user); + } + + /* Candidate is no longer a candidate for us */ + object->candidate->candidate_count--; + + /* Clone our cache control data into the candidate */ + llcache_object_clone_cache_data(object, object->candidate, + false); + /* Bring candidate's cache data up to date */ + llcache_object_cache_update(object->candidate); + /* Revert no-cache to normal, if required */ + if (object->candidate->cache.no_cache == + LLCACHE_VALIDATE_ONCE) { + object->candidate->cache.no_cache = + LLCACHE_VALIDATE_FRESH; + } + + /* Candidate is now our object */ + *replacement = object->candidate; + object->candidate = NULL; + } else { + /* There was no candidate: retain object */ + *replacement = object; + } + + /* Ensure fetch has stopped */ + fetch_abort(object->fetch.fetch); + object->fetch.fetch = NULL; + + /* Invalidate our cache-control data */ + llcache_invalidate_cache_control_data(object); + + /* Mark it complete */ + object->fetch.state = LLCACHE_FETCH_COMPLETE; + + /* Old object will be flushed from the cache on the next poll */ + + return NSERROR_OK; +} + +/** + * Process a chunk of fetched data + * + * \param object Object being fetched + * \param data Data to process + * \param len Byte length of data + * \return NSERROR_OK on success, appropriate error otherwise. + */ +static nserror llcache_fetch_process_data(llcache_object *object, const uint8_t *data, + size_t len) +{ + /* Resize source buffer if it's too small */ + if (object->source_len + len >= object->source_alloc) { + const size_t new_len = object->source_len + len + 64 * 1024; + uint8_t *temp = realloc(object->source_data, new_len); + if (temp == NULL) + return NSERROR_NOMEM; + + object->source_data = temp; + object->source_alloc = new_len; + } + + /* Append this data chunk to source buffer */ + memcpy(object->source_data + object->source_len, data, len); + object->source_len += len; + + return NSERROR_OK; +} + +/** + * Handle a query response + * + * \param proceed Whether to proceed with fetch + * \param cbpw Our context for query + * \return NSERROR_OK on success, appropriate error otherwise + */ +static nserror llcache_query_handle_response(bool proceed, void *cbpw) +{ + llcache_event event; + llcache_object *object = cbpw; + + object->fetch.outstanding_query = false; + + /* Refetch, using existing fetch parameters, if client allows us to */ + if (proceed) + return llcache_object_refetch(object); + + /* Invalidate cache-control data */ + llcache_invalidate_cache_control_data(object); + + /* Mark it complete */ + object->fetch.state = LLCACHE_FETCH_COMPLETE; + + /* Inform client(s) that object fetch failed */ + event.type = LLCACHE_EVENT_ERROR; + /** \todo More appropriate error message */ + event.data.msg = messages_get("FetchFailed"); + + return llcache_send_event_to_users(object, &event); +} + +/** + * Handle an authentication request + * + * \param object Object being fetched + * \param realm Authentication realm + * \return NSERROR_OK on success, appropriate error otherwise. + */ +static nserror llcache_fetch_auth(llcache_object *object, const char *realm) +{ + const char *auth; + nserror error = NSERROR_OK; + + /* Abort fetch for this object */ + fetch_abort(object->fetch.fetch); + object->fetch.fetch = NULL; + + /* Invalidate cache-control data */ + llcache_invalidate_cache_control_data(object); + + /* Destroy headers */ + llcache_destroy_headers(object); + + /* If there was no realm, then default to the URL */ + /** \todo If there was no WWW-Authenticate header, use response body */ + if (realm == NULL) + realm = nsurl_access(object->url); + + auth = urldb_get_auth_details(nsurl_access(object->url), realm); + + if (auth == NULL || object->fetch.tried_with_auth == true) { + /* No authentication details, or tried what we had, so ask */ + object->fetch.tried_with_auth = false; + + if (llcache->query_cb != NULL) { + llcache_query query; + + /* Emit query for authentication details */ + query.type = LLCACHE_QUERY_AUTH; + query.url = object->url; + query.data.auth.realm = realm; + + object->fetch.outstanding_query = true; + + error = llcache->query_cb(&query, llcache->query_cb_pw, + llcache_query_handle_response, object); + } else { + llcache_event event; + + /* Mark object complete */ + object->fetch.state = LLCACHE_FETCH_COMPLETE; + + /* Inform client(s) that object fetch failed */ + event.type = LLCACHE_EVENT_ERROR; + /** \todo More appropriate error message */ + event.data.msg = messages_get("FetchFailed"); + + error = llcache_send_event_to_users(object, &event); + } + } else { + /* Flag that we've tried to refetch with credentials, so + * that if the fetch fails again, we ask the user again */ + object->fetch.tried_with_auth = true; + error = llcache_object_refetch(object); + } + + return error; +} + +/** + * Handle a TLS certificate verification failure + * + * \param object Object being fetched + * \param certs Certificate chain + * \param num Number of certificates in chain + * \return NSERROR_OK on success, appropriate error otherwise + */ +static nserror llcache_fetch_cert_error(llcache_object *object, + const struct ssl_cert_info *certs, size_t num) +{ + nserror error = NSERROR_OK; + + /* Fetch has been stopped, and destroyed. Invalidate object's pointer */ + object->fetch.fetch = NULL; + + /* Invalidate cache-control data */ + llcache_invalidate_cache_control_data(object); + + if (llcache->query_cb != NULL) { + llcache_query query; + + /* Emit query for TLS */ + query.type = LLCACHE_QUERY_SSL; + query.url = object->url; + query.data.ssl.certs = certs; + query.data.ssl.num = num; + + object->fetch.outstanding_query = true; + + error = llcache->query_cb(&query, llcache->query_cb_pw, + llcache_query_handle_response, object); + } else { + llcache_event event; + + /* Mark object complete */ + object->fetch.state = LLCACHE_FETCH_COMPLETE; + + /* Inform client(s) that object fetch failed */ + event.type = LLCACHE_EVENT_ERROR; + /** \todo More appropriate error message */ + event.data.msg = messages_get("FetchFailed"); + + error = llcache_send_event_to_users(object, &event); + } + + return error; +} + +/** + * Handler for fetch events + * + * \param msg Fetch event + * \param p Our private data + */ +static void llcache_fetch_callback(const fetch_msg *msg, void *p) +{ + nserror error = NSERROR_OK; + llcache_object *object = p; + llcache_event event; + +#ifdef LLCACHE_TRACE + LOG(("Fetch event %d for %p", msg->type, object)); +#endif + + switch (msg->type) { + case FETCH_HEADER: + /* Received a fetch header */ + object->fetch.state = LLCACHE_FETCH_HEADERS; + + error = llcache_fetch_process_header(object, + msg->data.header_or_data.buf, + msg->data.header_or_data.len); + break; + + /* 3xx responses */ + case FETCH_REDIRECT: + /* Request resulted in a redirect */ + + /* Release candidate, if any */ + if (object->candidate != NULL) { + object->candidate->candidate_count--; + object->candidate = NULL; + } + + error = llcache_fetch_redirect(object, + msg->data.redirect, &object); + break; + case FETCH_NOTMODIFIED: + /* Conditional request determined that cached object is fresh */ + error = llcache_fetch_notmodified(object, &object); + break; + + /* Normal 2xx state machine */ + case FETCH_DATA: + /* Received some data */ + if (object->fetch.state != LLCACHE_FETCH_DATA) { + /* On entry into this state, check if we need to + * invalidate the cache control data. We are guaranteed + * to have received all response headers. + * + * There are two cases in which we want to suppress + * cacheing of an object: + * + * 1) The HTTP response code is not 200 or 203 + * 2) The request URI had a query string and the + * response headers did not provide an explicit + * object expiration time. + */ + long http_code = fetch_http_code(object->fetch.fetch); + + if ((http_code != 200 && http_code != 203) || + (object->has_query && + (object->cache.max_age == INVALID_AGE && + object->cache.expires == 0))) { + /* Invalidate cache control data */ + llcache_invalidate_cache_control_data(object); + } + + /* Release candidate, if any */ + if (object->candidate != NULL) { + object->candidate->candidate_count--; + object->candidate = NULL; + } + } + + object->fetch.state = LLCACHE_FETCH_DATA; + + error = llcache_fetch_process_data(object, + msg->data.header_or_data.buf, + msg->data.header_or_data.len); + break; + case FETCH_FINISHED: + /* Finished fetching */ + { + uint8_t *temp; + + object->fetch.state = LLCACHE_FETCH_COMPLETE; + object->fetch.fetch = NULL; + + /* Shrink source buffer to required size */ + temp = realloc(object->source_data, + object->source_len); + /* If source_len is 0, then temp may be NULL */ + if (temp != NULL || object->source_len == 0) { + object->source_data = temp; + object->source_alloc = object->source_len; + } + + llcache_object_cache_update(object); + } + break; + + /* Out-of-band information */ + case FETCH_ERROR: + /* An error occurred while fetching */ + /* The fetch has has already been cleaned up by the fetcher */ + object->fetch.state = LLCACHE_FETCH_COMPLETE; + object->fetch.fetch = NULL; + + /* Release candidate, if any */ + if (object->candidate != NULL) { + object->candidate->candidate_count--; + object->candidate = NULL; + } + + /* Invalidate cache control data */ + llcache_invalidate_cache_control_data(object); + + /** \todo Consider using errorcode for something */ + + event.type = LLCACHE_EVENT_ERROR; + event.data.msg = msg->data.error; + + error = llcache_send_event_to_users(object, &event); + + break; + case FETCH_PROGRESS: + /* Progress update */ + event.type = LLCACHE_EVENT_PROGRESS; + event.data.msg = msg->data.progress; + + error = llcache_send_event_to_users(object, &event); + + break; + + /* Events requiring action */ + case FETCH_AUTH: + /* Need Authentication */ + + /* Release candidate, if any */ + if (object->candidate != NULL) { + object->candidate->candidate_count--; + object->candidate = NULL; + } + + error = llcache_fetch_auth(object, msg->data.auth.realm); + break; + case FETCH_CERT_ERR: + /* Something went wrong when validating TLS certificates */ + + /* Release candidate, if any */ + if (object->candidate != NULL) { + object->candidate->candidate_count--; + object->candidate = NULL; + } + + error = llcache_fetch_cert_error(object, + msg->data.cert_err.certs, + msg->data.cert_err.num_certs); + break; + } + + /* Deal with any errors reported by event handlers */ + if (error != NSERROR_OK) { + if (object->fetch.fetch != NULL) { + fetch_abort(object->fetch.fetch); + object->fetch.fetch = NULL; + + /* Invalidate cache control data */ + llcache_invalidate_cache_control_data(object); + + object->fetch.state = LLCACHE_FETCH_COMPLETE; + } + return; + } +} + +/** + * Find a user of a low-level cache object + * + * \param handle External cache handle to search for + * \return Pointer to corresponding user, or NULL if not found + */ +static llcache_object_user *llcache_object_find_user(const llcache_handle *handle) +{ + llcache_object_user *user; + + assert(handle->object != NULL); + + for (user = handle->object->users; user != NULL; user = user->next) { + if (user->handle == handle) + break; + } + + return user; +} + +/** + * Remove a low-level cache object from a cache list + * + * \param object Object to remove + * \param list List to remove from + * \return NSERROR_OK + */ +static nserror llcache_object_remove_from_list(llcache_object *object, + llcache_object **list) +{ + if (object == *list) + *list = object->next; + else + object->prev->next = object->next; + + if (object->next != NULL) + object->next->prev = object->prev; + + return NSERROR_OK; +} + +/** + * Determine if a low-level cache object resides in a given list + * + * \param object Object to search for + * \param list List to search in + * \return True if object resides in list, false otherwise + */ +static bool llcache_object_in_list(const llcache_object *object, + const llcache_object *list) +{ + while (list != NULL) { + if (list == object) + break; + + list = list->next; + } + + return list != NULL; +} + +/** + * Notify users of an object's current state + * + * \param object Object to notify users about + * \return NSERROR_OK on success, appropriate error otherwise + */ +static nserror llcache_object_notify_users(llcache_object *object) +{ + nserror error; + llcache_object_user *user, *next_user; + llcache_event event; + +#ifdef LLCACHE_TRACE + bool emitted_notify = false; +#endif + + /** + * State transitions and event emission for users. + * Rows: user state. Cols: object state. + * + * User\Obj INIT HEADERS DATA COMPLETE + * INIT - T T* T* + * HEADERS - - T T* + * DATA - - M T + * COMPLETE - - - - + * + * T => transition user to object state + * M => no transition required, but may need to emit event + * + * The transitions marked with an asterisk can be removed by moving + * the user context into the subsequent state and then reevaluating. + * + * Events are issued as follows: + * + * HAD_HEADERS: on transition from HEADERS -> DATA state + * HAD_DATA : in DATA state, whenever there's new source data + * DONE : on transition from DATA -> COMPLETE state + */ + + for (user = object->users; user != NULL; user = next_user) { + /* Emit necessary events to bring the user up-to-date */ + llcache_handle *handle = user->handle; + const llcache_fetch_state objstate = object->fetch.state; + + /* Flag that this user is the current iteration target + * in case the client attempts to destroy it underneath us */ + user->iterator_target = true; + + /* A note on the computation of next_user: + * + * Within this loop, we may make a number of calls to + * client code. Our contract with clients is that they + * can do whatever they like from within their callback + * handlers. This is so that we limit the pain of + * reentrancy to this module alone. + * + * One of the things a client can do from within its + * callback handler is to remove users from this object's + * user list. In the common case, the user they attempt + * to remove is the current iteration target, and we + * already protect against that causing problems here. + * However, no such protection exists if the client + * attempts to remove other users from this object's + * user list. + * + * Therefore, we cannot compute next_user up-front + * and expect it to remain valid across calls to + * client code (as the identity of the next user + * in the list may change underneath us). Instead, + * we must compute next_user at the point where we + * are about to cause another iteration of this loop + * (i.e. at the very end, and also at the points where + * continue is used) + */ + +#ifdef LLCACHE_TRACE + if (handle->state != objstate) { + if (emitted_notify == false) { + LOG(("Notifying users of %p", object)); + emitted_notify = true; + } + + LOG(("User %p state: %d Object state: %d", + user, handle->state, objstate)); + } +#endif + + /* User: INIT, Obj: HEADERS, DATA, COMPLETE => User->HEADERS */ + if (handle->state == LLCACHE_FETCH_INIT && + objstate > LLCACHE_FETCH_INIT) { + handle->state = LLCACHE_FETCH_HEADERS; + } + + /* User: HEADERS, Obj: DATA, COMPLETE => User->DATA */ + if (handle->state == LLCACHE_FETCH_HEADERS && + objstate > LLCACHE_FETCH_HEADERS) { + handle->state = LLCACHE_FETCH_DATA; + + /* Emit HAD_HEADERS event */ + event.type = LLCACHE_EVENT_HAD_HEADERS; + + error = handle->cb(handle, &event, handle->pw); + + if (user->queued_for_delete) { + next_user = user->next; + llcache_object_remove_user(object, user); + llcache_object_user_destroy(user); + + if (error != NSERROR_OK) + return error; + + continue; + } else if (error == NSERROR_NEED_DATA) { + /* User requested replay */ + handle->state = LLCACHE_FETCH_HEADERS; + + /* Continue with the next user -- we'll + * reemit the event next time round */ + user->iterator_target = false; + next_user = user->next; + continue; + } else if (error != NSERROR_OK) { + user->iterator_target = false; + return error; + } + } + + /* User: DATA, Obj: DATA, COMPLETE, more source available */ + if (handle->state == LLCACHE_FETCH_DATA && + objstate >= LLCACHE_FETCH_DATA && + object->source_len > handle->bytes) { + size_t orig_handle_read; + + /* Construct HAD_DATA event */ + event.type = LLCACHE_EVENT_HAD_DATA; + event.data.data.buf = + object->source_data + handle->bytes; + event.data.data.len = + object->source_len - handle->bytes; + + /* Update record of last byte emitted */ + if (object->fetch.flags & + LLCACHE_RETRIEVE_STREAM_DATA) { + /* Streaming, so reset to zero to + * minimise amount of cached source data. + * Additionally, we don't support replay + * when streaming. */ + orig_handle_read = 0; + handle->bytes = object->source_len = 0; + } else { + orig_handle_read = handle->bytes; + handle->bytes = object->source_len; + } + + /* Emit event */ + error = handle->cb(handle, &event, handle->pw); + if (user->queued_for_delete) { + next_user = user->next; + llcache_object_remove_user(object, user); + llcache_object_user_destroy(user); + + if (error != NSERROR_OK) + return error; + + continue; + } else if (error == NSERROR_NEED_DATA) { + /* User requested replay */ + handle->bytes = orig_handle_read; + + /* Continue with the next user -- we'll + * reemit the data next time round */ + user->iterator_target = false; + next_user = user->next; + continue; + } else if (error != NSERROR_OK) { + user->iterator_target = false; + return error; + } + } + + /* User: DATA, Obj: COMPLETE => User->COMPLETE */ + if (handle->state == LLCACHE_FETCH_DATA && + objstate > LLCACHE_FETCH_DATA) { + handle->state = LLCACHE_FETCH_COMPLETE; + + /* Emit DONE event */ + event.type = LLCACHE_EVENT_DONE; + + error = handle->cb(handle, &event, handle->pw); + if (user->queued_for_delete) { + next_user = user->next; + llcache_object_remove_user(object, user); + llcache_object_user_destroy(user); + + if (error != NSERROR_OK) + return error; + + continue; + } else if (error == NSERROR_NEED_DATA) { + /* User requested replay */ + handle->state = LLCACHE_FETCH_DATA; + + /* Continue with the next user -- we'll + * reemit the event next time round */ + user->iterator_target = false; + next_user = user->next; + continue; + } else if (error != NSERROR_OK) { + user->iterator_target = false; + return error; + } + } + + /* No longer the target of an iterator */ + user->iterator_target = false; + + next_user = user->next; + } + + return NSERROR_OK; +} + +/** + * Make a snapshot of the current state of an llcache_object. + * + * This has the side-effect of the new object being non-cacheable, + * also not-fetching and not a candidate for any other object. + * + * Also note that this new object has no users and at least one + * should be assigned to it before llcache_clean is entered or it + * will be immediately cleaned up. + * + * \param object The object to take a snapshot of + * \param snapshot Pointer to receive snapshot of \a object + * \return NSERROR_OK on success, appropriate error otherwise + */ +static nserror llcache_object_snapshot(llcache_object *object, + llcache_object **snapshot) +{ + llcache_object *newobj; + nserror error; + + error = llcache_object_new(object->url, &newobj); + + if (error != NSERROR_OK) + return error; + + newobj->has_query = object->has_query; + + newobj->source_alloc = newobj->source_len = object->source_len; + + if (object->source_len > 0) { + newobj->source_data = malloc(newobj->source_alloc); + if (newobj->source_data == NULL) { + llcache_object_destroy(newobj); + return NSERROR_NOMEM; + } + memcpy(newobj->source_data, object->source_data, + newobj->source_len); + } + + if (object->num_headers > 0) { + newobj->headers = calloc(sizeof(llcache_header), + object->num_headers); + if (newobj->headers == NULL) { + llcache_object_destroy(newobj); + return NSERROR_NOMEM; + } + while (newobj->num_headers < object->num_headers) { + llcache_header *nh = + &(newobj->headers[newobj->num_headers]); + llcache_header *oh = + &(object->headers[newobj->num_headers]); + newobj->num_headers += 1; + nh->name = strdup(oh->name); + nh->value = strdup(oh->value); + if (nh->name == NULL || nh->value == NULL) { + llcache_object_destroy(newobj); + return NSERROR_NOMEM; + } + } + } + + newobj->fetch.state = LLCACHE_FETCH_COMPLETE; + + *snapshot = newobj; + + return NSERROR_OK; +} + /****************************************************************************** * Public API * ******************************************************************************/ +/** + * Attempt to clean the cache + */ +/* Exported interface documented in llcache.h */ +void llcache_clean(void) +{ + llcache_object *object, *next; + uint32_t llcache_size = 0; + +#ifdef LLCACHE_TRACE + LOG(("Attempting cache clean")); +#endif + + /* Candidates for cleaning are (in order of priority): + * + * 1) Uncacheable objects with no users + * 2) Stale cacheable objects with no users or pending fetches + * 3) Fresh cacheable objects with no users or pending fetches + */ + + /* 1) Uncacheable objects with no users or fetches */ + for (object = llcache->uncached_objects; object != NULL; object = next) { + next = object->next; + + /* The candidate count of uncacheable objects is always 0 */ + if ((object->users == NULL) && + (object->candidate_count == 0) && + (object->fetch.fetch == NULL) && + (object->fetch.outstanding_query == false)) { +#ifdef LLCACHE_TRACE + LOG(("Found victim %p", object)); +#endif + llcache_object_remove_from_list(object, + &llcache->uncached_objects); + llcache_object_destroy(object); + } else { + llcache_size += object->source_len + sizeof(*object); + } + } + + /* 2) Stale cacheable objects with no users or pending fetches */ + for (object = llcache->cached_objects; object != NULL; object = next) { + next = object->next; + + if ((object->users == NULL) && + (object->candidate_count == 0) && + (llcache_object_is_fresh(object) == false) && + (object->fetch.fetch == NULL) && + (object->fetch.outstanding_query == false)) { +#ifdef LLCACHE_TRACE + LOG(("Found victim %p", object)); +#endif + llcache_object_remove_from_list(object, + &llcache->cached_objects); + llcache_object_destroy(object); + } else { + llcache_size += object->source_len + sizeof(*object); + } + } + + /* 3) Fresh cacheable objects with no users or pending + * fetches, only if the cache exceeds the configured size. + */ + if (llcache->limit < llcache_size) { + for (object = llcache->cached_objects; object != NULL; + object = next) { + next = object->next; + + if (object->users == NULL && + object->candidate_count == 0 && + object->fetch.fetch == NULL && + object->fetch.outstanding_query == + false) { +#ifdef LLCACHE_TRACE + LOG(("Found victim %p", object)); +#endif + llcache_size -= + object->source_len + sizeof(*object); + + llcache_object_remove_from_list(object, + &llcache->cached_objects); + llcache_object_destroy(object); + } + } + } + +#ifdef LLCACHE_TRACE + LOG(("Size: %u", llcache_size)); +#endif + +} + /* See llcache.h for documentation */ nserror llcache_initialise(llcache_query_callback cb, void *pw, uint32_t llcache_limit) @@ -587,1972 +2500,3 @@ bool llcache_handle_references_same_object(const llcache_handle *a, return a->object == b->object; } -/****************************************************************************** - * Low-level cache internals * - ******************************************************************************/ - -/** - * Create a new object user - * - * \param cb Callback routine - * \param pw Private data for callback - * \param user Pointer to location to receive result - * \return NSERROR_OK on success, appropriate error otherwise - */ -nserror llcache_object_user_new(llcache_handle_callback cb, void *pw, - llcache_object_user **user) -{ - llcache_handle *h; - llcache_object_user *u; - - h = calloc(1, sizeof(llcache_handle)); - if (h == NULL) - return NSERROR_NOMEM; - - u = calloc(1, sizeof(llcache_object_user)); - if (u == NULL) { - free(h); - return NSERROR_NOMEM; - } - - h->cb = cb; - h->pw = pw; - - u->handle = h; - -#ifdef LLCACHE_TRACE - LOG(("Created user %p (%p, %p, %p)", u, h, (void *) cb, pw)); -#endif - - *user = u; - - return NSERROR_OK; -} - -/** - * Destroy an object user - * - * \param user User to destroy - * \return NSERROR_OK on success, appropriate error otherwise - * - * \pre User is not attached to an object - */ -nserror llcache_object_user_destroy(llcache_object_user *user) -{ -#ifdef LLCACHE_TRACE - LOG(("Destroyed user %p", user)); -#endif - - assert(user->next == NULL); - assert(user->prev == NULL); - - if (user->handle != NULL) - free(user->handle); - - free(user); - - return NSERROR_OK; -} - -/** - * Iterate the users of an object, calling their callbacks. - * - * \param object The object to iterate - * \param event The event to pass to the callback. - * \return NSERROR_OK on success, appropriate error otherwise. - */ -static nserror llcache_send_event_to_users(llcache_object *object, - llcache_event *event) -{ - nserror error = NSERROR_OK; - llcache_object_user *user, *next_user; - - user = object->users; - while (user != NULL) { - user->iterator_target = true; - - error = user->handle->cb(user->handle, event, - user->handle->pw); - - next_user = user->next; - - user->iterator_target = false; - - if (user->queued_for_delete) { - llcache_object_remove_user(object, user); - llcache_object_user_destroy(user); - } - - if (error != NSERROR_OK) - break; - - user = next_user; - } - - return error; -} - -/** - * Retrieve an object from the cache, fetching it if necessary. - * - * \param url URL of object to retrieve - * \param flags Fetch flags - * \param referer Referring URL, or NULL if none - * \param post POST data, or NULL for a GET request - * \param redirect_count Number of redirects followed so far - * \param result Pointer to location to recieve retrieved object - * \return NSERROR_OK on success, appropriate error otherwise - */ -nserror llcache_object_retrieve(nsurl *url, uint32_t flags, - nsurl *referer, const llcache_post_data *post, - uint32_t redirect_count, llcache_object **result) -{ - nserror error; - llcache_object *obj; - bool has_query; - nsurl *defragmented_url; - -#ifdef LLCACHE_TRACE - LOG(("Retrieve %s (%x, %s, %p)", url, flags, referer, post)); -#endif - - /** - * Caching Rules: - * - * 1) Forced fetches are never cached - * 2) POST requests are never cached - */ - - /* Look for a query segment */ - has_query = nsurl_has_component(url, NSURL_QUERY); - - /* Get rid of any url fragment */ - if (nsurl_has_component(url, NSURL_FRAGMENT)) { - error = nsurl_defragment(url, &defragmented_url); - if (error != NSERROR_OK) - return error; - } else { - defragmented_url = nsurl_ref(url); - } - - if (flags & LLCACHE_RETRIEVE_FORCE_FETCH || post != NULL) { - /* Create new object */ - error = llcache_object_new(defragmented_url, &obj); - if (error != NSERROR_OK) { - nsurl_unref(defragmented_url); - return error; - } - - /* Attempt to kick-off fetch */ - error = llcache_object_fetch(obj, flags, referer, post, - redirect_count); - if (error != NSERROR_OK) { - llcache_object_destroy(obj); - nsurl_unref(defragmented_url); - return error; - } - - /* Add new object to uncached list */ - llcache_object_add_to_list(obj, &llcache->uncached_objects); - } else { - error = llcache_object_retrieve_from_cache(defragmented_url, - flags, referer, post, redirect_count, &obj); - if (error != NSERROR_OK) { - nsurl_unref(defragmented_url); - return error; - } - - /* Returned object is already in the cached list */ - } - - obj->has_query = has_query; - -#ifdef LLCACHE_TRACE - LOG(("Retrieved %p", obj)); -#endif - - *result = obj; - - nsurl_unref(defragmented_url); - - return NSERROR_OK; -} - -/** - * Retrieve a potentially cached object - * - * \param url URL of object to retrieve - * \param flags Fetch flags - * \param referer Referring URL, or NULL if none - * \param post POST data, or NULL for a GET request - * \param redirect_count Number of redirects followed so far - * \param result Pointer to location to recieve retrieved object - * \return NSERROR_OK on success, appropriate error otherwise - */ -nserror llcache_object_retrieve_from_cache(nsurl *url, uint32_t flags, - nsurl *referer, const llcache_post_data *post, - uint32_t redirect_count, llcache_object **result) -{ - nserror error; - llcache_object *obj, *newest = NULL; - -#ifdef LLCACHE_TRACE - LOG(("Searching cache for %s (%x %s %p)", url, flags, referer, post)); -#endif - - /* Search for the most recently fetched matching object */ - for (obj = llcache->cached_objects; obj != NULL; obj = obj->next) { - - if ((newest == NULL || - obj->cache.req_time > newest->cache.req_time) && - nsurl_compare(obj->url, url, - NSURL_COMPLETE) == true) { - newest = obj; - } - } - - if (newest != NULL && llcache_object_is_fresh(newest)) { - /* Found a suitable object, and it's still fresh, so use it */ - obj = newest; - -#ifdef LLCACHE_TRACE - LOG(("Found fresh %p", obj)); -#endif - - /* The client needs to catch up with the object's state. - * This will occur the next time that llcache_poll is called. - */ - } else if (newest != NULL) { - /* Found a candidate object but it needs freshness validation */ - - /* Create a new object */ - error = llcache_object_new(url, &obj); - if (error != NSERROR_OK) - return error; - -#ifdef LLCACHE_TRACE - LOG(("Found candidate %p (%p)", obj, newest)); -#endif - - /* Clone candidate's cache data */ - error = llcache_object_clone_cache_data(newest, obj, true); - if (error != NSERROR_OK) { - llcache_object_destroy(obj); - return error; - } - - /* Record candidate, so we can fall back if it is still fresh */ - newest->candidate_count++; - obj->candidate = newest; - - /* Attempt to kick-off fetch */ - error = llcache_object_fetch(obj, flags, referer, post, - redirect_count); - if (error != NSERROR_OK) { - newest->candidate_count--; - llcache_object_destroy(obj); - return error; - } - - /* Add new object to cache */ - llcache_object_add_to_list(obj, &llcache->cached_objects); - } else { - /* No object found; create a new one */ - /* Create new object */ - error = llcache_object_new(url, &obj); - if (error != NSERROR_OK) - return error; - -#ifdef LLCACHE_TRACE - LOG(("Not found %p", obj)); -#endif - - /* Attempt to kick-off fetch */ - error = llcache_object_fetch(obj, flags, referer, post, - redirect_count); - if (error != NSERROR_OK) { - llcache_object_destroy(obj); - return error; - } - - /* Add new object to cache */ - llcache_object_add_to_list(obj, &llcache->cached_objects); - } - - *result = obj; - - return NSERROR_OK; -} - -/** - * Determine if an object is still fresh - * - * \param object Object to consider - * \return True if object is still fresh, false otherwise - */ -bool llcache_object_is_fresh(const llcache_object *object) -{ - const llcache_cache_control *cd = &object->cache; - int current_age, freshness_lifetime; - time_t now = time(NULL); - - /* Calculate staleness of cached object as per RFC 2616 13.2.3/13.2.4 */ - current_age = max(0, (cd->res_time - cd->date)); - current_age = max(current_age, (cd->age == INVALID_AGE) ? 0 : cd->age); - current_age += cd->res_time - cd->req_time + now - cd->res_time; - - /* Determine freshness lifetime of this object */ - if (cd->max_age != INVALID_AGE) - freshness_lifetime = cd->max_age; - else if (cd->expires != 0) - freshness_lifetime = cd->expires - cd->date; - else if (cd->last_modified != 0) - freshness_lifetime = (now - cd->last_modified) / 10; - else - freshness_lifetime = 0; - -#ifdef LLCACHE_TRACE - LOG(("%p: (%d > %d || %d != %d)", object, - freshness_lifetime, current_age, - object->fetch.state, LLCACHE_FETCH_COMPLETE)); -#endif - - /* The object is fresh if: - * - * it was not forbidden from being returned from the cache - * unvalidated (i.e. the response contained a no-cache directive) - * - * and: - * - * its current age is within the freshness lifetime - * or if we're still fetching the object - */ - return (cd->no_cache == LLCACHE_VALIDATE_FRESH && - (freshness_lifetime > current_age || - object->fetch.state != LLCACHE_FETCH_COMPLETE)); -} - -/** - * Update an object's cache state - * - * \param object Object to update cache for - * \return NSERROR_OK. - */ -nserror llcache_object_cache_update(llcache_object *object) -{ - if (object->cache.date == 0) - object->cache.date = time(NULL); - - return NSERROR_OK; -} - -/** - * Clone an object's cache data - * - * \param source Source object containing cache data to clone - * \param destination Destination object to clone cache data into - * \param deep Whether to deep-copy the data or not - * \return NSERROR_OK on success, appropriate error otherwise - * - * \post If \a deep is false, then any pointers in \a source will be set to NULL - */ -nserror llcache_object_clone_cache_data(llcache_object *source, - llcache_object *destination, bool deep) -{ - /* ETag must be first, as it can fail when deep cloning */ - if (source->cache.etag != NULL) { - char *etag = source->cache.etag; - - if (deep) { - /* Copy the etag */ - etag = strdup(source->cache.etag); - if (etag == NULL) - return NSERROR_NOMEM; - } else { - /* Destination takes ownership */ - source->cache.etag = NULL; - } - - if (destination->cache.etag != NULL) - free(destination->cache.etag); - - destination->cache.etag = etag; - } - - destination->cache.req_time = source->cache.req_time; - destination->cache.res_time = source->cache.res_time; - - if (source->cache.date != 0) - destination->cache.date = source->cache.date; - - if (source->cache.expires != 0) - destination->cache.expires = source->cache.expires; - - if (source->cache.age != INVALID_AGE) - destination->cache.age = source->cache.age; - - if (source->cache.max_age != INVALID_AGE) - destination->cache.max_age = source->cache.max_age; - - if (source->cache.no_cache != LLCACHE_VALIDATE_FRESH) - destination->cache.no_cache = source->cache.no_cache; - - if (source->cache.last_modified != 0) - destination->cache.last_modified = source->cache.last_modified; - - return NSERROR_OK; -} - -/** - * Kick-off a fetch for an object - * - * \param object Object to fetch - * \param flags Fetch flags - * \param referer Referring URL, or NULL for none - * \param post POST data, or NULL for GET - * \param redirect_count Number of redirects followed so far - * \return NSERROR_OK on success, appropriate error otherwise - * - * \pre object::url must contain the URL to fetch - * \pre If there is a freshness validation candidate, - * object::candidate and object::cache must be filled in - * \pre There must not be a fetch in progress for \a object - */ -nserror llcache_object_fetch(llcache_object *object, uint32_t flags, - nsurl *referer, const llcache_post_data *post, - uint32_t redirect_count) -{ - nserror error; - nsurl *referer_clone = NULL; - llcache_post_data *post_clone = NULL; - -#ifdef LLCACHE_TRACE - LOG(("Starting fetch for %p", object)); -#endif - - if (post != NULL) { - error = llcache_post_data_clone(post, &post_clone); - if (error != NSERROR_OK) - return error; - } - - if (referer != NULL) - referer_clone = nsurl_ref(referer); - - object->fetch.flags = flags; - object->fetch.referer = referer_clone; - object->fetch.post = post_clone; - object->fetch.redirect_count = redirect_count; - - return llcache_object_refetch(object); -} - -/** - * (Re)fetch an object - * - * \param object Object to refetch - * \return NSERROR_OK on success, appropriate error otherwise - * - * \pre The fetch parameters in object->fetch must be populated - */ -nserror llcache_object_refetch(llcache_object *object) -{ - const char *urlenc = NULL; - struct fetch_multipart_data *multipart = NULL; - char **headers = NULL; - int header_idx = 0; - - if (object->fetch.post != NULL) { - if (object->fetch.post->type == LLCACHE_POST_URL_ENCODED) - urlenc = object->fetch.post->data.urlenc; - else - multipart = object->fetch.post->data.multipart; - } - - /* Generate cache-control headers */ - headers = malloc(3 * sizeof(char *)); - if (headers == NULL) - return NSERROR_NOMEM; - - if (object->cache.etag != NULL) { - const size_t len = SLEN("If-None-Match: ") + - strlen(object->cache.etag) + 1; - - headers[header_idx] = malloc(len); - if (headers[header_idx] == NULL) { - free(headers); - return NSERROR_NOMEM; - } - - snprintf(headers[header_idx], len, "If-None-Match: %s", - object->cache.etag); - - header_idx++; - } - if (object->cache.date != 0) { - /* Maximum length of an RFC 1123 date is 29 bytes */ - const size_t len = SLEN("If-Modified-Since: ") + 29 + 1; - - headers[header_idx] = malloc(len); - if (headers[header_idx] == NULL) { - while (--header_idx >= 0) - free(headers[header_idx]); - free(headers); - return NSERROR_NOMEM; - } - - snprintf(headers[header_idx], len, "If-Modified-Since: %s", - rfc1123_date(object->cache.date)); - - header_idx++; - } - headers[header_idx] = NULL; - - /* Reset cache control data */ - llcache_invalidate_cache_control_data(object); - object->cache.req_time = time(NULL); - - /* Reset fetch state */ - object->fetch.state = LLCACHE_FETCH_INIT; - -#ifdef LLCACHE_TRACE - LOG(("Refetching %p", object)); -#endif - - /* Kick off fetch */ - object->fetch.fetch = fetch_start(object->url, object->fetch.referer, - llcache_fetch_callback, object, - object->fetch.flags & LLCACHE_RETRIEVE_NO_ERROR_PAGES, - urlenc, multipart, - object->fetch.flags & LLCACHE_RETRIEVE_VERIFIABLE, - (const char **) headers); - - /* Clean up cache-control headers */ - while (--header_idx >= 0) - free(headers[header_idx]); - free(headers); - - /* Did we succeed in creating a fetch? */ - if (object->fetch.fetch == NULL) - return NSERROR_NOMEM; - - return NSERROR_OK; -} - -/** - * Create a new low-level cache object - * - * \param url URL of object to create - * \param result Pointer to location to receive result - * \return NSERROR_OK on success, appropriate error otherwise - */ -nserror llcache_object_new(nsurl *url, llcache_object **result) -{ - llcache_object *obj = calloc(1, sizeof(llcache_object)); - if (obj == NULL) - return NSERROR_NOMEM; - -#ifdef LLCACHE_TRACE - LOG(("Created object %p (%s)", obj, nsurl_access(url))); -#endif - - obj->url = nsurl_ref(url); - - *result = obj; - - return NSERROR_OK; -} - -/** - * Destroy a low-level cache object - * - * \param object Object to destroy - * \return NSERROR_OK on success, appropriate error otherwise - * - * \pre Object is detached from cache list - * \pre Object has no users - * \pre Object is not a candidate (i.e. object::candidate_count == 0) - */ -nserror llcache_object_destroy(llcache_object *object) -{ - size_t i; - -#ifdef LLCACHE_TRACE - LOG(("Destroying object %p", object)); -#endif - - nsurl_unref(object->url); - free(object->source_data); - - if (object->fetch.fetch != NULL) { - fetch_abort(object->fetch.fetch); - object->fetch.fetch = NULL; - } - - if (object->fetch.referer != NULL) - nsurl_unref(object->fetch.referer); - - if (object->fetch.post != NULL) { - if (object->fetch.post->type == LLCACHE_POST_URL_ENCODED) { - free(object->fetch.post->data.urlenc); - } else { - fetch_multipart_data_destroy( - object->fetch.post->data.multipart); - } - - free(object->fetch.post); - } - - free(object->cache.etag); - - for (i = 0; i < object->num_headers; i++) { - free(object->headers[i].name); - free(object->headers[i].value); - } - free(object->headers); - - free(object); - - return NSERROR_OK; -} - -/** - * Add a user to a low-level cache object - * - * \param object Object to add user to - * \param user User to add - * \return NSERROR_OK. - */ -nserror llcache_object_add_user(llcache_object *object, - llcache_object_user *user) -{ - assert(user->next == NULL); - assert(user->prev == NULL); - - user->handle->object = object; - - user->prev = NULL; - user->next = object->users; - - if (object->users != NULL) - object->users->prev = user; - object->users = user; - -#ifdef LLCACHE_TRACE - LOG(("Adding user %p to %p", user, object)); -#endif - - return NSERROR_OK; -} - -/** - * Remove a user from a low-level cache object - * - * \param object Object to remove user from - * \param user User to remove - * \return NSERROR_OK. - */ -nserror llcache_object_remove_user(llcache_object *object, - llcache_object_user *user) -{ - assert(user != NULL); - assert(object != NULL); - assert(object->users != NULL); - assert(user->handle == NULL || user->handle->object == object); - assert((user->prev != NULL) || (object->users == user)); - - if (user == object->users) - object->users = user->next; - else - user->prev->next = user->next; - - if (user->next != NULL) - user->next->prev = user->prev; - - user->next = user->prev = NULL; - -#ifdef LLCACHE_TRACE - LOG(("Removing user %p from %p", user, object)); -#endif - - return NSERROR_OK; -} - -/** - * Find a user of a low-level cache object - * - * \param handle External cache handle to search for - * \return Pointer to corresponding user, or NULL if not found - */ -llcache_object_user *llcache_object_find_user(const llcache_handle *handle) -{ - llcache_object_user *user; - - assert(handle->object != NULL); - - for (user = handle->object->users; user != NULL; user = user->next) { - if (user->handle == handle) - break; - } - - return user; -} - -/** - * Add a low-level cache object to a cache list - * - * \param object Object to add - * \param list List to add to - * \return NSERROR_OK - */ -nserror llcache_object_add_to_list(llcache_object *object, - llcache_object **list) -{ - object->prev = NULL; - object->next = *list; - - if (*list != NULL) - (*list)->prev = object; - *list = object; - - return NSERROR_OK; -} - -/** - * Remove a low-level cache object from a cache list - * - * \param object Object to remove - * \param list List to remove from - * \return NSERROR_OK - */ -nserror llcache_object_remove_from_list(llcache_object *object, - llcache_object **list) -{ - if (object == *list) - *list = object->next; - else - object->prev->next = object->next; - - if (object->next != NULL) - object->next->prev = object->prev; - - return NSERROR_OK; -} - -/** - * Determine if a low-level cache object resides in a given list - * - * \param object Object to search for - * \param list List to search in - * \return True if object resides in list, false otherwise - */ -bool llcache_object_in_list(const llcache_object *object, - const llcache_object *list) -{ - while (list != NULL) { - if (list == object) - break; - - list = list->next; - } - - return list != NULL; -} - -/** - * Notify users of an object's current state - * - * \param object Object to notify users about - * \return NSERROR_OK on success, appropriate error otherwise - */ -nserror llcache_object_notify_users(llcache_object *object) -{ - nserror error; - llcache_object_user *user, *next_user; - llcache_event event; - -#ifdef LLCACHE_TRACE - bool emitted_notify = false; -#endif - - /** - * State transitions and event emission for users. - * Rows: user state. Cols: object state. - * - * User\Obj INIT HEADERS DATA COMPLETE - * INIT - T T* T* - * HEADERS - - T T* - * DATA - - M T - * COMPLETE - - - - - * - * T => transition user to object state - * M => no transition required, but may need to emit event - * - * The transitions marked with an asterisk can be removed by moving - * the user context into the subsequent state and then reevaluating. - * - * Events are issued as follows: - * - * HAD_HEADERS: on transition from HEADERS -> DATA state - * HAD_DATA : in DATA state, whenever there's new source data - * DONE : on transition from DATA -> COMPLETE state - */ - - for (user = object->users; user != NULL; user = next_user) { - /* Emit necessary events to bring the user up-to-date */ - llcache_handle *handle = user->handle; - const llcache_fetch_state objstate = object->fetch.state; - - /* Flag that this user is the current iteration target - * in case the client attempts to destroy it underneath us */ - user->iterator_target = true; - - /* A note on the computation of next_user: - * - * Within this loop, we may make a number of calls to - * client code. Our contract with clients is that they - * can do whatever they like from within their callback - * handlers. This is so that we limit the pain of - * reentrancy to this module alone. - * - * One of the things a client can do from within its - * callback handler is to remove users from this object's - * user list. In the common case, the user they attempt - * to remove is the current iteration target, and we - * already protect against that causing problems here. - * However, no such protection exists if the client - * attempts to remove other users from this object's - * user list. - * - * Therefore, we cannot compute next_user up-front - * and expect it to remain valid across calls to - * client code (as the identity of the next user - * in the list may change underneath us). Instead, - * we must compute next_user at the point where we - * are about to cause another iteration of this loop - * (i.e. at the very end, and also at the points where - * continue is used) - */ - -#ifdef LLCACHE_TRACE - if (handle->state != objstate) { - if (emitted_notify == false) { - LOG(("Notifying users of %p", object)); - emitted_notify = true; - } - - LOG(("User %p state: %d Object state: %d", - user, handle->state, objstate)); - } -#endif - - /* User: INIT, Obj: HEADERS, DATA, COMPLETE => User->HEADERS */ - if (handle->state == LLCACHE_FETCH_INIT && - objstate > LLCACHE_FETCH_INIT) { - handle->state = LLCACHE_FETCH_HEADERS; - } - - /* User: HEADERS, Obj: DATA, COMPLETE => User->DATA */ - if (handle->state == LLCACHE_FETCH_HEADERS && - objstate > LLCACHE_FETCH_HEADERS) { - handle->state = LLCACHE_FETCH_DATA; - - /* Emit HAD_HEADERS event */ - event.type = LLCACHE_EVENT_HAD_HEADERS; - - error = handle->cb(handle, &event, handle->pw); - - if (user->queued_for_delete) { - next_user = user->next; - llcache_object_remove_user(object, user); - llcache_object_user_destroy(user); - - if (error != NSERROR_OK) - return error; - - continue; - } else if (error == NSERROR_NEED_DATA) { - /* User requested replay */ - handle->state = LLCACHE_FETCH_HEADERS; - - /* Continue with the next user -- we'll - * reemit the event next time round */ - user->iterator_target = false; - next_user = user->next; - continue; - } else if (error != NSERROR_OK) { - user->iterator_target = false; - return error; - } - } - - /* User: DATA, Obj: DATA, COMPLETE, more source available */ - if (handle->state == LLCACHE_FETCH_DATA && - objstate >= LLCACHE_FETCH_DATA && - object->source_len > handle->bytes) { - size_t orig_handle_read; - - /* Construct HAD_DATA event */ - event.type = LLCACHE_EVENT_HAD_DATA; - event.data.data.buf = - object->source_data + handle->bytes; - event.data.data.len = - object->source_len - handle->bytes; - - /* Update record of last byte emitted */ - if (object->fetch.flags & - LLCACHE_RETRIEVE_STREAM_DATA) { - /* Streaming, so reset to zero to - * minimise amount of cached source data. - * Additionally, we don't support replay - * when streaming. */ - orig_handle_read = 0; - handle->bytes = object->source_len = 0; - } else { - orig_handle_read = handle->bytes; - handle->bytes = object->source_len; - } - - /* Emit event */ - error = handle->cb(handle, &event, handle->pw); - if (user->queued_for_delete) { - next_user = user->next; - llcache_object_remove_user(object, user); - llcache_object_user_destroy(user); - - if (error != NSERROR_OK) - return error; - - continue; - } else if (error == NSERROR_NEED_DATA) { - /* User requested replay */ - handle->bytes = orig_handle_read; - - /* Continue with the next user -- we'll - * reemit the data next time round */ - user->iterator_target = false; - next_user = user->next; - continue; - } else if (error != NSERROR_OK) { - user->iterator_target = false; - return error; - } - } - - /* User: DATA, Obj: COMPLETE => User->COMPLETE */ - if (handle->state == LLCACHE_FETCH_DATA && - objstate > LLCACHE_FETCH_DATA) { - handle->state = LLCACHE_FETCH_COMPLETE; - - /* Emit DONE event */ - event.type = LLCACHE_EVENT_DONE; - - error = handle->cb(handle, &event, handle->pw); - if (user->queued_for_delete) { - next_user = user->next; - llcache_object_remove_user(object, user); - llcache_object_user_destroy(user); - - if (error != NSERROR_OK) - return error; - - continue; - } else if (error == NSERROR_NEED_DATA) { - /* User requested replay */ - handle->state = LLCACHE_FETCH_DATA; - - /* Continue with the next user -- we'll - * reemit the event next time round */ - user->iterator_target = false; - next_user = user->next; - continue; - } else if (error != NSERROR_OK) { - user->iterator_target = false; - return error; - } - } - - /* No longer the target of an iterator */ - user->iterator_target = false; - - next_user = user->next; - } - - return NSERROR_OK; -} - -/** - * Make a snapshot of the current state of an llcache_object. - * - * This has the side-effect of the new object being non-cacheable, - * also not-fetching and not a candidate for any other object. - * - * Also note that this new object has no users and at least one - * should be assigned to it before llcache_clean is entered or it - * will be immediately cleaned up. - * - * \param object The object to take a snapshot of - * \param snapshot Pointer to receive snapshot of \a object - * \return NSERROR_OK on success, appropriate error otherwise - */ -nserror llcache_object_snapshot(llcache_object *object, - llcache_object **snapshot) -{ - llcache_object *newobj; - nserror error; - - error = llcache_object_new(object->url, &newobj); - - if (error != NSERROR_OK) - return error; - - newobj->has_query = object->has_query; - - newobj->source_alloc = newobj->source_len = object->source_len; - - if (object->source_len > 0) { - newobj->source_data = malloc(newobj->source_alloc); - if (newobj->source_data == NULL) { - llcache_object_destroy(newobj); - return NSERROR_NOMEM; - } - memcpy(newobj->source_data, object->source_data, - newobj->source_len); - } - - if (object->num_headers > 0) { - newobj->headers = calloc(sizeof(llcache_header), - object->num_headers); - if (newobj->headers == NULL) { - llcache_object_destroy(newobj); - return NSERROR_NOMEM; - } - while (newobj->num_headers < object->num_headers) { - llcache_header *nh = - &(newobj->headers[newobj->num_headers]); - llcache_header *oh = - &(object->headers[newobj->num_headers]); - newobj->num_headers += 1; - nh->name = strdup(oh->name); - nh->value = strdup(oh->value); - if (nh->name == NULL || nh->value == NULL) { - llcache_object_destroy(newobj); - return NSERROR_NOMEM; - } - } - } - - newobj->fetch.state = LLCACHE_FETCH_COMPLETE; - - *snapshot = newobj; - - return NSERROR_OK; -} - -/** - * Attempt to clean the cache - */ -void llcache_clean(void) -{ - llcache_object *object, *next; - uint32_t llcache_size = 0; - -#ifdef LLCACHE_TRACE - LOG(("Attempting cache clean")); -#endif - - /* Candidates for cleaning are (in order of priority): - * - * 1) Uncacheable objects with no users - * 2) Stale cacheable objects with no users or pending fetches - * 3) Fresh cacheable objects with no users or pending fetches - */ - - /* 1) Uncacheable objects with no users or fetches */ - for (object = llcache->uncached_objects; object != NULL; object = next) { - next = object->next; - - /* The candidate count of uncacheable objects is always 0 */ - if (object->users == NULL && object->candidate_count == 0 && - object->fetch.fetch == NULL && - object->fetch.outstanding_query == false) { -#ifdef LLCACHE_TRACE - LOG(("Found victim %p", object)); -#endif - llcache_object_remove_from_list(object, - &llcache->uncached_objects); - llcache_object_destroy(object); - } else { - llcache_size += object->source_len + sizeof(*object); - } - } - - /* 2) Stale cacheable objects with no users or pending fetches */ - for (object = llcache->cached_objects; object != NULL; object = next) { - next = object->next; - - if (object->users == NULL && object->candidate_count == 0 && - llcache_object_is_fresh(object) == false && - object->fetch.fetch == NULL && - object->fetch.outstanding_query == false) { -#ifdef LLCACHE_TRACE - LOG(("Found victim %p", object)); -#endif - llcache_object_remove_from_list(object, - &llcache->cached_objects); - llcache_object_destroy(object); - } else { - llcache_size += object->source_len + sizeof(*object); - } - } - - if (llcache->limit < llcache_size) { - /* 3) Fresh cacheable objects with - * no users or pending fetches */ - for (object = llcache->cached_objects; object != NULL; - object = next) { - next = object->next; - - if (object->users == NULL && - object->candidate_count == 0 && - object->fetch.fetch == NULL && - object->fetch.outstanding_query == - false) { -#ifdef LLCACHE_TRACE - LOG(("Found victim %p", object)); -#endif - llcache_size -= - object->source_len + sizeof(*object); - - llcache_object_remove_from_list(object, - &llcache->cached_objects); - llcache_object_destroy(object); - } - } - } - -#ifdef LLCACHE_TRACE - LOG(("Size: %u", llcache_size)); -#endif - -} - -/** - * Clone a POST data object - * - * \param orig Object to clone - * \param clone Pointer to location to receive clone - * \return NSERROR_OK on success, appropriate error otherwise - */ -nserror llcache_post_data_clone(const llcache_post_data *orig, - llcache_post_data **clone) -{ - llcache_post_data *post_clone; - - post_clone = calloc(1, sizeof(llcache_post_data)); - if (post_clone == NULL) - return NSERROR_NOMEM; - - post_clone->type = orig->type; - - /* Deep-copy the type-specific data */ - if (orig->type == LLCACHE_POST_URL_ENCODED) { - post_clone->data.urlenc = strdup(orig->data.urlenc); - if (post_clone->data.urlenc == NULL) { - free(post_clone); - - return NSERROR_NOMEM; - } - } else { - post_clone->data.multipart = fetch_multipart_data_clone( - orig->data.multipart); - if (post_clone->data.multipart == NULL) { - free(post_clone); - - return NSERROR_NOMEM; - } - } - - *clone = post_clone; - - return NSERROR_OK; -} - -/** - * Handle a query response - * - * \param proceed Whether to proceed with fetch - * \param cbpw Our context for query - * \return NSERROR_OK on success, appropriate error otherwise - */ -nserror llcache_query_handle_response(bool proceed, void *cbpw) -{ - llcache_event event; - llcache_object *object = cbpw; - - object->fetch.outstanding_query = false; - - /* Refetch, using existing fetch parameters, if client allows us to */ - if (proceed) - return llcache_object_refetch(object); - - /* Invalidate cache-control data */ - llcache_invalidate_cache_control_data(object); - - /* Mark it complete */ - object->fetch.state = LLCACHE_FETCH_COMPLETE; - - /* Inform client(s) that object fetch failed */ - event.type = LLCACHE_EVENT_ERROR; - /** \todo More appropriate error message */ - event.data.msg = messages_get("FetchFailed"); - - return llcache_send_event_to_users(object, &event); -} - -/** - * Handler for fetch events - * - * \param msg Fetch event - * \param p Our private data - */ -void llcache_fetch_callback(const fetch_msg *msg, void *p) -{ - nserror error = NSERROR_OK; - llcache_object *object = p; - llcache_event event; - -#ifdef LLCACHE_TRACE - LOG(("Fetch event %d for %p", msg->type, object)); -#endif - - switch (msg->type) { - case FETCH_HEADER: - /* Received a fetch header */ - object->fetch.state = LLCACHE_FETCH_HEADERS; - - error = llcache_fetch_process_header(object, - msg->data.header_or_data.buf, - msg->data.header_or_data.len); - break; - - /* 3xx responses */ - case FETCH_REDIRECT: - /* Request resulted in a redirect */ - - /* Release candidate, if any */ - if (object->candidate != NULL) { - object->candidate->candidate_count--; - object->candidate = NULL; - } - - error = llcache_fetch_redirect(object, - msg->data.redirect, &object); - break; - case FETCH_NOTMODIFIED: - /* Conditional request determined that cached object is fresh */ - error = llcache_fetch_notmodified(object, &object); - break; - - /* Normal 2xx state machine */ - case FETCH_DATA: - /* Received some data */ - if (object->fetch.state != LLCACHE_FETCH_DATA) { - /* On entry into this state, check if we need to - * invalidate the cache control data. We are guaranteed - * to have received all response headers. - * - * There are two cases in which we want to suppress - * cacheing of an object: - * - * 1) The HTTP response code is not 200 or 203 - * 2) The request URI had a query string and the - * response headers did not provide an explicit - * object expiration time. - */ - long http_code = fetch_http_code(object->fetch.fetch); - - if ((http_code != 200 && http_code != 203) || - (object->has_query && - (object->cache.max_age == INVALID_AGE && - object->cache.expires == 0))) { - /* Invalidate cache control data */ - llcache_invalidate_cache_control_data(object); - } - - /* Release candidate, if any */ - if (object->candidate != NULL) { - object->candidate->candidate_count--; - object->candidate = NULL; - } - } - - object->fetch.state = LLCACHE_FETCH_DATA; - - error = llcache_fetch_process_data(object, - msg->data.header_or_data.buf, - msg->data.header_or_data.len); - break; - case FETCH_FINISHED: - /* Finished fetching */ - { - uint8_t *temp; - - object->fetch.state = LLCACHE_FETCH_COMPLETE; - object->fetch.fetch = NULL; - - /* Shrink source buffer to required size */ - temp = realloc(object->source_data, - object->source_len); - /* If source_len is 0, then temp may be NULL */ - if (temp != NULL || object->source_len == 0) { - object->source_data = temp; - object->source_alloc = object->source_len; - } - - llcache_object_cache_update(object); - } - break; - - /* Out-of-band information */ - case FETCH_ERROR: - /* An error occurred while fetching */ - /* The fetch has has already been cleaned up by the fetcher */ - object->fetch.state = LLCACHE_FETCH_COMPLETE; - object->fetch.fetch = NULL; - - /* Release candidate, if any */ - if (object->candidate != NULL) { - object->candidate->candidate_count--; - object->candidate = NULL; - } - - /* Invalidate cache control data */ - llcache_invalidate_cache_control_data(object); - - /** \todo Consider using errorcode for something */ - - event.type = LLCACHE_EVENT_ERROR; - event.data.msg = msg->data.error; - - error = llcache_send_event_to_users(object, &event); - - break; - case FETCH_PROGRESS: - /* Progress update */ - event.type = LLCACHE_EVENT_PROGRESS; - event.data.msg = msg->data.progress; - - error = llcache_send_event_to_users(object, &event); - - break; - - /* Events requiring action */ - case FETCH_AUTH: - /* Need Authentication */ - - /* Release candidate, if any */ - if (object->candidate != NULL) { - object->candidate->candidate_count--; - object->candidate = NULL; - } - - error = llcache_fetch_auth(object, msg->data.auth.realm); - break; - case FETCH_CERT_ERR: - /* Something went wrong when validating TLS certificates */ - - /* Release candidate, if any */ - if (object->candidate != NULL) { - object->candidate->candidate_count--; - object->candidate = NULL; - } - - error = llcache_fetch_cert_error(object, - msg->data.cert_err.certs, - msg->data.cert_err.num_certs); - break; - } - - /* Deal with any errors reported by event handlers */ - if (error != NSERROR_OK) { - if (object->fetch.fetch != NULL) { - fetch_abort(object->fetch.fetch); - object->fetch.fetch = NULL; - - /* Invalidate cache control data */ - llcache_invalidate_cache_control_data(object); - - object->fetch.state = LLCACHE_FETCH_COMPLETE; - } - return; - } -} - -/** - * Handle FETCH_REDIRECT event - * - * \param object Object being redirected - * \param target Target of redirect (may be relative) - * \param replacement Pointer to location to receive replacement object - * \return NSERROR_OK on success, appropriate error otherwise - */ -nserror llcache_fetch_redirect(llcache_object *object, const char *target, - llcache_object **replacement) -{ - nserror error; - llcache_object *dest; - llcache_object_user *user, *next; - const llcache_post_data *post = object->fetch.post; - nsurl *url; - lwc_string *scheme; - lwc_string *object_scheme; - bool match; - /* Extract HTTP response code from the fetch object */ - long http_code = fetch_http_code(object->fetch.fetch); - - /* Abort fetch for this object */ - fetch_abort(object->fetch.fetch); - object->fetch.fetch = NULL; - - /* Invalidate the cache control data */ - llcache_invalidate_cache_control_data(object); - - /* And mark it complete */ - object->fetch.state = LLCACHE_FETCH_COMPLETE; - - /* Forcibly stop redirecting if we've followed too many redirects */ -#define REDIRECT_LIMIT 10 - if (object->fetch.redirect_count > REDIRECT_LIMIT) { - llcache_event event; - - LOG(("Too many nested redirects")); - - event.type = LLCACHE_EVENT_ERROR; - event.data.msg = messages_get("BadRedirect"); - - return llcache_send_event_to_users(object, &event); - } -#undef REDIRECT_LIMIT - - /* Make target absolute */ - error = nsurl_join(object->url, target, &url); - if (error != NSERROR_OK) - return error; - - /* Reject attempts to redirect from unvalidated to validated schemes - * A "validated" scheme is one over which we have some guarantee that - * the source is trustworthy. */ - object_scheme = nsurl_get_component(object->url, NSURL_SCHEME); - scheme = nsurl_get_component(url, NSURL_SCHEME); - - /* resource: and about: are allowed to redirect anywhere */ - if ((lwc_string_isequal(object_scheme, llcache_resource_lwc, - &match) == lwc_error_ok && match == false) && - (lwc_string_isequal(object_scheme, llcache_about_lwc, - &match) == lwc_error_ok && match == false)) { - /* file, about and resource are not valid redirect targets */ - if ((lwc_string_isequal(object_scheme, llcache_file_lwc, - &match) == lwc_error_ok && match == true) || - (lwc_string_isequal(object_scheme, llcache_about_lwc, - &match) == lwc_error_ok && match == true) || - (lwc_string_isequal(object_scheme, llcache_resource_lwc, - &match) == lwc_error_ok && match == true)) { - lwc_string_unref(object_scheme); - lwc_string_unref(scheme); - nsurl_unref(url); - return NSERROR_OK; - } - } - - lwc_string_unref(scheme); - lwc_string_unref(object_scheme); - - /* Bail out if we've no way of handling this URL */ - if (fetch_can_fetch(url) == false) { - nsurl_unref(url); - return NSERROR_OK; - } - - if (http_code == 301 || http_code == 302 || http_code == 303) { - /* 301, 302, 303 redirects are all unconditional GET requests */ - post = NULL; - } else if (http_code != 307 || post != NULL) { - /** \todo 300, 305, 307 with POST */ - nsurl_unref(url); - return NSERROR_OK; - } - - /* Attempt to fetch target URL */ - error = llcache_object_retrieve(url, object->fetch.flags, - object->fetch.referer, post, - object->fetch.redirect_count + 1, &dest); - - /* No longer require url */ - nsurl_unref(url); - - if (error != NSERROR_OK) - return error; - - /* Move user(s) to replacement object */ - for (user = object->users; user != NULL; user = next) { - next = user->next; - - llcache_object_remove_user(object, user); - llcache_object_add_user(dest, user); - } - - /* Dest is now our object */ - *replacement = dest; - - return NSERROR_OK; -} - -/** - * Handle FETCH_NOTMODIFIED event - * - * \param object Object to process - * \param replacement Pointer to location to receive replacement object - * \return NSERROR_OK. - */ -nserror llcache_fetch_notmodified(llcache_object *object, - llcache_object **replacement) -{ - /* There may be no candidate if the server erroneously responded - * to an unconditional request with a 304 Not Modified response. - * In this case, we simply retain the initial object, having - * invalidated it and marked it as complete. - */ - if (object->candidate != NULL) { - llcache_object_user *user, *next; - - /* Move user(s) to candidate content */ - for (user = object->users; user != NULL; user = next) { - next = user->next; - - llcache_object_remove_user(object, user); - llcache_object_add_user(object->candidate, user); - } - - /* Candidate is no longer a candidate for us */ - object->candidate->candidate_count--; - - /* Clone our cache control data into the candidate */ - llcache_object_clone_cache_data(object, object->candidate, - false); - /* Bring candidate's cache data up to date */ - llcache_object_cache_update(object->candidate); - /* Revert no-cache to normal, if required */ - if (object->candidate->cache.no_cache == - LLCACHE_VALIDATE_ONCE) { - object->candidate->cache.no_cache = - LLCACHE_VALIDATE_FRESH; - } - - /* Candidate is now our object */ - *replacement = object->candidate; - object->candidate = NULL; - } else { - /* There was no candidate: retain object */ - *replacement = object; - } - - /* Ensure fetch has stopped */ - fetch_abort(object->fetch.fetch); - object->fetch.fetch = NULL; - - /* Invalidate our cache-control data */ - llcache_invalidate_cache_control_data(object); - - /* Mark it complete */ - object->fetch.state = LLCACHE_FETCH_COMPLETE; - - /* Old object will be flushed from the cache on the next poll */ - - return NSERROR_OK; -} - -/** - * Split a fetch header into name and value - * - * \param data Header string - * \param len Byte length of header - * \param name Pointer to location to receive header name - * \param value Pointer to location to receive header value - * \return NSERROR_OK on success, appropriate error otherwise - */ -nserror llcache_fetch_split_header(const uint8_t *data, size_t len, - char **name, char **value) -{ - char *n, *v; - const uint8_t *colon; - - /* Find colon */ - colon = (const uint8_t *) strchr((const char *) data, ':'); - if (colon == NULL) { - /* Failed, assume a key with no value */ - n = strdup((const char *) data); - if (n == NULL) - return NSERROR_NOMEM; - - v = strdup(""); - if (v == NULL) { - free(n); - return NSERROR_NOMEM; - } - } else { - /* Split header into name & value */ - - /* Strip leading whitespace from name */ - while (data[0] == ' ' || data[0] == '\t' || - data[0] == '\r' || data[0] == '\n') { - data++; - } - - /* Strip trailing whitespace from name */ - while (colon > data && (colon[-1] == ' ' || - colon[-1] == '\t' || colon[-1] == '\r' || - colon[-1] == '\n')) - colon--; - - n = strndup((const char *) data, colon - data); - if (n == NULL) - return NSERROR_NOMEM; - - /* Find colon again */ - while (*colon != ':') { - colon++; - } - - /* Skip over colon and any subsequent whitespace */ - do { - colon++; - } while (*colon == ' ' || *colon == '\t' || - *colon == '\r' || *colon == '\n'); - - /* Strip trailing whitespace from value */ - while (len > 0 && (data[len - 1] == ' ' || - data[len - 1] == '\t' || - data[len - 1] == '\r' || - data[len - 1] == '\n')) { - len--; - } - - v = strndup((const char *) colon, len - (colon - data)); - if (v == NULL) { - free(n); - return NSERROR_NOMEM; - } - } - - *name = n; - *value = v; - - return NSERROR_OK; -} - -/** - * Parse a fetch header - * - * \param object Object to parse header for - * \param data Header string - * \param len Byte length of header - * \param name Pointer to location to receive header name - * \param value Pointer to location to receive header value - * \return NSERROR_OK on success, appropriate error otherwise - * - * \note This function also has the side-effect of updating - * the cache control data for the object if an interesting - * header is encountered - */ -nserror llcache_fetch_parse_header(llcache_object *object, - const uint8_t *data, size_t len, char **name, char **value) -{ - nserror error; - - /* Set fetch response time if not already set */ - if (object->cache.res_time == 0) - object->cache.res_time = time(NULL); - - /* Decompose header into name-value pair */ - error = llcache_fetch_split_header(data, len, name, value); - if (error != NSERROR_OK) - return error; - - /* Parse cache headers to populate cache control data */ -#define SKIP_ST(p) while (*p != '\0' && (*p == ' ' || *p == '\t')) p++ - - if (5 < len && strcasecmp(*name, "Date") == 0) { - /* extract Date header */ - object->cache.date = curl_getdate(*value, NULL); - } else if (4 < len && strcasecmp(*name, "Age") == 0) { - /* extract Age header */ - if ('0' <= **value && **value <= '9') - object->cache.age = atoi(*value); - } else if (8 < len && strcasecmp(*name, "Expires") == 0) { - /* extract Expires header */ - object->cache.expires = curl_getdate(*value, NULL); - } else if (14 < len && strcasecmp(*name, "Cache-Control") == 0) { - /* extract and parse Cache-Control header */ - const char *start = *value; - const char *comma = *value; - - while (*comma != '\0') { - while (*comma != '\0' && *comma != ',') - comma++; - - if (8 < comma - start && (strncasecmp(start, - "no-cache", 8) == 0 || - strncasecmp(start, "no-store", 8) == 0)) - /* When we get a disk cache we should - * distinguish between these two */ - object->cache.no_cache = LLCACHE_VALIDATE_ALWAYS; - else if (7 < comma - start && - strncasecmp(start, "max-age", 7) == 0) { - /* Find '=' */ - while (start < comma && *start != '=') - start++; - - /* Skip over it */ - start++; - - /* Skip whitespace */ - SKIP_ST(start); - - if (start < comma) - object->cache.max_age = atoi(start); - } - - if (*comma != '\0') { - /* Skip past comma */ - comma++; - /* Skip whitespace */ - SKIP_ST(comma); - } - - /* Set start for next token */ - start = comma; - } - } else if (5 < len && strcasecmp(*name, "ETag") == 0) { - /* extract ETag header */ - free(object->cache.etag); - object->cache.etag = strdup(*value); - if (object->cache.etag == NULL) - return NSERROR_NOMEM; - } else if (14 < len && strcasecmp(*name, "Last-Modified") == 0) { - /* extract Last-Modified header */ - object->cache.last_modified = curl_getdate(*value, NULL); - } - -#undef SKIP_ST - - return NSERROR_OK; -} - -/** - * Process a fetch header - * - * \param object Object being fetched - * \param data Header string - * \param len Byte length of header - * \return NSERROR_OK on success, appropriate error otherwise - */ -nserror llcache_fetch_process_header(llcache_object *object, - const uint8_t *data, size_t len) -{ - nserror error; - char *name, *value; - llcache_header *temp; - - /* The headers for multiple HTTP responses may be delivered to us if - * the fetch layer receives a 401 response for which it has - * authentication credentials. This will result in a silent re-request - * after which we'll receive the actual response headers for the - * object we want to fetch (assuming that the credentials were correct - * of course) - * - * Therefore, if the header is an HTTP response start marker, then we - * must discard any headers we've read so far, reset the cache data - * that we might have computed, and start again. - */ - /** \todo Properly parse the response line */ - if (strncmp((const char *) data, "HTTP/", SLEN("HTTP/")) == 0) { - time_t req_time = object->cache.req_time; - - llcache_invalidate_cache_control_data(object); - - /* Restore request time, so we compute object's age correctly */ - object->cache.req_time = req_time; - - llcache_destroy_headers(object); - } - - error = llcache_fetch_parse_header(object, data, len, &name, &value); - if (error != NSERROR_OK) - return error; - - /* Append header data to the object's headers array */ - temp = realloc(object->headers, (object->num_headers + 1) * - sizeof(llcache_header)); - if (temp == NULL) { - free(name); - free(value); - return NSERROR_NOMEM; - } - - object->headers = temp; - - object->headers[object->num_headers].name = name; - object->headers[object->num_headers].value = value; - - object->num_headers++; - - return NSERROR_OK; -} - -/** - * Process a chunk of fetched data - * - * \param object Object being fetched - * \param data Data to process - * \param len Byte length of data - * \return NSERROR_OK on success, appropriate error otherwise. - */ -nserror llcache_fetch_process_data(llcache_object *object, const uint8_t *data, - size_t len) -{ - /* Resize source buffer if it's too small */ - if (object->source_len + len >= object->source_alloc) { - const size_t new_len = object->source_len + len + 64 * 1024; - uint8_t *temp = realloc(object->source_data, new_len); - if (temp == NULL) - return NSERROR_NOMEM; - - object->source_data = temp; - object->source_alloc = new_len; - } - - /* Append this data chunk to source buffer */ - memcpy(object->source_data + object->source_len, data, len); - object->source_len += len; - - return NSERROR_OK; -} - -/** - * Handle an authentication request - * - * \param object Object being fetched - * \param realm Authentication realm - * \return NSERROR_OK on success, appropriate error otherwise. - */ -nserror llcache_fetch_auth(llcache_object *object, const char *realm) -{ - const char *auth; - nserror error = NSERROR_OK; - - /* Abort fetch for this object */ - fetch_abort(object->fetch.fetch); - object->fetch.fetch = NULL; - - /* Invalidate cache-control data */ - llcache_invalidate_cache_control_data(object); - - /* Destroy headers */ - llcache_destroy_headers(object); - - /* If there was no realm, then default to the URL */ - /** \todo If there was no WWW-Authenticate header, use response body */ - if (realm == NULL) - realm = nsurl_access(object->url); - - auth = urldb_get_auth_details(nsurl_access(object->url), realm); - - if (auth == NULL || object->fetch.tried_with_auth == true) { - /* No authentication details, or tried what we had, so ask */ - object->fetch.tried_with_auth = false; - - if (llcache->query_cb != NULL) { - llcache_query query; - - /* Emit query for authentication details */ - query.type = LLCACHE_QUERY_AUTH; - query.url = object->url; - query.data.auth.realm = realm; - - object->fetch.outstanding_query = true; - - error = llcache->query_cb(&query, llcache->query_cb_pw, - llcache_query_handle_response, object); - } else { - llcache_event event; - - /* Mark object complete */ - object->fetch.state = LLCACHE_FETCH_COMPLETE; - - /* Inform client(s) that object fetch failed */ - event.type = LLCACHE_EVENT_ERROR; - /** \todo More appropriate error message */ - event.data.msg = messages_get("FetchFailed"); - - error = llcache_send_event_to_users(object, &event); - } - } else { - /* Flag that we've tried to refetch with credentials, so - * that if the fetch fails again, we ask the user again */ - object->fetch.tried_with_auth = true; - error = llcache_object_refetch(object); - } - - return error; -} - -/** - * Handle a TLS certificate verification failure - * - * \param object Object being fetched - * \param certs Certificate chain - * \param num Number of certificates in chain - * \return NSERROR_OK on success, appropriate error otherwise - */ -nserror llcache_fetch_cert_error(llcache_object *object, - const struct ssl_cert_info *certs, size_t num) -{ - nserror error = NSERROR_OK; - - /* Fetch has been stopped, and destroyed. Invalidate object's pointer */ - object->fetch.fetch = NULL; - - /* Invalidate cache-control data */ - llcache_invalidate_cache_control_data(object); - - if (llcache->query_cb != NULL) { - llcache_query query; - - /* Emit query for TLS */ - query.type = LLCACHE_QUERY_SSL; - query.url = object->url; - query.data.ssl.certs = certs; - query.data.ssl.num = num; - - object->fetch.outstanding_query = true; - - error = llcache->query_cb(&query, llcache->query_cb_pw, - llcache_query_handle_response, object); - } else { - llcache_event event; - - /* Mark object complete */ - object->fetch.state = LLCACHE_FETCH_COMPLETE; - - /* Inform client(s) that object fetch failed */ - event.type = LLCACHE_EVENT_ERROR; - /** \todo More appropriate error message */ - event.data.msg = messages_get("FetchFailed"); - - error = llcache_send_event_to_users(object, &event); - } - - return error; -} -