mirrors/netsurf
1
0
Fork 0
mirror of https://github.com/netsurf-browser/netsurf synced 2024-11-26 00:09:41 +03:00
Code Issues Packages Projects Releases Wiki Activity
0ab6249a64
netsurf/content/llcache.c
Michael Drake f840833723 Low level cache: Squash use of uninitialised value. 
Coverity issue:

CID 1419828:  Memory - corruptions  (UNINIT)
>>>     Using uninitialized value "output_length".
2020-02-24 15:37:20 +00:00

4119 lines
106 KiB
C
Raw Blame History

/*
* Copyright 2009 John-Mark Bell <jmb@netsurf-browser.org>
*
* This file is part of NetSurf, http://www.netsurf-browser.org/
*
* NetSurf is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; version 2 of the License.
*
* NetSurf is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* \file
*
* Low-level resource cache implementation
*
* This is the implementation of the low level cache. This cache
* stores source objects in memory and may use a persistent backing
* store to extend their lifetime.
*
* \todo fix writeout conditions and ordering.
*
* \todo instrument and (auto)tune
*
*/
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <strings.h>
#include <nsutils/time.h>
#include <nsutils/base64.h>
#include "netsurf/inttypes.h"
#include "utils/config.h"
#include "utils/corestrings.h"
#include "utils/log.h"
#include "utils/messages.h"
#include "utils/nsurl.h"
#include "utils/utils.h"
#include "utils/time.h"
#include "utils/http.h"
#include "netsurf/misc.h"
#include "desktop/gui_internal.h"
#include "content/fetch.h"
#include "content/backing_store.h"
#include "content/urldb.h"
/**
* State of a low-level cache object fetch.
*/
typedef enum {
LLCACHE_FETCH_INIT, /**< Initial state, before fetch */
LLCACHE_FETCH_HEADERS, /**< Fetching headers */
LLCACHE_FETCH_DATA, /**< Fetching object data */
LLCACHE_FETCH_COMPLETE /**< Fetch completed */
} llcache_fetch_state;
/**
* Type of low-level cache object.
*/
typedef struct llcache_object llcache_object;
/**
* Handle to low-level cache object.
*/
struct llcache_handle {
llcache_object *object; /**< Pointer to associated object */
llcache_handle_callback cb; /**< Client callback */
void *pw; /**< Client data */
llcache_fetch_state state; /**< Last known state of object fetch */
size_t bytes; /**< Last reported byte count */
};
/**
* Low-level cache object user record.
*/
typedef struct llcache_object_user {
llcache_handle *handle; /**< Handle data for client */
bool iterator_target; /**< This is the iterator target */
bool queued_for_delete; /**< This user is queued for deletion */
struct llcache_object_user *prev; /**< Previous in list */
struct llcache_object_user *next; /**< Next in list */
} llcache_object_user;
/**
* Low-level cache object fetch context.
*/
typedef struct {
uint32_t flags; /**< Fetch flags */
nsurl *referer; /**< Referring URL, or NULL if none */
llcache_post_data *post; /**< POST data, or NULL for GET */
struct fetch *fetch; /**< Fetch handle for this object */
llcache_fetch_state state; /**< Current state of object fetch */
uint32_t redirect_count; /**< Count of redirects followed */
uint32_t retries_remaining; /**< Number of times to retry on timeout */
bool hsts_in_use; /**< Whether HSTS applies to this fetch */
bool tried_with_auth; /**< Whether we've tried with auth */
bool tried_with_tls_downgrade; /**< Whether we've tried TLS <= 1.0 */
bool tainted_tls; /**< Whether the TLS transport is tainted */
} llcache_fetch_ctx;
/**
* Validation control.
*/
typedef enum {
LLCACHE_VALIDATE_FRESH, /**< Only revalidate if not fresh */
LLCACHE_VALIDATE_ALWAYS, /**< Always revalidate */
LLCACHE_VALIDATE_ONCE /**< Revalidate once only */
} llcache_validate;
/**
* Cache control value for invalid age.
*/
#define INVALID_AGE -1
/** Cache control data */
typedef struct {
time_t req_time; /**< Time of request */
time_t res_time; /**< Time of response */
time_t fin_time; /**< Time of request completion */
time_t date; /**< Date: response header */
time_t expires; /**< Expires: response header */
int age; /**< Age: response header */
int max_age; /**< Max-Age Cache-control parameter */
llcache_validate no_cache; /**< No-Cache Cache-control parameter */
char *etag; /**< Etag: response header */
time_t last_modified; /**< Last-Modified: response header */
} llcache_cache_control;
/** Representation of a fetch header */
typedef struct {
char *name; /**< Header name */
char *value; /**< Header value */
} llcache_header;
/** Current status of an object's data */
typedef enum {
LLCACHE_STATE_RAM = 0, /**< source data is stored in RAM only */
LLCACHE_STATE_DISC, /**< source data is stored on disc */
} llcache_store_state;
/**
* Low-level cache object
*
* \todo Consider whether a list is a sane container.
*/
struct llcache_object {
llcache_object *prev; /**< Previous in list */
llcache_object *next; /**< Next in list */
nsurl *url; /**< Post-redirect URL for object */
/** \todo We need a generic dynamic buffer object */
uint8_t *source_data; /**< Source data for object */
size_t source_len; /**< Byte length of source data */
size_t source_alloc; /**< Allocated size of source buffer */
struct cert_chain *chain; /**< Certificate chain from the fetch */
llcache_store_state store_state; /**< where the data for the object is stored */
llcache_object_user *users; /**< List of users */
llcache_fetch_ctx fetch; /**< Fetch context for object */
llcache_cache_control cache; /**< Cache control data for object */
llcache_object *candidate; /**< Object to use, if fetch determines
* that it is still fresh
*/
uint32_t candidate_count; /**< Count of objects this is a
* candidate for
*/
llcache_header *headers; /**< Fetch headers */
size_t num_headers; /**< Number of fetch headers */
/* Instrumentation. These elements are strictly for information
* to improve the cache performance and to provide performance
* metrics. The values are non-authoritative and must not be used to
* determine object lifetime etc.
*/
time_t last_used; /**< time the last user was removed from the object */
};
/**
* Core llcache control context.
*/
struct llcache_s {
/** Head of the low-level cached object list */
llcache_object *cached_objects;
/** Head of the low-level uncached object list */
llcache_object *uncached_objects;
/** The target upper bound for the RAM cache size */
uint32_t limit;
/** The number of fetch attempts we make when timing out */
uint32_t fetch_attempts;
/** Whether or not our users are caught up */
bool all_caught_up;
/* backing store elements */
/**
* The minimum lifetime to consider sending objects to backing
* store.
*/
int minimum_lifetime;
/**
* The time over which to apply the bandwidth calculations in ms
*/
unsigned long time_quantum;
/**
* The minimum bandwidth to allow the backing store to use in
* bytes/second. Below this the backing store will be
* disabled.
*/
size_t minimum_bandwidth;
/**
* The maximum bandwidth to allow the backing store to use in
* bytes/second
*/
size_t maximum_bandwidth;
/**
* Total number of bytes written to backing store.
*/
uint64_t total_written;
/**
* Total number of milliseconds taken to write to backing store.
*/
uint64_t total_elapsed;
};
/** low level cache state */
static struct llcache_s *llcache = NULL;
/* forward referenced callback function */
static void llcache_fetch_callback(const fetch_msg *msg, void *p);
/* forward referenced catch up function */
static void llcache_users_not_caught_up(void);
/******************************************************************************
* Low-level cache internals *
******************************************************************************/
/**
* Create a new object user.
*
* \param cb Callback routine.
* \param pw Private data for callback.
* \param user Pointer to location to receive result.
* \return NSERROR_OK on success, appropriate error otherwise
*/
static nserror llcache_object_user_new(llcache_handle_callback cb, void *pw,
llcache_object_user **user)
{
llcache_handle *h;
llcache_object_user *u;
h = calloc(1, sizeof(llcache_handle));
if (h == NULL) {
return NSERROR_NOMEM;
}
u = calloc(1, sizeof(llcache_object_user));
if (u == NULL) {
free(h);
return NSERROR_NOMEM;
}
h->cb = cb;
h->pw = pw;
u->handle = h;
NSLOG(llcache, DEBUG,
"Created user %p (%p, %p, %p)", u, h, (void *) cb, pw);
*user = u;
return NSERROR_OK;
}
/**
* Destroy an object user
*
* \param user User to destroy
* \return NSERROR_OK on success, appropriate error otherwise
*
* \pre User is not attached to an object
*/
static nserror llcache_object_user_destroy(llcache_object_user *user)
{
NSLOG(llcache, DEBUG, "Destroyed user %p", user);
assert(user->next == NULL);
assert(user->prev == NULL);
if (user->handle != NULL)
free(user->handle);
free(user);
return NSERROR_OK;
}
/**
* Remove a user from a low-level cache object
*
* \param object Object to remove user from
* \param user User to remove
* \return NSERROR_OK.
*/
static nserror llcache_object_remove_user(llcache_object *object,
llcache_object_user *user)
{
assert(user != NULL);
assert(object != NULL);
assert(object->users != NULL);
assert(user->handle == NULL || user->handle->object == object);
assert((user->prev != NULL) || (object->users == user));
if (user == object->users)
object->users = user->next;
else
user->prev->next = user->next;
if (user->next != NULL)
user->next->prev = user->prev;
user->next = user->prev = NULL;
/* record the time the last user was removed from the object */
if (object->users == NULL) {
object->last_used = time(NULL);
}
NSLOG(llcache, DEBUG, "Removing user %p from %p", user, object);
return NSERROR_OK;
}
/**
* Iterate the users of an object, calling their callbacks.
*
* \param object The object to iterate
* \param event The event to pass to the callback.
* \return NSERROR_OK on success, appropriate error otherwise.
*/
static nserror llcache_send_event_to_users(llcache_object *object,
llcache_event *event)
{
nserror error = NSERROR_OK;
llcache_object_user *user, *next_user;
user = object->users;
while (user != NULL) {
bool was_target = user->iterator_target;
user->iterator_target = true;
error = user->handle->cb(user->handle, event,
user->handle->pw);
next_user = user->next;
user->iterator_target = was_target;
if (user->queued_for_delete && !was_target) {
llcache_object_remove_user(object, user);
llcache_object_user_destroy(user);
}
if (error != NSERROR_OK)
break;
user = next_user;
}
return error;
}
/**
* Create a new low-level cache object
*
* \param url URL of object to create
* \param result Pointer to location to receive result
* \return NSERROR_OK on success, appropriate error otherwise
*/
static nserror llcache_object_new(nsurl *url, llcache_object **result)
{
llcache_object *obj = calloc(1, sizeof(llcache_object));
if (obj == NULL)
return NSERROR_NOMEM;
NSLOG(llcache, DEBUG, "Created object %p (%s)", obj, nsurl_access(url));
obj->url = nsurl_ref(url);
*result = obj;
return NSERROR_OK;
}
/**
* Clone a POST data object
*
* \param orig Object to clone
* \param clone Pointer to location to receive clone
* \return NSERROR_OK on success, appropriate error otherwise
*/
static nserror llcache_post_data_clone(const llcache_post_data *orig,
llcache_post_data **clone)
{
llcache_post_data *post_clone;
post_clone = calloc(1, sizeof(llcache_post_data));
if (post_clone == NULL)
return NSERROR_NOMEM;
post_clone->type = orig->type;
/* Deep-copy the type-specific data */
if (orig->type == LLCACHE_POST_URL_ENCODED) {
post_clone->data.urlenc = strdup(orig->data.urlenc);
if (post_clone->data.urlenc == NULL) {
free(post_clone);
return NSERROR_NOMEM;
}
} else {
post_clone->data.multipart = fetch_multipart_data_clone(
orig->data.multipart);
if (post_clone->data.multipart == NULL) {
free(post_clone);
return NSERROR_NOMEM;
}
}
*clone = post_clone;
return NSERROR_OK;
}
/**
* Split a fetch header into name and value
*
* HTTP header splitting according to grammar defined in RFC7230 section 3.2
* https://tools.ietf.org/html/rfc7230#section-3.2
*
* This implementation is non conformant in that it:
* - includes carrige return and newline in whitespace (3.2.3)
* - allows whitespace before and after the field-name token (3.2.4)
* - does not handle obsolete line folding (3.2.4)
*
* \param data Header string
* \param len Byte length of header
* \param name Pointer to location to receive header name
* \param value Pointer to location to receive header value
* \return NSERROR_OK on success, appropriate error otherwise
*/
static nserror llcache_fetch_split_header(const uint8_t *data, size_t len,
char **name, char **value)
{
char *n, *v;
const uint8_t *colon;
/* Strip leading whitespace from name */
while (data[0] == ' ' || data[0] == '\t' ||
data[0] == '\r' || data[0] == '\n') {
data++;
}
/* Find colon */
colon = (const uint8_t *) strchr((const char *) data, ':');
if (colon == NULL) {
/* Failed, assume a key with no value */
colon = data + strlen((const char *)data);
/* Strip trailing whitespace from name */
while ((colon > data) &&
(colon[-1] == ' ' || colon[-1] == '\t' ||
colon[-1] == '\r' || colon[-1] == '\n')) {
colon--;
}
n = strndup((const char *) data, colon - data);
if (n == NULL)
return NSERROR_NOMEM;
v = strdup("");
if (v == NULL) {
free(n);
return NSERROR_NOMEM;
}
} else {
/* Split header into name & value */
/* Strip trailing whitespace from name */
while (colon > data && (colon[-1] == ' ' ||
colon[-1] == '\t' || colon[-1] == '\r' ||
colon[-1] == '\n'))
colon--;
n = strndup((const char *) data, colon - data);
if (n == NULL)
return NSERROR_NOMEM;
/* Find colon again */
while (*colon != ':') {
colon++;
}
/* Skip over colon and any subsequent whitespace */
do {
colon++;
} while (*colon == ' ' || *colon == '\t' ||
*colon == '\r' || *colon == '\n');
/* Strip trailing whitespace from value */
while (len > 0 && (data[len - 1] == ' ' ||
data[len - 1] == '\t' ||
data[len - 1] == '\r' ||
data[len - 1] == '\n')) {
len--;
}
v = strndup((const char *) colon, len - (colon - data));
if (v == NULL) {
free(n);
return NSERROR_NOMEM;
}
}
*name = n;
*value = v;
return NSERROR_OK;
}
/**
* parse cache control header value
*
* \param object Object to parse header for
* \param value header value
* \return NSERROR_OK on success, appropriate error otherwise
*/
static nserror
llcache_fetch_parse_cache_control(llcache_object *object, char *value)
{
http_cache_control *cc;
nserror error;
error = http_parse_cache_control(value, &cc);
if (error != NSERROR_OK) {
/* Ignore parse errors */
return NSERROR_OK;
}
if (http_cache_control_no_cache(cc) ||
http_cache_control_no_store(cc)) {
/**
* \todo When we get a disk cache we should
* distinguish between these two.
*/
object->cache.no_cache = LLCACHE_VALIDATE_ALWAYS;
}
if (http_cache_control_has_max_age(cc)) {
object->cache.max_age = http_cache_control_max_age(cc);
}
http_cache_control_destroy(cc);
return NSERROR_OK;
}
/**
* Update cache control from appropriate header
*
* \param object Object to parse header for
* \param name header name
* \param value header value
* \return NSERROR_OK on success, appropriate error otherwise
*/
static nserror
llcache_fetch_header_cache_control(llcache_object *object,
char *name,
char *value)
{
nserror res;
size_t name_len;
/* Parse cache headers to populate cache control data */
name_len = strlen(name);
switch (name_len) {
case 3:
if (strcasecmp(name, "Age") == 0) {
/* extract Age header */
if ('0' <= *value && *value <= '9') {
object->cache.age = atoi(value);
}
}
break;
case 4:
if (strcasecmp(name, "Date") == 0) {
/* extract Date header */
res = nsc_strntimet(value,
strlen(value),
&object->cache.date);
if (res != NSERROR_OK) {
NSLOG(llcache, INFO,
"Processing Date header value \"%s\" returned %d",
value, res);
}
} else if (strcasecmp(name, "ETag") == 0) {
/* extract ETag header */
free(object->cache.etag);
object->cache.etag = strdup(value);
if (object->cache.etag == NULL) {
NSLOG(llcache, INFO,
"No memory to duplicate ETag");
return NSERROR_NOMEM;
}
}
break;
case 7:
if (strcasecmp(name, "Expires") == 0) {
/* process Expires header value */
res = nsc_strntimet(value,
strlen(value),
&object->cache.expires);
if (res != NSERROR_OK) {
NSLOG(llcache, INFO,
"Processing Expires header value \"%s\" returned %d",
value, res);
object->cache.expires = (time_t)0x7fffffff;
}
}
break;
case 13:
if (strcasecmp(name, "Cache-Control") == 0) {
/* parse Cache-Control header value */
llcache_fetch_parse_cache_control(object,value);
} else if (strcasecmp(name, "Last-Modified") == 0) {
/* parse Last-Modified header value */
nsc_strntimet(value,
strlen(value),
&object->cache.last_modified);
}
break;
}
return NSERROR_OK;
}
/**
* Destroy headers.
*
* \param object The object to destroy headers within.
*/
static inline void llcache_destroy_headers(llcache_object *object)
{
while (object->num_headers > 0) {
object->num_headers--;
free(object->headers[object->num_headers].name);
free(object->headers[object->num_headers].value);
}
free(object->headers);
object->headers = NULL;
}
/**
* Invalidate cache control data.
*
* \param object The object to invalidate cache control for.
*/
static inline void llcache_invalidate_cache_control_data(llcache_object *object)
{
free(object->cache.etag);
memset(&(object->cache), 0, sizeof(llcache_cache_control));
object->cache.age = INVALID_AGE;
object->cache.max_age = INVALID_AGE;
}
/**
* Process a fetch header
*
* \param object Object being fetched
* \param data Header string
* \param len Byte length of header
* \return NSERROR_OK on success, appropriate error otherwise
*/
static nserror llcache_fetch_process_header(llcache_object *object,
const uint8_t *data, size_t len)
{
nserror res;
char *name, *value;
llcache_header *temp;
/**
* \note The headers for multiple HTTP responses may be
* delivered to us if the fetch layer receives a 401 response
* for which it has authentication credentials. This will
* result in a silent re-request after which we'll receive the
* actual response headers for the object we want to fetch
* (assuming that the credentials were correct of course)
*
* Therefore, if the header is an HTTP response start marker, then we
* must discard any headers we've read so far, reset the cache data
* that we might have computed, and start again.
*/
/** \todo Properly parse the response line */
if (strncmp((const char *) data, "HTTP/", SLEN("HTTP/")) == 0) {
time_t req_time = object->cache.req_time;
llcache_invalidate_cache_control_data(object);
/* Restore request time, so we compute object's age correctly */
object->cache.req_time = req_time;
llcache_destroy_headers(object);
}
/* Set fetch response time if not already set */
if (object->cache.res_time == 0) {
object->cache.res_time = time(NULL);
}
/* Parse header into name-value pair */
res = llcache_fetch_split_header(data, len, &name, &value);
if (res != NSERROR_OK) {
return res;
}
/* deal with empty header */
if (name[0] == 0) {
free(name);
free(value);
return NSERROR_OK;
}
/* update cache control data from header */
res = llcache_fetch_header_cache_control(object, name, value);
if (res != NSERROR_OK) {
free(name);
free(value);
return res;
}
/* Append header data to the object's headers array */
temp = realloc(object->headers,
(object->num_headers + 1) * sizeof(llcache_header));
if (temp == NULL) {
free(name);
free(value);
return NSERROR_NOMEM;
}
object->headers = temp;
object->headers[object->num_headers].name = name;
object->headers[object->num_headers].value = value;
object->num_headers++;
return NSERROR_OK;
}
/**
* (Re)fetch an object
*
* Sets up headers and attempts to start an actual fetch from the
* fetchers system updating the llcache object with the new fetch on
* successful start.
*
* \pre The fetch parameters in object->fetch must be populated
*
* \param object Object to refetch
* \return NSERROR_OK on success, appropriate error otherwise
*/
static nserror llcache_object_refetch(llcache_object *object)
{
const char *urlenc = NULL;
struct fetch_multipart_data *multipart = NULL;
char **headers = NULL;
int header_idx = 0;
nserror res;
if (object->fetch.post != NULL) {
if (object->fetch.post->type == LLCACHE_POST_URL_ENCODED) {
urlenc = object->fetch.post->data.urlenc;
} else {
multipart = object->fetch.post->data.multipart;
}
}
/* Generate cache-control headers */
headers = malloc(3 * sizeof(char *));
if (headers == NULL) {
return NSERROR_NOMEM;
}
if (object->cache.etag != NULL) {
const size_t len = SLEN("If-None-Match: ") +
strlen(object->cache.etag) + 1;
headers[header_idx] = malloc(len);
if (headers[header_idx] == NULL) {
free(headers);
return NSERROR_NOMEM;
}
snprintf(headers[header_idx], len, "If-None-Match: %s",
object->cache.etag);
header_idx++;
}
if (object->cache.last_modified != 0) {
/* Maximum length of an RFC 1123 date is 29 bytes */
const size_t len = SLEN("If-Modified-Since: ") + 29 + 1;
headers[header_idx] = malloc(len);
if (headers[header_idx] == NULL) {
while (--header_idx >= 0)
free(headers[header_idx]);
free(headers);
return NSERROR_NOMEM;
}
snprintf(headers[header_idx], len, "If-Modified-Since: %s",
rfc1123_date(object->cache.last_modified));
header_idx++;
}
headers[header_idx] = NULL;
/* Reset cache control data */
llcache_invalidate_cache_control_data(object);
object->cache.req_time = time(NULL);
object->cache.fin_time = object->cache.req_time;
/* Reset fetch state */
object->fetch.state = LLCACHE_FETCH_INIT;
NSLOG(llcache, DEBUG, "Re-fetching %p", object);
/* Kick off fetch */
res = fetch_start(object->url,
object->fetch.referer,
llcache_fetch_callback,
object,
object->fetch.flags & LLCACHE_RETRIEVE_NO_ERROR_PAGES,
urlenc,
multipart,
object->fetch.flags & LLCACHE_RETRIEVE_VERIFIABLE,
object->fetch.tried_with_tls_downgrade,
(const char **)headers,
&object->fetch.fetch);
/* Clean up cache-control headers */
while (--header_idx >= 0) {
free(headers[header_idx]);
}
free(headers);
return res;
}
/**
* Kick-off a fetch for an object
*
* \pre object::url must contain the URL to fetch
* \pre If there is a freshness validation candidate,
* object::candidate and object::cache must be filled in
* \pre There must not be a fetch in progress for \a object
*
* \param object Object to fetch
* \param flags Fetch flags
* \param referer Referring URL, or NULL for none
* \param post POST data, or NULL for GET
* \param redirect_count Number of redirects followed so far
* \param hsts_in_use Whether HSTS applies to this fetch
* \return NSERROR_OK on success, appropriate error otherwise
*/
static nserror llcache_object_fetch(llcache_object *object, uint32_t flags,
nsurl *referer, const llcache_post_data *post,
uint32_t redirect_count, bool hsts_in_use)
{
nserror error;
nsurl *referer_clone = NULL;
llcache_post_data *post_clone = NULL;
NSLOG(llcache, DEBUG, "Starting fetch for %p", object);
if (post != NULL) {
error = llcache_post_data_clone(post, &post_clone);
if (error != NSERROR_OK)
return error;
}
if (referer != NULL)
referer_clone = nsurl_ref(referer);
object->fetch.flags = flags;
object->fetch.referer = referer_clone;
object->fetch.post = post_clone;
object->fetch.redirect_count = redirect_count;
object->fetch.retries_remaining = llcache->fetch_attempts;
object->fetch.hsts_in_use = hsts_in_use;
return llcache_object_refetch(object);
}
/**
* Destroy a low-level cache object
*
* \pre Object is detached from cache list
* \pre Object has no users
* \pre Object is not a candidate (i.e. object::candidate_count == 0)
*
* \param object Object to destroy
* \return NSERROR_OK on success, appropriate error otherwise
*/
static nserror llcache_object_destroy(llcache_object *object)
{
size_t i;
NSLOG(llcache, DEBUG, "Destroying object %p, %s", object,
nsurl_access(object->url));
cert_chain_free(object->chain);
if (object->source_data != NULL) {
if (object->store_state == LLCACHE_STATE_DISC) {
guit->llcache->release(object->url, BACKING_STORE_NONE);
} else {
free(object->source_data);
}
}
nsurl_unref(object->url);
if (object->fetch.fetch != NULL) {
fetch_abort(object->fetch.fetch);
object->fetch.fetch = NULL;
}
if (object->fetch.referer != NULL)
nsurl_unref(object->fetch.referer);
if (object->fetch.post != NULL) {
if (object->fetch.post->type == LLCACHE_POST_URL_ENCODED) {
free(object->fetch.post->data.urlenc);
} else {
fetch_multipart_data_destroy(
object->fetch.post->data.multipart);
}
free(object->fetch.post);
}
free(object->cache.etag);
for (i = 0; i < object->num_headers; i++) {
free(object->headers[i].name);
free(object->headers[i].value);
}
free(object->headers);
free(object);
return NSERROR_OK;
}
/**
* Add a low-level cache object to a cache list
*
* \param object Object to add
* \param list List to add to
* \return NSERROR_OK
*/
static nserror llcache_object_add_to_list(llcache_object *object,
llcache_object **list)
{
object->prev = NULL;
object->next = *list;
if (*list != NULL)
(*list)->prev = object;
*list = object;
return NSERROR_OK;
}
/**
* Determine the remaining lifetime of a cache object using the
*
* \param cd cache control data.
* \return The length of time remaining for the object or 0 if expired.
*/
static int
llcache_object_rfc2616_remaining_lifetime(const llcache_cache_control *cd)
{
int current_age, freshness_lifetime;
time_t now = time(NULL);
/* Calculate staleness of cached object as per RFC 2616 13.2.3/13.2.4 */
current_age = max(0, (cd->res_time - cd->date));
current_age = max(current_age, (cd->age == INVALID_AGE) ? 0 : cd->age);
current_age += cd->res_time - cd->req_time + now - cd->res_time;
/* Determine freshness lifetime of this object */
if (cd->max_age != INVALID_AGE) {
freshness_lifetime = cd->max_age;
} else if (cd->expires != 0) {
freshness_lifetime = cd->expires - cd->date;
} else if (cd->last_modified != 0) {
freshness_lifetime = (now - cd->last_modified) / 10;
} else {
freshness_lifetime = 0;
}
NSLOG(llcache, DEBUG, "%d:%d", freshness_lifetime, current_age);
if ((cd->no_cache == LLCACHE_VALIDATE_FRESH) &&
(freshness_lifetime > current_age)) {
/* object was not forbidden from being returned from
* the cache unvalidated (i.e. the response contained
* a no-cache directive)
*
* The object current age is within the freshness lifetime.
*/
return freshness_lifetime - current_age;
}
return 0; /* object has no remaining lifetime */
}
/**
* Determine if an object is still fresh
*
* \param object Object to consider
* \return True if object is still fresh, false otherwise
*/
static bool llcache_object_is_fresh(const llcache_object *object)
{
int remaining_lifetime;
const llcache_cache_control *cd = &object->cache;
remaining_lifetime = llcache_object_rfc2616_remaining_lifetime(cd);
NSLOG(llcache, DEBUG, "%p: (%d > 0 || %d != %d)", object,
remaining_lifetime,
object->fetch.state, LLCACHE_FETCH_COMPLETE);
/* The object is fresh if:
* - it was not forbidden from being returned from the cache
* unvalidated.
*
* - it has remaining lifetime or still being fetched.
*/
return ((cd->no_cache == LLCACHE_VALIDATE_FRESH) &&
((remaining_lifetime > 0) ||
(object->fetch.state != LLCACHE_FETCH_COMPLETE)));
}
/**
* Clone an object's cache data
*
* \post If \a deep is false, then any pointers in \a source will be set to NULL
*
* \param source Source object containing cache data to clone
* \param destination Destination object to clone cache data into
* \param deep Whether to deep-copy the data or not
* \return NSERROR_OK on success, appropriate error otherwise
*/
static nserror llcache_object_clone_cache_data(llcache_object *source,
llcache_object *destination, bool deep)
{
/* ETag must be first, as it can fail when deep cloning */
if (source->cache.etag != NULL) {
char *etag = source->cache.etag;
if (deep) {
/* Copy the etag */
etag = strdup(source->cache.etag);
if (etag == NULL)
return NSERROR_NOMEM;
} else {
/* Destination takes ownership */
source->cache.etag = NULL;
}
if (destination->cache.etag != NULL)
free(destination->cache.etag);
destination->cache.etag = etag;
}
destination->cache.req_time = source->cache.req_time;
destination->cache.res_time = source->cache.res_time;
destination->cache.fin_time = source->cache.fin_time;
if (source->cache.date != 0)
destination->cache.date = source->cache.date;
if (source->cache.expires != 0)
destination->cache.expires = source->cache.expires;
if (source->cache.age != INVALID_AGE)
destination->cache.age = source->cache.age;
if (source->cache.max_age != INVALID_AGE)
destination->cache.max_age = source->cache.max_age;
if (source->cache.no_cache != LLCACHE_VALIDATE_FRESH)
destination->cache.no_cache = source->cache.no_cache;
if (source->cache.last_modified != 0)
destination->cache.last_modified = source->cache.last_modified;
return NSERROR_OK;
}
/**
* Remove a low-level cache object from a cache list
*
* \param object Object to remove
* \param list List to remove from
* \return NSERROR_OK
*/
static nserror
llcache_object_remove_from_list(llcache_object *object, llcache_object **list)
{
if (object == *list)
*list = object->next;
else
object->prev->next = object->next;
if (object->next != NULL)
object->next->prev = object->prev;
return NSERROR_OK;
}
/**
* Retrieve source data for an object from persistent store if necessary.
*
* If an object's source data has been placed in the persistent store
* and there is no in-memory copy, then attempt to retrieve the source
* data.
*
* \param object the object to operate on.
* \return appropriate error code.
*/
static nserror llcache_retrieve_persisted_data(llcache_object *object)
{
/* ensure the source data is present if necessary */
if ((object->source_data != NULL) ||
(object->store_state != LLCACHE_STATE_DISC)) {
/* source data does not require retrieving from
* persistent store.
*/
return NSERROR_OK;
}
/* Source data for the object may be in the persistent store */
return guit->llcache->fetch(object->url,
BACKING_STORE_NONE,
&object->source_data,
&object->source_len);
}
/**
* Generate a serialised version of an object's metadata
*
* The metadata includes object headers.
*
* \param object The cache object to serialise the metadata of.
* \param data_out Where the serialised buffer will be placed.
* \param datasize_out The size of the serialised data.
* \return NSERROR_OK on success with \a data_out and \a datasize_out
* updated, NSERROR_NOMEM on memory exhaustion or
* NSERROR_INVALID if there was an error serialising the
* stream.
*/
static nserror
llcache_serialise_metadata(llcache_object *object,
uint8_t **data_out,
size_t *datasize_out)
{
size_t allocsize;
int datasize;
uint8_t *data;
char *op;
unsigned int hloop;
int use;
size_t cert_chain_depth;
if (object->chain != NULL) {
cert_chain_depth = object->chain->depth;
} else {
cert_chain_depth = 0;
}
allocsize = 10 + 1; /* object length */
allocsize += 10 + 1; /* request time */
allocsize += 10 + 1; /* response time */
allocsize += 10 + 1; /* completion time */
allocsize += 10 + 1; /* space for number of header entries */
for (hloop = 0 ; hloop < object->num_headers ; hloop++) {
allocsize += strlen(object->headers[hloop].name) + 1;
allocsize += strlen(object->headers[hloop].value) + 1;
}
allocsize += nsurl_length(object->url) + 1;
/* space for number of DER formatted certificates */
allocsize += 10 + 1;
for (hloop = 0; hloop < cert_chain_depth; hloop++) {
allocsize += 10 + 1; /* error status */
allocsize += 4 * ((object->chain->certs[hloop].der_length + 2) / 3);
}
data = malloc(allocsize);
if (data == NULL) {
return NSERROR_NOMEM;
}
op = (char *)data;
datasize = allocsize;
/* the url, used for checking for collisions */
use = snprintf(op, datasize, "%s", nsurl_access(object->url));
if (use < 0) {
goto operror;
}
use++; /* does not count the null */
if (use > datasize) {
goto overflow;
}
op += use;
datasize -= use;
/* object size */
use = snprintf(op, datasize, "%" PRIsizet, object->source_len);
if (use < 0) {
goto operror;
}
use++; /* does not count the null */
if (use > datasize)
goto overflow;
op += use;
datasize -= use;
/* Time of request */
use = nsc_sntimet(op, datasize, &object->cache.req_time);
if (use == 0)
goto overflow;
use++; /* does not count the null */
op += use;
datasize -= use;
/* Time of response */
use = nsc_sntimet(op, datasize, &object->cache.res_time);
if (use == 0)
goto overflow;
use++; /* does not count the null */
op += use;
datasize -= use;
/* Time of completion */
use = nsc_sntimet(op, datasize, &object->cache.fin_time);
if (use == 0)
goto overflow;
use++; /* does not count the null */
op += use;
datasize -= use;
/* number of headers */
use = snprintf(op, datasize, "%" PRIsizet, object->num_headers);
if (use < 0) {
goto operror;
}
use++; /* does not count the null */
if (use > datasize)
goto overflow;
op += use;
datasize -= use;
/* headers */
for (hloop = 0 ; hloop < object->num_headers ; hloop++) {
use = snprintf(op, datasize,
"%s:%s",
object->headers[hloop].name,
object->headers[hloop].value);
if (use < 0) {
goto operror;
}
use++; /* does not count the null */
if (use > datasize)
goto overflow;
op += use;
datasize -= use;
}
/* number of DER formatted ssl certificates */
use = snprintf(op, datasize, "%" PRIsizet, cert_chain_depth);
if (use < 0) {
goto operror;
}
use++; /* does not count the null */
if (use > datasize)
goto overflow;
op += use;
datasize -= use;
/* SSL certificates */
for (hloop = 0; hloop < cert_chain_depth; hloop++) {
nsuerror res;
/* Certificate error code */
use = snprintf(op, datasize, "%d",
(int)(object->chain->certs[hloop].err));
if (use < 0) {
goto operror;
}
use++; /* does not count the null */
if (use > datasize)
goto overflow;
op += use;
datasize -= use;
/* DER certificate data in base64 encoding */
if (object->chain->certs[hloop].der != NULL) {
size_t output_length = datasize;
res = nsu_base64_encode(
object->chain->certs[hloop].der,
object->chain->certs[hloop].der_length,
(uint8_t *)op,
&output_length);
if (res != NSUERROR_OK) {
goto operror;
}
use = output_length;
} else {
use = 0;
}
use++; /* allow for null */
if (use > datasize)
goto overflow;
op += use;
*(op - 1) = 0;
datasize -= use;
}
NSLOG(llcache, DEBUG, "Filled buffer with %d spare", datasize);
*data_out = data;
*datasize_out = allocsize - datasize;
return NSERROR_OK;
overflow:
/* somehow we overflowed the buffer - hth? */
NSLOG(llcache, INFO, "Overflowed metadata buffer");
free(data);
return NSERROR_INVALID;
operror:
/* output error */
NSLOG(llcache, INFO, "Output error");
free(data);
return NSERROR_INVALID;
}
/**
* Deserialisation of an object's metadata.
*
* Attempt to retrieve and deserialise the metadata for an object from
* the backing store.
*
* This must only update object if it is successful otherwise difficult
* to debug crashes happen later by using bad leftover object state.
*
* \param object The object to retrieve the metadata for.
* \return NSERROR_OK if the metatdata was retrieved and deserialised
* or error code if URL is not in persistent storage or in
* event of deserialisation error.
*/
static nserror
llcache_process_metadata(llcache_object *object)
{
nserror res;
uint8_t *metadata = NULL;
size_t metadatalen = 0;
size_t remaining = 0;
nsurl *metadataurl;
unsigned int line;
char *ln;
int lnsize;
size_t source_length;
time_t request_time;
time_t response_time;
time_t completion_time;
size_t num_headers;
size_t hloop;
size_t ssl_cert_count = 0;
struct cert_chain *chain = NULL;
NSLOG(llcache, INFO, "Retrieving metadata");
/* attempt to retrieve object metadata from the backing store */
res = guit->llcache->fetch(object->url,
BACKING_STORE_META,
&metadata,
&metadatalen);
if (res != NSERROR_OK) {
return res;
}
NSLOG(llcache, INFO, "Processing retrieved data");
/* metadata is stored as a sequence of NULL terminated strings
* which we call 'line's here.
*/
/* We track remaining data because as we extend this data structure
* we need to know if we should continue to parse
*/
remaining = metadatalen;
/* metadata line 1 is the url the metadata referrs to */
line = 1;
ln = (char *)metadata;
lnsize = strlen(ln);
remaining -= lnsize + 1;
if (lnsize < 7) {
res = NSERROR_INVALID;
goto format_error;
}
res = nsurl_create(ln, &metadataurl);
if (res != NSERROR_OK)
goto format_error;
if (nsurl_compare(object->url, metadataurl, NSURL_COMPLETE) != true) {
/* backing store returned the wrong object for the
* request. This may occur if the backing store had
* a collision in its storage method. We cope with this
* by simply skipping caching of this object.
*/
NSLOG(llcache, INFO, "Got metadata for %s instead of %s",
nsurl_access(metadataurl), nsurl_access(object->url));
nsurl_unref(metadataurl);
guit->llcache->release(object->url, BACKING_STORE_META);
return NSERROR_BAD_URL;
}
nsurl_unref(metadataurl);
/* metadata line 2 is the object's length */
line = 2;
ln += lnsize + 1;
lnsize = strlen(ln);
remaining -= lnsize + 1;
if ((lnsize < 1) || (sscanf(ln, "%" PRIsizet, &source_length) != 1)) {
res = NSERROR_INVALID;
goto format_error;
}
/* metadata line 3 is the time of request */
line = 3;
ln += lnsize + 1;
lnsize = strlen(ln);
remaining -= lnsize + 1;
res = nsc_snptimet(ln, lnsize, &request_time);
if (res != NSERROR_OK)
goto format_error;
/* metadata line 4 is the time of response */
line = 4;
ln += lnsize + 1;
lnsize = strlen(ln);
remaining -= lnsize + 1;
res = nsc_snptimet(ln, lnsize, &response_time);
if (res != NSERROR_OK)
goto format_error;
/* metadata line 5 is the time of request completion */
line = 5;
ln += lnsize + 1;
lnsize = strlen(ln);
remaining -= lnsize + 1;
res = nsc_snptimet(ln, lnsize, &completion_time);
if (res != NSERROR_OK)
goto format_error;
/* metadata line 6 is the number of headers */
line = 6;
ln += lnsize + 1;
lnsize = strlen(ln);
remaining -= lnsize + 1;
if ((lnsize < 1) || (sscanf(ln, "%" PRIsizet, &num_headers) != 1)) {
res = NSERROR_INVALID;
goto format_error;
}
/* read headers */
for (hloop = 0 ; hloop < num_headers; hloop++) {
line++;
ln += lnsize + 1;
lnsize = strlen(ln);
remaining -= lnsize + 1;
res = llcache_fetch_process_header(object,
(uint8_t *)ln,
lnsize);
if (res != NSERROR_OK)
goto format_error;
}
if (remaining == 0) {
goto skip_ssl_certificates;
}
/* Next line is the number of DER base64 encoded certificates */
line++;
ln += lnsize + 1;
lnsize = strlen(ln);
remaining -= lnsize + 1;
if ((lnsize < 1) || (sscanf(ln, "%" PRIsizet, &ssl_cert_count) != 1)) {
res = NSERROR_INVALID;
goto format_error;
}
if (ssl_cert_count == 0) {
goto skip_ssl_certificates;
}
if (ssl_cert_count > MAX_CERT_DEPTH) {
res = NSERROR_INVALID;
goto format_error;
}
res = cert_chain_alloc(ssl_cert_count, &chain);
if (res != NSERROR_OK) {
goto format_error;
}
for (hloop = 0; hloop < ssl_cert_count; hloop++) {
int errcode;
nsuerror nsures;
/* Certificate error code */
line++;
ln += lnsize + 1;
lnsize = strlen(ln);
remaining -= lnsize + 1;
if ((lnsize < 1) || (sscanf(ln, "%d", &errcode) != 1)) {
res = NSERROR_INVALID;
goto format_error;
}
if (errcode < SSL_CERT_ERR_OK ||
errcode > SSL_CERT_ERR_MAX_KNOWN) {
/* Error with the cert code, assume UNKNOWN */
chain->certs[hloop].err = SSL_CERT_ERR_UNKNOWN;
} else {
chain->certs[hloop].err = (ssl_cert_err)errcode;
}
/* base64 encoded DER certificate data */
line++;
ln += lnsize + 1;
lnsize = strlen(ln);
remaining -= lnsize + 1;
if (lnsize > 0) {
nsures = nsu_base64_decode_alloc((const uint8_t *)ln,
lnsize,
&chain->certs[hloop].der,
&chain->certs[hloop].der_length);
if (nsures != NSUERROR_OK) {
res = NSERROR_NOMEM;
goto format_error;
}
}
}
skip_ssl_certificates:
guit->llcache->release(object->url, BACKING_STORE_META);
/* update object on successful parse of metadata */
object->source_len = source_length;
/** \todo really not sure this is right, nothing is allocated here? */
object->source_alloc = metadatalen;
object->cache.req_time = request_time;
object->cache.res_time = response_time;
object->cache.fin_time = completion_time;
object->chain = chain;
/* object stored in backing store */
object->store_state = LLCACHE_STATE_DISC;
return NSERROR_OK;
format_error:
NSLOG(llcache, INFO,
"metadata error on line %d error code %d\n",
line, res);
guit->llcache->release(object->url, BACKING_STORE_META);
cert_chain_free(chain);
return res;
}
/**
* Check whether a scheme is persistable.
*
* \param url URL to check.
* \return true iff url has a persistable scheme.
*/
static inline bool llcache__scheme_is_persistable(const nsurl *url)
{
lwc_string *scheme = nsurl_get_component(url, NSURL_SCHEME);
bool persistable = false;
bool match;
/* nsurl ensures lower case schemes, and corestrings are lower
* case, so it's safe to use case-sensitive comparison. */
if ((lwc_string_isequal(scheme, corestring_lwc_http,
&match) == lwc_error_ok &&
(match == true)) ||
(lwc_string_isequal(scheme, corestring_lwc_https,
&match) == lwc_error_ok &&
(match == true))) {
persistable = true;
}
lwc_string_unref(scheme);
return persistable;
}
/**
* Check whether a scheme is cachable.
*
* \param url URL to check.
* \return true iff url has a cachable scheme.
*/
static inline bool llcache__scheme_is_cachable(const nsurl *url)
{
lwc_string *scheme = nsurl_get_component(url, NSURL_SCHEME);
bool cachable = false;
bool match;
/* nsurl ensures lower case schemes, and corestrings are lower
* case, so it's safe to use case-sensitive comparison. */
if ((lwc_string_isequal(scheme, corestring_lwc_http,
&match) == lwc_error_ok &&
(match == true)) ||
(lwc_string_isequal(scheme, corestring_lwc_https,
&match) == lwc_error_ok &&
(match == true)) ||
(lwc_string_isequal(scheme, corestring_lwc_data,
&match) == lwc_error_ok &&
(match == true)) ||
(lwc_string_isequal(scheme, corestring_lwc_resource,
&match) == lwc_error_ok &&
(match == true)) ||
(lwc_string_isequal(scheme, corestring_lwc_file,
&match) == lwc_error_ok &&
(match == true))) {
cachable = true;
}
lwc_string_unref(scheme);
return cachable;
}
/**
* Attempt to retrieve an object from persistent storage.
*
* \param object The object to populate from persistent store.
* \param flags Fetch flags.
* \param referer The referring url.
* \param post Post data for fetch.
* \param redirect_count how many times this fetch has been redirected.
* \return NSERROR_OK if the object was successfully retrieved from the
* cache else appropriate error code.
*/
static nserror
llcache_object_fetch_persistent(llcache_object *object,
uint32_t flags,
nsurl *referer,
const llcache_post_data *post,
uint32_t redirect_count)
{
nserror error;
nsurl *referer_clone = NULL;
llcache_post_data *post_clone = NULL;
if (!llcache__scheme_is_persistable(object->url)) {
/* Don't bother looking up non-http(s) stuff; we don't
* persist it. */
return NSERROR_NOT_FOUND;
}
object->cache.req_time = time(NULL);
object->cache.fin_time = object->cache.req_time;
/* retrieve and process metadata */
error = llcache_process_metadata(object);
if (error != NSERROR_OK) {
return error;
}
/* entry came out of cache - need to setup object state */
if (post != NULL) {
error = llcache_post_data_clone(post, &post_clone);
if (error != NSERROR_OK)
return error;
}
if (referer != NULL) {
referer_clone = nsurl_ref(referer);
}
object->fetch.flags = flags;
object->fetch.referer = referer_clone;
object->fetch.post = post_clone;
object->fetch.redirect_count = redirect_count;
/* fetch is "finished" */
object->fetch.state = LLCACHE_FETCH_COMPLETE;
object->fetch.fetch = NULL;
return NSERROR_OK;
}
/**
* Retrieve a potentially cached object
*
* \param url URL of object to retrieve
* \param flags Fetch flags
* \param referer Referring URL, or NULL if none
* \param post POST data, or NULL for a GET request
* \param redirect_count Number of redirects followed so far
* \param hsts_in_use Whether HSTS applies to this fetch
* \param result Pointer to location to receive retrieved object
* \return NSERROR_OK on success, appropriate error otherwise
*/
static nserror
llcache_object_retrieve_from_cache(nsurl *url,
uint32_t flags,
nsurl *referer,
const llcache_post_data *post,
uint32_t redirect_count,
bool hsts_in_use,
llcache_object **result)
{
nserror error;
llcache_object *obj, *newest = NULL;
NSLOG(llcache, DEBUG,
"Searching cache for %s flags:%x referer:%s post:%p",
nsurl_access(url), flags,
referer==NULL?"":nsurl_access(referer),
post);
/* Search for the most recently fetched matching object */
for (obj = llcache->cached_objects; obj != NULL; obj = obj->next) {
if ((newest == NULL ||
obj->cache.req_time > newest->cache.req_time) &&
nsurl_compare(obj->url, url,
NSURL_COMPLETE) == true) {
newest = obj;
}
}
/* No viable object found in cache create one and attempt to
* pull from persistent store.
*/
if (newest == NULL) {
NSLOG(llcache, DEBUG, "No viable object found in llcache");
error = llcache_object_new(url, &obj);
if (error != NSERROR_OK)
return error;
/* attempt to retrieve object from persistent store */
error = llcache_object_fetch_persistent(obj, flags, referer, post, redirect_count);
if (error == NSERROR_OK) {
NSLOG(llcache, DEBUG, "retrieved object from persistent store");
/* set newest object from persistent store which
* will cause the normal object handling to be used.
*/
newest = obj;
/* Add new object to cached object list */
llcache_object_add_to_list(obj, &llcache->cached_objects);
}
/* else no object found and irretrievable from cache,
* fall through with newest unset to start fetch
*/
}
if ((newest != NULL) && (llcache_object_is_fresh(newest))) {
/* Found a suitable object, and it's still fresh */
NSLOG(llcache, DEBUG, "Found fresh %p", newest);
/* The client needs to catch up with the object's state.
* This will occur the next time that llcache_poll is called.
*/
/* ensure the source data is present */
error = llcache_retrieve_persisted_data(newest);
if (error == NSERROR_OK) {
/* source data was successfully retrieved from
* persistent store
*/
*result = newest;
return NSERROR_OK;
}
/* retrieval of source data from persistent store
* failed, destroy cache object and fall though to
* cache miss to re-fetch
*/
NSLOG(llcache, DEBUG, "Persistent retrieval failed for %p", newest);
llcache_object_remove_from_list(newest, &llcache->cached_objects);
llcache_object_destroy(newest);
error = llcache_object_new(url, &obj);
if (error != NSERROR_OK) {
return error;
}
} else if (newest != NULL) {
/* Found a candidate object but it needs freshness validation */
/* ensure the source data is present */
error = llcache_retrieve_persisted_data(newest);
if (error == NSERROR_OK) {
/* Create a new object */
error = llcache_object_new(url, &obj);
if (error != NSERROR_OK)
return error;
NSLOG(llcache, DEBUG, "Found candidate %p (%p)", obj, newest);
/* Clone candidate's cache data */
error = llcache_object_clone_cache_data(newest, obj, true);
if (error != NSERROR_OK) {
llcache_object_destroy(obj);
return error;
}
/* Record candidate, so we can fall back if it is still fresh */
newest->candidate_count++;
obj->candidate = newest;
/* Attempt to kick-off fetch */
error = llcache_object_fetch(obj, flags, referer, post,
redirect_count, hsts_in_use);
if (error != NSERROR_OK) {
newest->candidate_count--;
llcache_object_destroy(obj);
return error;
}
/* Add new object to cache */
llcache_object_add_to_list(obj, &llcache->cached_objects);
*result = obj;
return NSERROR_OK;
}
NSLOG(llcache, DEBUG, "Persistent retrieval failed for %p", newest);
/* retrieval of source data from persistent store
* failed, destroy cache object and fall though to
* cache miss to re-retch
*/
llcache_object_remove_from_list(newest,
&llcache->cached_objects);
llcache_object_destroy(newest);
error = llcache_object_new(url, &obj);
if (error != NSERROR_OK) {
return error;
}
}
/* Attempt to kick-off fetch */
error = llcache_object_fetch(obj, flags, referer, post,
redirect_count, hsts_in_use);
if (error != NSERROR_OK) {
llcache_object_destroy(obj);
return error;
}
/* Add new object to cache */
llcache_object_add_to_list(obj, &llcache->cached_objects);
*result = obj;
return NSERROR_OK;
}
/**
* Retrieve an object from the cache, fetching it if necessary.
*
* \param url URL of object to retrieve
* \param flags Fetch flags
* \param referer Referring URL, or NULL if none
* \param post POST data, or NULL for a GET request
* \param redirect_count Number of redirects followed so far
* \param hsts_in_use Whether HSTS applies to this fetch
* \param result Pointer to location to receive retrieved object
* \return NSERROR_OK on success, appropriate error otherwise
*/
static nserror
llcache_object_retrieve(nsurl *url,
uint32_t flags,
nsurl *referer,
const llcache_post_data *post,
uint32_t redirect_count,
bool hsts_in_use,
llcache_object **result)
{
nserror error;
llcache_object *obj;
nsurl *defragmented_url;
bool uncachable = false;
NSLOG(llcache, DEBUG, "Retrieve %s (%x, %s, %p)", nsurl_access(url), flags,
referer==NULL?"":nsurl_access(referer), post);
/* Get rid of any url fragment */
error = nsurl_defragment(url, &defragmented_url);
if (error != NSERROR_OK)
return error;
/* determine if content is cachable */
if ((flags & LLCACHE_RETRIEVE_FORCE_FETCH) != 0) {
/* Forced fetches are never cached */
uncachable = true;
} else if (post != NULL) {
/* POST requests are never cached */
uncachable = true;
} else {
uncachable = !llcache__scheme_is_cachable(defragmented_url);
}
if (uncachable) {
/* Create new object */
error = llcache_object_new(defragmented_url, &obj);
if (error != NSERROR_OK) {
nsurl_unref(defragmented_url);
return error;
}
/* Attempt to kick-off fetch */
error = llcache_object_fetch(obj, flags, referer, post,
redirect_count, hsts_in_use);
if (error != NSERROR_OK) {
llcache_object_destroy(obj);
nsurl_unref(defragmented_url);
return error;
}
/* Add new object to uncached list */
llcache_object_add_to_list(obj, &llcache->uncached_objects);
} else {
error = llcache_object_retrieve_from_cache(defragmented_url,
flags, referer, post, redirect_count,
hsts_in_use, &obj);
if (error != NSERROR_OK) {
nsurl_unref(defragmented_url);
return error;
}
/* Returned object is already in the cached list */
}
NSLOG(llcache, DEBUG, "Retrieved %p", obj);
*result = obj;
nsurl_unref(defragmented_url);
return NSERROR_OK;
}
/**
* Add a user to a low-level cache object
*
* \param object Object to add user to
* \param user User to add
* \return NSERROR_OK.
*/
static nserror llcache_object_add_user(llcache_object *object,
llcache_object_user *user)
{
assert(user->next == NULL);
assert(user->prev == NULL);
assert(user->handle != NULL);
user->handle->object = object;
user->prev = NULL;
user->next = object->users;
if (object->users != NULL)
object->users->prev = user;
object->users = user;
NSLOG(llcache, DEBUG, "Adding user %p to %p", user, object);
return NSERROR_OK;
}
/**
* Transform a request-URI based on HSTS policy
*
* \param url URL to transform
* \param result Pointer to location to receive transformed URL
* \param hsts_in_use Pointer to location to receive HSTS in-use flag
* \return NSERROR_OK on success, appropriate error otherwise
*/
static nserror llcache_hsts_transform_url(nsurl *url, nsurl **result,
bool *hsts_in_use)
{
lwc_string *scheme = NULL;
bool match;
nserror error = NSERROR_OK;
scheme = nsurl_get_component(url, NSURL_SCHEME);
if (lwc_string_caseless_isequal(scheme, corestring_lwc_http,
&match) != lwc_error_ok || match == false) {
/* Non-HTTP fetch: ignore */
lwc_string_unref(scheme);
*result = nsurl_ref(url);
*hsts_in_use = false;
return error;
}
lwc_string_unref(scheme);
if (urldb_get_hsts_enabled(url)) {
/* Only need to force HTTPS. If original port was explicitly
* specified as 80, nsurl_create/join will remove it (as
* it's redundant) */
error = nsurl_replace_scheme(url, corestring_lwc_https,
result);
*hsts_in_use = (error == NSERROR_OK);
} else {
*result = nsurl_ref(url);
*hsts_in_use = false;
}
return error;
}
/**
* Update HSTS policy for target domain.
*
* \param object Newly-fetched cache object
* \return NSERROR_OK on success, appropriate error otherwise
*/
static nserror llcache_hsts_update_policy(llcache_object *object)
{
size_t i;
lwc_string *scheme = NULL;
bool match = false;
scheme = nsurl_get_component(object->url, NSURL_SCHEME);
if (lwc_string_caseless_isequal(scheme, corestring_lwc_https,
&match) != lwc_error_ok || match == false) {
/* Non-HTTPS fetch: ignore */
lwc_string_unref(scheme);
return NSERROR_OK;
}
lwc_string_unref(scheme);
if (object->fetch.tainted_tls) {
/* Transport is tainted: ignore */
return NSERROR_OK;
}
for (i = 0; i < object->num_headers; i++) {
if (strcasecmp("Strict-Transport-Security",
object->headers[i].name) == 0) {
urldb_set_hsts_policy(object->url,
object->headers[i].value);
/* Only process the first one we find */
break;
}
}
return NSERROR_OK;
}
/**
* Handle FETCH_REDIRECT event
*
* \param object Object being redirected
* \param target Target of redirect (may be relative)
* \param replacement Pointer to location to receive replacement object
* \return NSERROR_OK on success, appropriate error otherwise
*/
static nserror llcache_fetch_redirect(llcache_object *object,
const char *target, llcache_object **replacement)
{
nserror error;
llcache_object *dest;
llcache_object_user *user, *next;
const llcache_post_data *post = object->fetch.post;
nsurl *url, *hsts_url;
lwc_string *scheme;
lwc_string *object_scheme;
bool match, hsts_in_use;
/* Extract HTTP response code from the fetch object */
long http_code = fetch_http_code(object->fetch.fetch);
llcache_event event;
/* Abort fetch for this object */
fetch_abort(object->fetch.fetch);
object->fetch.fetch = NULL;
/* Invalidate the cache control data */
llcache_invalidate_cache_control_data(object);
/* And mark it complete */
object->fetch.state = LLCACHE_FETCH_COMPLETE;
(void) llcache_hsts_update_policy(object);
/* Forcibly stop redirecting if we've followed too many redirects */
#define REDIRECT_LIMIT 10
if (object->fetch.redirect_count > REDIRECT_LIMIT) {
NSLOG(llcache, INFO, "Too many nested redirects");
event.type = LLCACHE_EVENT_ERROR;
event.data.error.code = NSERROR_BAD_REDIRECT;
event.data.error.msg = messages_get("BadRedirect");
return llcache_send_event_to_users(object, &event);
}
#undef REDIRECT_LIMIT
/* Make target absolute */
error = nsurl_join(object->url, target, &url);
if (error != NSERROR_OK)
return error;
/* Perform HSTS transform */
error = llcache_hsts_transform_url(url, &hsts_url, &hsts_in_use);
if (error != NSERROR_OK) {
nsurl_unref(url);
return error;
}
nsurl_unref(url);
/* Inform users of redirect */
event.type = LLCACHE_EVENT_REDIRECT;
event.data.redirect.from = object->url;
event.data.redirect.to = hsts_url;
error = llcache_send_event_to_users(object, &event);
if (error != NSERROR_OK) {
nsurl_unref(hsts_url);
return error;
}
/* Reject attempts to redirect from unvalidated to validated schemes
* A "validated" scheme is one over which we have some guarantee that
* the source is trustworthy. */
object_scheme = nsurl_get_component(object->url, NSURL_SCHEME);
scheme = nsurl_get_component(hsts_url, NSURL_SCHEME);
/* resource: and about: are allowed to redirect anywhere */
if ((lwc_string_isequal(object_scheme, corestring_lwc_resource,
&match) == lwc_error_ok && match == false) &&
(lwc_string_isequal(object_scheme, corestring_lwc_about,
&match) == lwc_error_ok && match == false)) {
/* file, about and resource are not valid redirect targets */
if ((lwc_string_isequal(object_scheme, corestring_lwc_file,
&match) == lwc_error_ok && match == true) ||
(lwc_string_isequal(object_scheme, corestring_lwc_about,
&match) == lwc_error_ok && match == true) ||
(lwc_string_isequal(object_scheme, corestring_lwc_resource,
&match) == lwc_error_ok && match == true)) {
lwc_string_unref(object_scheme);
lwc_string_unref(scheme);
nsurl_unref(hsts_url);
return NSERROR_OK;
}
}
lwc_string_unref(scheme);
lwc_string_unref(object_scheme);
/* Bail out if we've no way of handling this URL */
if (fetch_can_fetch(hsts_url) == false) {
nsurl_unref(hsts_url);
return NSERROR_OK;
}
if (http_code == 301 || http_code == 302 || http_code == 303) {
/* 301, 302, 303 redirects are all unconditional GET requests */
post = NULL;
} else if (http_code != 307 || post != NULL) {
/** \todo 300, 305, 307 with POST */
nsurl_unref(hsts_url);
return NSERROR_OK;
}
/* Attempt to fetch target URL */
error = llcache_object_retrieve(hsts_url, object->fetch.flags,
object->fetch.referer, post,
object->fetch.redirect_count + 1,
hsts_in_use, &dest);
/* No longer require url */
nsurl_unref(hsts_url);
if (error != NSERROR_OK)
return error;
/* Move user(s) to replacement object */
for (user = object->users; user != NULL; user = next) {
next = user->next;
llcache_object_remove_user(object, user);
llcache_object_add_user(dest, user);
}
/* Dest is now our object */
*replacement = dest;
return NSERROR_OK;
}
/**
* Update an object's cache state
*
* \param object Object to update cache for
* \return NSERROR_OK.
*/
static nserror llcache_object_cache_update(llcache_object *object)
{
if (object->cache.date == 0)
object->cache.date = time(NULL);
return NSERROR_OK;
}
/**
* Handle FETCH_NOTMODIFIED event
*
* \param object Object to process
* \param replacement Pointer to location to receive replacement object
* \return NSERROR_OK.
*/
static nserror llcache_fetch_notmodified(llcache_object *object,
llcache_object **replacement)
{
/* There may be no candidate if the server erroneously responded
* to an unconditional request with a 304 Not Modified response.
* In this case, we simply retain the initial object, having
* invalidated it and marked it as complete.
*/
if (object->candidate != NULL) {
llcache_object_user *user, *next;
/* Move user(s) to candidate content */
for (user = object->users; user != NULL; user = next) {
next = user->next;
llcache_object_remove_user(object, user);
llcache_object_add_user(object->candidate, user);
}
/* Candidate is no longer a candidate for us */
object->candidate->candidate_count--;
/* Clone our cache control data into the candidate */
llcache_object_clone_cache_data(object, object->candidate,
false);
/* Bring candidate's cache data up to date */
llcache_object_cache_update(object->candidate);
/* Revert no-cache to normal, if required */
if (object->candidate->cache.no_cache ==
LLCACHE_VALIDATE_ONCE) {
object->candidate->cache.no_cache =
LLCACHE_VALIDATE_FRESH;
}
/* Candidate is now our object */
*replacement = object->candidate;
object->candidate = NULL;
} else {
/* There was no candidate: retain object */
*replacement = object;
}
/* Ensure fetch has stopped */
fetch_abort(object->fetch.fetch);
object->fetch.fetch = NULL;
/* Invalidate our cache-control data */
llcache_invalidate_cache_control_data(object);
/* Mark it complete */
object->fetch.state = LLCACHE_FETCH_COMPLETE;
(void) llcache_hsts_update_policy(object);
/* Old object will be flushed from the cache on the next poll */
return NSERROR_OK;
}
/**
* Process a chunk of fetched data
*
* \param object Object being fetched
* \param data Data to process
* \param len Byte length of data
* \return NSERROR_OK on success, appropriate error otherwise.
*/
static nserror
llcache_fetch_process_data(llcache_object *object,
const uint8_t *data,
size_t len)
{
if (object->fetch.state != LLCACHE_FETCH_DATA) {
/**
* \note
* On entry into this state, check if we need to
* invalidate the cache control data. We are guaranteed
* to have received all response headers.
*
* There are two cases in which we want to suppress
* cacheing of an object:
*
* 1) The HTTP response code is not 200 or 203
* 2) The request URI had a query string and the
* response headers did not provide an explicit
* object expiration time.
*/
long http_code = fetch_http_code(object->fetch.fetch);
if ((http_code != 200 && http_code != 203) ||
(nsurl_has_component(object->url, NSURL_QUERY) &&
(object->cache.max_age == INVALID_AGE &&
object->cache.expires == 0))) {
/* Invalidate cache control data */
llcache_invalidate_cache_control_data(object);
}
/* Release candidate, if any */
if (object->candidate != NULL) {
object->candidate->candidate_count--;
object->candidate = NULL;
}
object->fetch.state = LLCACHE_FETCH_DATA;
}
/* Resize source buffer if it's too small */
if (object->source_len + len >= object->source_alloc) {
const size_t new_len = object->source_len + len + 64 * 1024;
uint8_t *temp = realloc(object->source_data, new_len);
if (temp == NULL)
return NSERROR_NOMEM;
object->source_data = temp;
object->source_alloc = new_len;
}
/* Append this data chunk to source buffer */
memcpy(object->source_data + object->source_len, data, len);
object->source_len += len;
return NSERROR_OK;
}
/**
* Handle an authentication request
*
* \param object Object being fetched
* \param realm Authentication realm
* \return NSERROR_OK on success, appropriate error otherwise.
*/
static nserror llcache_fetch_auth(llcache_object *object, const char *realm)
{
const char *auth;
nserror error = NSERROR_OK;
/* Abort fetch for this object */
fetch_abort(object->fetch.fetch);
object->fetch.fetch = NULL;
/* Invalidate cache-control data */
llcache_invalidate_cache_control_data(object);
/* Destroy headers */
llcache_destroy_headers(object);
/* If there was no realm, then default to the URL */
/** \todo If there was no WWW-Authenticate header, use response body */
if (realm == NULL)
realm = nsurl_access(object->url);
auth = urldb_get_auth_details(object->url, realm);
if (auth == NULL || object->fetch.tried_with_auth == true) {
llcache_event event;
/* No authentication details, or tried what we had, so ask */
object->fetch.tried_with_auth = false;
/* Mark object complete */
object->fetch.state = LLCACHE_FETCH_COMPLETE;
/* Inform client(s) that object fetch failed */
event.type = LLCACHE_EVENT_ERROR;
/** \todo More appropriate error message */
event.data.error.code = NSERROR_BAD_AUTH;
event.data.error.msg = realm;
error = llcache_send_event_to_users(object, &event);
} else {
/* Flag that we've tried to refetch with credentials, so
* that if the fetch fails again, we ask the user again */
object->fetch.tried_with_auth = true;
error = llcache_object_refetch(object);
}
return error;
}
/**
* Handle a TLS certificate verification failure
*
* \param object Object being fetched
* \return NSERROR_OK on success, appropriate error otherwise
*/
static nserror llcache_fetch_cert_error(llcache_object *object)
{
nserror error = NSERROR_OK;
/* Fetch has been stopped, and destroyed. Invalidate object's pointer */
object->fetch.fetch = NULL;
/* Invalidate cache-control data */
llcache_invalidate_cache_control_data(object);
/* Consider the TLS transport tainted */
object->fetch.tainted_tls = true;
/* Only give the user a chance if HSTS isn't in use for this fetch */
if (object->fetch.hsts_in_use == false) {
llcache_event event;
/* Mark object complete */
object->fetch.state = LLCACHE_FETCH_COMPLETE;
/* Inform client(s) that object fetch failed */
event.type = LLCACHE_EVENT_ERROR;
/** \todo More appropriate error message */
event.data.error.code = NSERROR_BAD_CERTS;
event.data.error.msg = messages_get("FetchFailed");
error = llcache_send_event_to_users(object, &event);
} else {
llcache_event event;
/* Mark object complete */
object->fetch.state = LLCACHE_FETCH_COMPLETE;
/* Inform client(s) that object fetch failed */
event.type = LLCACHE_EVENT_ERROR;
/** \todo More appropriate error message */
event.data.error.code = NSERROR_UNKNOWN;
event.data.error.msg = messages_get("FetchFailed");
error = llcache_send_event_to_users(object, &event);
}
return error;
}
/**
* Handle a TLS connection setup failure
*
* \param object Object being fetched
* \return NSERROR_OK on success, appropriate error otherwise
*/
static nserror llcache_fetch_ssl_error(llcache_object *object)
{
nserror error = NSERROR_OK;
/* Fetch has been stopped, and destroyed. Invalidate object's pointer */
object->fetch.fetch = NULL;
/* Invalidate cache-control data */
llcache_invalidate_cache_control_data(object);
/* Consider the TLS transport tainted */
object->fetch.tainted_tls = true;
/* Make no attempt to downgrade if HSTS is in use
* (i.e. assume server does TLS properly) */
if (object->fetch.hsts_in_use ||
object->fetch.tried_with_tls_downgrade) {
/* Have already tried to downgrade, so give up */
llcache_event event;
/* Mark object complete */
object->fetch.state = LLCACHE_FETCH_COMPLETE;
/* Inform client(s) that object fetch failed */
event.type = LLCACHE_EVENT_ERROR;
/** \todo More appropriate error message */
event.data.error.code = NSERROR_UNKNOWN;
event.data.error.msg = messages_get("FetchFailed");
error = llcache_send_event_to_users(object, &event);
} else {
/* Flag that we've tried to downgrade, so that if the
* fetch fails again, we give up */
object->fetch.tried_with_tls_downgrade = true;
error = llcache_object_refetch(object);
}
return error;
}
/**
* handle time out while trying to fetch.
*
* \param object Object being fetched
* \return NSERROR_OK on success otherwise error code
*/
static nserror llcache_fetch_timeout(llcache_object *object)
{
llcache_event event;
/* The fetch has already been cleaned up by the fetcher but
* we would like to retry if we can.
*/
if (object->fetch.retries_remaining > 1) {
object->fetch.retries_remaining--;
return llcache_object_refetch(object);
}
/* The fetch has has already been cleaned up by the fetcher */
object->fetch.state = LLCACHE_FETCH_COMPLETE;
object->fetch.fetch = NULL;
/* Release candidate, if any */
if (object->candidate != NULL) {
object->candidate->candidate_count--;
object->candidate = NULL;
}
/* Invalidate cache control data */
llcache_invalidate_cache_control_data(object);
event.type = LLCACHE_EVENT_ERROR;
event.data.error.code = NSERROR_TIMEOUT;
event.data.error.msg = NULL;
return llcache_send_event_to_users(object, &event);
}
/**
* Construct a sorted list of objects available for writeout operation.
*
* The list contains fresh cacheable objects held in RAM with no
* pending fetches. Any objects with a remaining lifetime less than
* the configured minimum lifetime are simply not considered, they will
* become stale before pushing to backing store is worth the cost.
*
* \todo calculate useful cost metrics to improve sorting.
*
* \param[out] lst_out list of candidate objects.
* \param[out] lst_len_out Number of candidate objects in result.
* \return NSERROR_OK with \a lst_out and \a lst_len_out updated or
* error code.
*/
static nserror
build_candidate_list(struct llcache_object ***lst_out, int *lst_len_out)
{
llcache_object *object, *next;
struct llcache_object **lst;
int lst_len = 0;
int remaining_lifetime;
#define MAX_PERSIST_PER_RUN 128
lst = calloc(MAX_PERSIST_PER_RUN, sizeof(struct llcache_object *));
if (lst == NULL) {
return NSERROR_NOMEM;
}
for (object = llcache->cached_objects; object != NULL; object = next) {
next = object->next;
/* Only consider http(s) for the disc cache. */
if (!llcache__scheme_is_persistable(object->url)) {
continue;
}
remaining_lifetime = llcache_object_rfc2616_remaining_lifetime(
&object->cache);
/* cacehable objects with no pending fetches, not
* already on disc and with sufficient lifetime to
* make disc cache worthwhile
*/
if ((object->candidate_count == 0) &&
(object->fetch.fetch == NULL) &&
(object->store_state == LLCACHE_STATE_RAM) &&
(remaining_lifetime > llcache->minimum_lifetime)) {
lst[lst_len] = object;
lst_len++;
if (lst_len == MAX_PERSIST_PER_RUN)
break;
}
}
if (lst_len == 0) {
free(lst);
return NSERROR_NOT_FOUND;
}
/** \todo sort list here */
*lst_len_out = lst_len;
*lst_out = lst;
#undef MAX_PERSIST_PER_RUN
return NSERROR_OK;
}
/**
* Write an object to the backing store.
*
* \param object The object to put in the backing store.
* \param written_out The amount of data written out.
* \param elapsed The time in ms it took to complete the write to backing store.
* \return NSERROR_OK on success or appropriate error code.
*/
static nserror
write_backing_store(struct llcache_object *object,
size_t *written_out,
unsigned long *elapsed)
{
nserror ret;
uint8_t *metadata;
size_t metadatasize;
uint64_t startms = 0;
uint64_t endms = 1000;
nsu_getmonotonic_ms(&startms);
/* put object data in backing store */
ret = guit->llcache->store(object->url,
BACKING_STORE_NONE,
object->source_data,
object->source_len);
if (ret != NSERROR_OK) {
/* unable to put source data in backing store */
return ret;
}
ret = llcache_serialise_metadata(object, &metadata, &metadatasize);
if (ret != NSERROR_OK) {
/* There has been a metadata serialisation error. Ensure the
* already written data object is invalidated.
*/
guit->llcache->invalidate(object->url);
return ret;
}
ret = guit->llcache->store(object->url,
BACKING_STORE_META,
metadata,
metadatasize);
guit->llcache->release(object->url, BACKING_STORE_META);
if (ret != NSERROR_OK) {
/* There has been an error putting the metadata in the
* backing store. Ensure the data object is invalidated.
*/
guit->llcache->invalidate(object->url);
return ret;
}
nsu_getmonotonic_ms(&endms);
object->store_state = LLCACHE_STATE_DISC;
*written_out = object->source_len + metadatasize;
/* by ignoring the overflow this assumes the writeout took
* less than 5 weeks.
*/
*elapsed = endms - startms;
/* ensure the writeout is reported to have taken at least the
* minimal amount of time
*/
if (*elapsed == 0) {
*elapsed = 1;
}
return NSERROR_OK;
}
/**
* Check for overall write performance.
*
* If the overall write bandwidth has fallen below a useful level for
* the backing store to be effective disable it.
*
* It is important to ensure a useful amount of data has been written
* before calculating bandwidths otherwise tiny files taking a
* disproportionately long time to write might trigger this erroneously.
*
* \param p The context pointer passed to the callback.
*/
static void llcache_persist_slowcheck(void *p)
{
uint64_t total_bandwidth; /* total bandwidth */
if (llcache->total_written > (2 * llcache->minimum_bandwidth)) {
total_bandwidth = (llcache->total_written * 1000) / llcache->total_elapsed;
if (total_bandwidth < llcache->minimum_bandwidth) {
NSLOG(llcache, INFO,
"Current bandwidth %"PRIu64" less than minimum %"PRIsizet,
total_bandwidth,
llcache->minimum_bandwidth);
guit->llcache->finalise();
}
}
}
/**
* Possibly write objects data to backing store.
*
* \param p The context pointer passed to the callback.
*/
static void llcache_persist(void *p)
{
nserror ret;
struct llcache_object **lst; /* candidate object list */
int lst_count; /* number of candidates in list */
int idx; /* current candidate object index in list */
int next = -1; /* when the next run should be scheduled for */
unsigned long write_limit; /* max number of bytes to write in this run*/
size_t written; /* all bytes written for a single object */
unsigned long elapsed; /* how long writing an object took */
size_t total_written = 0; /* total bytes written in this run */
unsigned long total_elapsed = 1; /* total ms used to write bytes */
unsigned long total_bandwidth = 0; /* total bandwidth */
ret = build_candidate_list(&lst, &lst_count);
if (ret != NSERROR_OK) {
NSLOG(llcache, DEBUG, "Unable to construct candidate list for persistent writeout");
return;
}
write_limit = (llcache->maximum_bandwidth * llcache->time_quantum) / 1000;
/* obtained a candidate list, make each object persistent in turn */
for (idx = 0; idx < lst_count; idx++) {
ret = write_backing_store(lst[idx], &written, &elapsed);
if (ret != NSERROR_OK) {
continue;
}
/* successfully wrote object to backing store */
total_written += written;
total_elapsed += elapsed;
total_bandwidth = (total_written * 1000) / total_elapsed;
NSLOG(llcache, DEBUG,
"Wrote %"PRIssizet" bytes in %lums bw:%lu %s",
written, elapsed, (written * 1000) / elapsed,
nsurl_access(lst[idx]->url) );
/* check to for the time quantum or the size
* (bandwidth) for this run being exceeded.
*/
if (total_elapsed > llcache->time_quantum) {
NSLOG(llcache, INFO, "Overran timeslot");
/* writeout has exhausted the available time.
* Either the writeout is slow or the last
* object was very large.
*/
if (total_bandwidth < llcache->minimum_bandwidth) {
/* Writeout was slow in this time quantum.
* Schedule a check in the future to see if
* overall performance is too slow to be useful.
*/
guit->misc->schedule(
llcache->time_quantum * 100,
llcache_persist_slowcheck,
NULL);
break;
} else {
if (total_bandwidth > llcache->maximum_bandwidth) {
/* fast writeout of large file
* so calculate delay as if
* write happened only at max
* limit
*/
next = ((total_written * llcache->time_quantum) / write_limit) - total_elapsed;
} else {
next = llcache->time_quantum;
}
break;
}
} else if (total_written > write_limit) {
/* The bandwidth limit has been reached. */
if (total_bandwidth > llcache->maximum_bandwidth) {
/* fast writeout of large file so
* calculate delay as if write
* happened only at max limit
*/
next = ((total_written * llcache->time_quantum) / write_limit) - total_elapsed;
} else {
next = llcache->time_quantum - total_elapsed;
}
break;
}
}
free(lst);
/* Completed list without running out of allowed bytes or time */
if (idx == lst_count) {
/* only reschedule if writing is making any progress at all */
if (total_written > 0) {
next = llcache->time_quantum - total_elapsed;
} else {
next = -1;
}
}
llcache->total_written += total_written;
llcache->total_elapsed += total_elapsed;
NSLOG(llcache, DEBUG,
"writeout size:%"PRIssizet" time:%lu bandwidth:%lubytes/s",
total_written, total_elapsed, total_bandwidth);
NSLOG(llcache, DEBUG, "Rescheduling writeout in %dms", next);
guit->misc->schedule(next, llcache_persist, NULL);
}
/**
* Handler for fetch events
*
* \param msg Fetch event
* \param p Our private data
*/
static void llcache_fetch_callback(const fetch_msg *msg, void *p)
{
nserror error = NSERROR_OK;
llcache_object *object = p;
llcache_event event;
if (llcache == NULL) {
NSLOG(llcache, CRITICAL, "Callback happened after llcache finalisation");
assert(false);
/* In case assertions are off, return here */
return;
}
NSLOG(llcache, DEBUG, "Fetch event %d for %p", msg->type, object);
switch (msg->type) {
case FETCH_HEADER:
/* Received a fetch header */
object->fetch.state = LLCACHE_FETCH_HEADERS;
error = llcache_fetch_process_header(object,
msg->data.header_or_data.buf,
msg->data.header_or_data.len);
break;
/* 3xx responses */
case FETCH_REDIRECT:
/* Request resulted in a redirect */
/* Release candidate, if any */
if (object->candidate != NULL) {
object->candidate->candidate_count--;
object->candidate = NULL;
}
error = llcache_fetch_redirect(object,
msg->data.redirect, &object);
break;
case FETCH_NOTMODIFIED:
/* Conditional request determined that cached object is fresh */
error = llcache_fetch_notmodified(object, &object);
break;
/* Normal 2xx state machine */
case FETCH_DATA:
/* Received some data */
error = llcache_fetch_process_data(object,
msg->data.header_or_data.buf,
msg->data.header_or_data.len);
break;
case FETCH_FINISHED:
/* Finished fetching */
{
uint8_t *temp;
object->fetch.state = LLCACHE_FETCH_COMPLETE;
object->fetch.fetch = NULL;
/* Shrink source buffer to required size */
temp = realloc(object->source_data,
object->source_len);
/* If source_len is 0, then temp may be NULL */
if (temp != NULL || object->source_len == 0) {
object->source_data = temp;
object->source_alloc = object->source_len;
}
llcache_object_cache_update(object);
/* record when the fetch finished */
object->cache.fin_time = time(NULL);
(void) llcache_hsts_update_policy(object);
guit->misc->schedule(5000, llcache_persist, NULL);
}
break;
/* Out-of-band information */
case FETCH_TIMEDOUT:
/* Timed out while trying to fetch. */
error = llcache_fetch_timeout(object);
break;
case FETCH_ERROR:
/* An error occurred while fetching */
/* The fetch has has already been cleaned up by the fetcher */
object->fetch.state = LLCACHE_FETCH_COMPLETE;
object->fetch.fetch = NULL;
/* Release candidate, if any */
if (object->candidate != NULL) {
object->candidate->candidate_count--;
object->candidate = NULL;
}
/* Invalidate cache control data */
llcache_invalidate_cache_control_data(object);
/** \todo Consider using errorcode for something */
event.type = LLCACHE_EVENT_ERROR;
event.data.error.code = NSERROR_UNKNOWN;
event.data.error.msg = msg->data.error;
error = llcache_send_event_to_users(object, &event);
break;
case FETCH_PROGRESS:
/* Progress update */
event.type = LLCACHE_EVENT_PROGRESS;
event.data.progress_msg = msg->data.progress;
error = llcache_send_event_to_users(object, &event);
break;
case FETCH_CERTS:
/* Certificate information from the fetch */
/* Persist the chain onto our object */
error = cert_chain_dup(msg->data.chain, &object->chain);
if (error != NSERROR_OK) {
/* Now pass on the event */
event.type = LLCACHE_EVENT_GOT_CERTS;
event.data.chain = msg->data.chain;
error = llcache_send_event_to_users(object, &event);
}
break;
/* Events requiring action */
case FETCH_AUTH:
/* Need Authentication */
/* Release candidate, if any */
if (object->candidate != NULL) {
object->candidate->candidate_count--;
object->candidate = NULL;
}
error = llcache_fetch_auth(object, msg->data.auth.realm);
break;
case FETCH_CERT_ERR:
/* Something went wrong when validating TLS certificates */
/* Release candidate, if any */
if (object->candidate != NULL) {
object->candidate->candidate_count--;
object->candidate = NULL;
}
error = llcache_fetch_cert_error(object);
break;
case FETCH_SSL_ERR:
/* TLS connection setup failed */
/* Release candidate, if any */
if (object->candidate != NULL) {
object->candidate->candidate_count--;
object->candidate = NULL;
}
error = llcache_fetch_ssl_error(object);
break;
}
/* Deal with any errors reported by event handlers */
if (error != NSERROR_OK) {
if (error == NSERROR_NOMEM) {
/* attempt to purge the cache to free some
* memory. will not help this fetch, but may
* allow the UI to report errors etc.
*/
llcache_clean(true);
}
if (object->fetch.fetch != NULL) {
fetch_abort(object->fetch.fetch);
object->fetch.fetch = NULL;
/* Invalidate cache control data */
llcache_invalidate_cache_control_data(object);
object->fetch.state = LLCACHE_FETCH_COMPLETE;
}
}
/* There may be users which are not caught up so schedule ourselves */
llcache_users_not_caught_up();
}
/**
* Find a user of a low-level cache object
*
* \param handle External cache handle to search for
* \return Pointer to corresponding user, or NULL if not found
*/
static llcache_object_user *llcache_object_find_user(
const llcache_handle *handle)
{
llcache_object_user *user;
assert(handle->object != NULL);
for (user = handle->object->users; user != NULL; user = user->next) {
if (user->handle == handle)
break;
}
return user;
}
/**
* Determine if a low-level cache object resides in a given list
*
* \param object Object to search for
* \param list List to search in
* \return True if object resides in list, false otherwise
*/
static bool llcache_object_in_list(const llcache_object *object,
const llcache_object *list)
{
while (list != NULL) {
if (list == object)
break;
list = list->next;
}
return list != NULL;
}
/**
* Notify users of an object's current state
*
* \param object Object to notify users about
* \return NSERROR_OK on success, appropriate error otherwise
*/
static nserror llcache_object_notify_users(llcache_object *object)
{
nserror error;
llcache_object_user *user, *next_user;
llcache_event event;
bool emitted_notify = false;
/**
* State transitions and event emission for users.
* Rows: user state. Cols: object state.
*
* User - Obj INIT HEADERS DATA COMPLETE
* INIT - T T* T*
* HEADERS - - T T*
* DATA - - M T
* COMPLETE - - - -
*
* T => transition user to object state
* M => no transition required, but may need to emit event
*
* The transitions marked with an asterisk can be removed by moving
* the user context into the subsequent state and then reevaluating.
*
* Events are issued as follows:
*
* HAD_HEADERS: on transition from HEADERS -> DATA state
* HAD_DATA : in DATA state, whenever there's new source data
* DONE : on transition from DATA -> COMPLETE state
*/
for (user = object->users; user != NULL; user = next_user) {
/* Emit necessary events to bring the user up-to-date */
llcache_handle *handle = user->handle;
const llcache_fetch_state objstate = object->fetch.state;
/* Flag that this user is the current iteration target
* in case the client attempts to destroy it underneath us */
user->iterator_target = true;
/* A note on the computation of next_user:
*
* Within this loop, we may make a number of calls to
* client code. Our contract with clients is that they
* can do whatever they like from within their callback
* handlers. This is so that we limit the pain of
* reentrancy to this module alone.
*
* One of the things a client can do from within its
* callback handler is to remove users from this object's
* user list. In the common case, the user they attempt
* to remove is the current iteration target, and we
* already protect against that causing problems here.
* However, no such protection exists if the client
* attempts to remove other users from this object's
* user list.
*
* Therefore, we cannot compute next_user up-front
* and expect it to remain valid across calls to
* client code (as the identity of the next user
* in the list may change underneath us). Instead,
* we must compute next_user at the point where we
* are about to cause another iteration of this loop
* (i.e. at the very end, and also at the points where
* continue is used)
*/
if (handle->state != objstate) {
if (emitted_notify == false) {
NSLOG(llcache, DEBUG,
"Notifying users of %p",
object);
emitted_notify = true;
}
NSLOG(llcache, DEBUG,
"User %p state: %d Object state: %d",
user,
handle->state,
objstate);
}
/* User: INIT, Obj: HEADERS, DATA, COMPLETE => User->HEADERS */
if (handle->state == LLCACHE_FETCH_INIT &&
objstate > LLCACHE_FETCH_INIT) {
handle->state = LLCACHE_FETCH_HEADERS;
/* Emit any certificate data we hold */
if (object->chain != NULL) {
event.type = LLCACHE_EVENT_GOT_CERTS;
event.data.chain = object->chain;
error = handle->cb(handle, &event, handle->pw);
} else {
error = NSERROR_OK;
}
if (user->queued_for_delete) {
next_user = user->next;
llcache_object_remove_user(object, user);
llcache_object_user_destroy(user);
if (error != NSERROR_OK)
return error;
continue;
} else if (error == NSERROR_NEED_DATA) {
/* User requested replay */
handle->state = LLCACHE_FETCH_HEADERS;
/* Continue with the next user -- we'll
* reemit the event next time round */
user->iterator_target = false;
next_user = user->next;
llcache_users_not_caught_up();
continue;
} else if (error != NSERROR_OK) {
user->iterator_target = false;
return error;
}
}
/* User: HEADERS, Obj: DATA, COMPLETE => User->DATA */
if (handle->state == LLCACHE_FETCH_HEADERS &&
objstate > LLCACHE_FETCH_HEADERS) {
handle->state = LLCACHE_FETCH_DATA;
/* Emit HAD_HEADERS event */
event.type = LLCACHE_EVENT_HAD_HEADERS;
error = handle->cb(handle, &event, handle->pw);
if (user->queued_for_delete) {
next_user = user->next;
llcache_object_remove_user(object, user);
llcache_object_user_destroy(user);
if (error != NSERROR_OK)
return error;
continue;
} else if (error == NSERROR_NEED_DATA) {
/* User requested replay */
handle->state = LLCACHE_FETCH_HEADERS;
/* Continue with the next user -- we'll
* reemit the event next time round */
user->iterator_target = false;
next_user = user->next;
llcache_users_not_caught_up();
continue;
} else if (error != NSERROR_OK) {
user->iterator_target = false;
return error;
}
}
/* User: DATA, Obj: DATA, COMPLETE, more source available */
if (handle->state == LLCACHE_FETCH_DATA &&
objstate >= LLCACHE_FETCH_DATA &&
object->source_len > handle->bytes) {
size_t orig_handle_read;
/* Construct HAD_DATA event */
event.type = LLCACHE_EVENT_HAD_DATA;
event.data.data.buf =
object->source_data + handle->bytes;
event.data.data.len =
object->source_len - handle->bytes;
/* Update record of last byte emitted */
if (object->fetch.flags &
LLCACHE_RETRIEVE_STREAM_DATA) {
/* Streaming, so reset to zero to
* minimise amount of cached source data.
* Additionally, we don't support replay
* when streaming. */
orig_handle_read = 0;
handle->bytes = object->source_len = 0;
} else {
orig_handle_read = handle->bytes;
handle->bytes = object->source_len;
}
/* Emit event */
error = handle->cb(handle, &event, handle->pw);
if (user->queued_for_delete) {
next_user = user->next;
llcache_object_remove_user(object, user);
llcache_object_user_destroy(user);
if (error != NSERROR_OK)
return error;
continue;
} else if (error == NSERROR_NEED_DATA) {
/* User requested replay */
handle->bytes = orig_handle_read;
/* Continue with the next user -- we'll
* reemit the data next time round */
user->iterator_target = false;
next_user = user->next;
llcache_users_not_caught_up();
continue;
} else if (error != NSERROR_OK) {
user->iterator_target = false;
return error;
}
}
/* User: DATA, Obj: COMPLETE => User->COMPLETE */
if (handle->state == LLCACHE_FETCH_DATA &&
objstate > LLCACHE_FETCH_DATA) {
handle->state = LLCACHE_FETCH_COMPLETE;
/* Emit DONE event */
event.type = LLCACHE_EVENT_DONE;
error = handle->cb(handle, &event, handle->pw);
if (user->queued_for_delete) {
next_user = user->next;
llcache_object_remove_user(object, user);
llcache_object_user_destroy(user);
if (error != NSERROR_OK)
return error;
continue;
} else if (error == NSERROR_NEED_DATA) {
/* User requested replay */
handle->state = LLCACHE_FETCH_DATA;
/* Continue with the next user -- we'll
* reemit the event next time round */
user->iterator_target = false;
next_user = user->next;
llcache_users_not_caught_up();
continue;
} else if (error != NSERROR_OK) {
user->iterator_target = false;
return error;
}
}
/* No longer the target of an iterator */
user->iterator_target = false;
next_user = user->next;
}
return NSERROR_OK;
}
/**
* Make a snapshot of the current state of an llcache_object.
*
* This has the side-effect of the new object being non-cacheable,
* also not-fetching and not a candidate for any other object.
*
* Also note that this new object has no users and at least one
* should be assigned to it before llcache_clean is entered or it
* will be immediately cleaned up.
*
* \param object The object to take a snapshot of
* \param snapshot Pointer to receive snapshot of \a object
* \return NSERROR_OK on success, appropriate error otherwise
*/
static nserror
llcache_object_snapshot(llcache_object *object, llcache_object **snapshot)
{
llcache_object *newobj;
nserror error;
error = llcache_object_new(object->url, &newobj);
if (error != NSERROR_OK)
return error;
newobj->source_alloc = newobj->source_len = object->source_len;
if (object->source_len > 0) {
newobj->source_data = malloc(newobj->source_alloc);
if (newobj->source_data == NULL) {
llcache_object_destroy(newobj);
return NSERROR_NOMEM;
}
memcpy(newobj->source_data, object->source_data,
newobj->source_len);
}
if (object->num_headers > 0) {
newobj->headers = calloc(sizeof(llcache_header),
object->num_headers);
if (newobj->headers == NULL) {
llcache_object_destroy(newobj);
return NSERROR_NOMEM;
}
while (newobj->num_headers < object->num_headers) {
llcache_header *nh =
&(newobj->headers[newobj->num_headers]);
llcache_header *oh =
&(object->headers[newobj->num_headers]);
newobj->num_headers += 1;
nh->name = strdup(oh->name);
nh->value = strdup(oh->value);
if (nh->name == NULL || nh->value == NULL) {
llcache_object_destroy(newobj);
return NSERROR_NOMEM;
}
}
}
if (object->chain != NULL) {
error = cert_chain_dup(object->chain, &newobj->chain);
if (error != NSERROR_OK) {
llcache_object_destroy(newobj);
return error;
}
}
newobj->fetch.state = LLCACHE_FETCH_COMPLETE;
*snapshot = newobj;
return NSERROR_OK;
}
/**
* total ram usage of object
*
* \param object The object to calculate the total RAM usage of.
* \return The total RAM usage in bytes.
*/
static inline uint32_t
total_object_size(llcache_object *object)
{
uint32_t tot;
size_t hdrc;
tot = sizeof(*object);
tot += nsurl_length(object->url);
if (object->source_data != NULL) {
tot += object->source_len;
}
tot += sizeof(llcache_header) * object->num_headers;
for (hdrc = 0; hdrc < object->num_headers; hdrc++) {
if (object->headers[hdrc].name != NULL) {
tot += strlen(object->headers[hdrc].name);
}
if (object->headers[hdrc].value != NULL) {
tot += strlen(object->headers[hdrc].value);
}
}
tot += cert_chain_size(object->chain);
return tot;
}
/******************************************************************************
* Public API *
******************************************************************************/
/*
* Attempt to clean the cache
*
* The memory cache cleaning discards objects in order of increasing value.
*
* Exported interface documented in llcache.h
*/
void llcache_clean(bool purge)
{
llcache_object *object, *next;
uint32_t llcache_size = 0;
int remaining_lifetime;
uint32_t limit;
NSLOG(llcache, DEBUG, "Attempting cache clean");
/* If the cache is being purged set the size limit to zero. */
if (purge) {
limit = 0;
} else {
limit = llcache->limit;
}
/* Uncacheable objects with no users or fetches */
for (object = llcache->uncached_objects;
object != NULL;
object = next) {
next = object->next;
/* The candidate count of uncacheable objects is always 0 */
if ((object->users == NULL) &&
(object->candidate_count == 0) &&
(object->fetch.fetch == NULL)) {
NSLOG(llcache, DEBUG, "Discarding uncachable object with no users (%p) %s",
object, nsurl_access(object->url));
llcache_object_remove_from_list(object,
&llcache->uncached_objects);
llcache_object_destroy(object);
} else {
llcache_size += total_object_size(object);
}
}
/* Stale cacheable objects with no users or pending fetches */
for (object = llcache->cached_objects;
object != NULL;
object = next) {
next = object->next;
remaining_lifetime = llcache_object_rfc2616_remaining_lifetime(
&object->cache);
if ((object->users == NULL) &&
(object->candidate_count == 0) &&
(object->fetch.fetch == NULL) &&
(remaining_lifetime <= 0)) {
/* object is stale */
NSLOG(llcache, DEBUG, "discarding stale cacheable object with no "
"users or pending fetches (%p) %s",
object, nsurl_access(object->url));
llcache_object_remove_from_list(object,
&llcache->cached_objects);
if (object->store_state == LLCACHE_STATE_DISC) {
guit->llcache->invalidate(object->url);
}
llcache_object_destroy(object);
} else {
/* object has users so account for the storage */
llcache_size += total_object_size(object);
}
}
/* if the cache limit is exceeded try to make some objects
* persistent so their RAM can be reclaimed in the next
* step
*/
if (limit < llcache_size) {
llcache_persist(NULL);
}
/* Source data of fresh cacheable objects with no users, no
* pending fetches and pushed to persistent store while the
* cache exceeds the configured size.
*/
for (object = llcache->cached_objects;
((limit < llcache_size) && (object != NULL));
object = next) {
next = object->next;
if ((object->users == NULL) &&
(object->candidate_count == 0) &&
(object->fetch.fetch == NULL) &&
(object->store_state == LLCACHE_STATE_DISC)) {
guit->llcache->release(object->url, BACKING_STORE_NONE);
object->source_data = NULL;
llcache_size -= object->source_len;
NSLOG(llcache, DEBUG,
"Freeing source data for %p len:%"PRIssizet,
object, object->source_len);
}
}
/* Fresh cacheable objects with no users, no pending fetches
* and pushed to persistent store while the cache exceeds
* the configured size. Effectively just the llcache object metadata.
*/
for (object = llcache->cached_objects;
((limit < llcache_size) && (object != NULL));
object = next) {
next = object->next;
if ((object->users == NULL) &&
(object->candidate_count == 0) &&
(object->fetch.fetch == NULL) &&
(object->store_state == LLCACHE_STATE_DISC) &&
(object->source_data == NULL)) {
NSLOG(llcache, DEBUG,
"discarding backed object len:%"PRIssizet" age:%ld (%p) %s",
object->source_len,
(long)(time(NULL) - object->last_used),
object,
nsurl_access(object->url));
llcache_size -= total_object_size(object);
llcache_object_remove_from_list(object,
&llcache->cached_objects);
llcache_object_destroy(object);
}
}
/* Fresh cacheable objects with no users or pending fetches
* while the cache exceeds the configured size. These are the
* most valuable objects as replacing them is a full network
* fetch
*/
for (object = llcache->cached_objects;
((limit < llcache_size) && (object != NULL));
object = next) {
next = object->next;
if ((object->users == NULL) &&
(object->candidate_count == 0) &&
(object->fetch.fetch == NULL) &&
(object->store_state == LLCACHE_STATE_RAM)) {
NSLOG(llcache, DEBUG,
"discarding fresh object len:%"PRIssizet" age:%ld (%p) %s",
object->source_len,
(long)(time(NULL) - object->last_used),
object,
nsurl_access(object->url));
llcache_size -= object->source_len + sizeof(*object);
llcache_object_remove_from_list(object,
&llcache->cached_objects);
llcache_object_destroy(object);
}
}
NSLOG(llcache, DEBUG, "Size: %u (limit: %u)", llcache_size, limit);
}
/* Exported interface documented in content/llcache.h */
nserror
llcache_initialise(const struct llcache_parameters *prm)
{
llcache = calloc(1, sizeof(struct llcache_s));
if (llcache == NULL) {
return NSERROR_NOMEM;
}
llcache->limit = prm->limit;
llcache->minimum_lifetime = prm->minimum_lifetime;
llcache->minimum_bandwidth = prm->minimum_bandwidth;
llcache->maximum_bandwidth = prm->maximum_bandwidth;
llcache->time_quantum = prm->time_quantum;
llcache->fetch_attempts = prm->fetch_attempts;
llcache->all_caught_up = true;
NSLOG(llcache, INFO,
"llcache initialising with a limit of %d bytes",
llcache->limit);
/* backing store initialisation */
return guit->llcache->initialise(&prm->store);
}
/* Exported interface documented in content/llcache.h */
void llcache_finalise(void)
{
llcache_object *object, *next;
uint64_t total_bandwidth = 0; /* total bandwidth */
/* Attempt to persist anything we have left lying around */
llcache_persist(NULL);
/* Now clear the persistence callback */
guit->misc->schedule(-1, llcache_persist, NULL);
/* Clean uncached objects */
for (object = llcache->uncached_objects; object != NULL; object = next) {
llcache_object_user *user, *next_user;
next = object->next;
for (user = object->users; user != NULL; user = next_user) {
next_user = user->next;
if (user->handle != NULL)
free(user->handle);
free(user);
}
/* Fetch system has already been destroyed */
object->fetch.fetch = NULL;
llcache_object_destroy(object);
}
/* Clean cached objects */
for (object = llcache->cached_objects; object != NULL; object = next) {
llcache_object_user *user, *next_user;
next = object->next;
for (user = object->users; user != NULL; user = next_user) {
next_user = user->next;
if (user->handle != NULL)
free(user->handle);
free(user);
}
/* Fetch system has already been destroyed */
object->fetch.fetch = NULL;
llcache_object_destroy(object);
}
/* backing store finalisation */
guit->llcache->finalise();
if (llcache->total_elapsed > 0) {
total_bandwidth = (llcache->total_written * 1000) /
llcache->total_elapsed;
}
NSLOG(llcache, INFO,
"Backing store wrote %"PRIu64" bytes in %"PRIu64" ms ""(average %"PRIu64" bytes/second)",
llcache->total_written,
llcache->total_elapsed,
total_bandwidth);
free(llcache);
llcache = NULL;
}
/**
* Catch up the cache users with state changes from fetchers.
*
* \param ignored We ignore this because all our state comes from llcache.
*/
static void llcache_catch_up_all_users(void *ignored)
{
llcache_object *object;
/* Assume after this we'll be all caught up. If any user of a handle
* defers then we'll invalidate all_caught_up and reschedule via
* llcache_users_not_caught_up()
*/
llcache->all_caught_up = true;
/* Catch new users up with state of objects */
for (object = llcache->cached_objects; object != NULL;
object = object->next) {
llcache_object_notify_users(object);
}
for (object = llcache->uncached_objects; object != NULL;
object = object->next) {
llcache_object_notify_users(object);
}
}
/**
* Ask for ::llcache_catch_up_all_users to be scheduled ASAP to pump the
* user state machines.
*/
static void llcache_users_not_caught_up(void)
{
if (llcache->all_caught_up) {
llcache->all_caught_up = false;
guit->misc->schedule(0, llcache_catch_up_all_users, NULL);
}
}
/* Exported interface documented in content/llcache.h */
nserror llcache_handle_retrieve(nsurl *url, uint32_t flags,
nsurl *referer, const llcache_post_data *post,
llcache_handle_callback cb, void *pw,
llcache_handle **result)
{
nserror error;
llcache_object_user *user;
llcache_object *object;
nsurl *hsts_url;
bool hsts_in_use;
/* Perform HSTS transform */
error = llcache_hsts_transform_url(url, &hsts_url, &hsts_in_use);
if (error != NSERROR_OK) {
return error;
}
/* Can we fetch this URL at all? */
if (fetch_can_fetch(hsts_url) == false) {
nsurl_unref(hsts_url);
return NSERROR_NO_FETCH_HANDLER;
}
/* Create a new object user */
error = llcache_object_user_new(cb, pw, &user);
if (error != NSERROR_OK) {
nsurl_unref(hsts_url);
return error;
}
/* Retrieve a suitable object from the cache,
* creating a new one if needed. */
error = llcache_object_retrieve(hsts_url, flags, referer, post, 0,
hsts_in_use, &object);
if (error != NSERROR_OK) {
llcache_object_user_destroy(user);
nsurl_unref(hsts_url);
return error;
}
/* Add user to object */
llcache_object_add_user(object, user);
*result = user->handle;
/* Users exist which are now not caught up! */
llcache_users_not_caught_up();
nsurl_unref(hsts_url);
return NSERROR_OK;
}
/* Exported interface documented in content/llcache.h */
nserror llcache_handle_change_callback(llcache_handle *handle,
llcache_handle_callback cb, void *pw)
{
handle->cb = cb;
handle->pw = pw;
return NSERROR_OK;
}
/* Exported interface documented in content/llcache.h */
nserror llcache_handle_release(llcache_handle *handle)
{
nserror error = NSERROR_OK;
llcache_object *object = handle->object;
llcache_object_user *user = llcache_object_find_user(handle);
assert(user != NULL);
if (user->iterator_target) {
/* Can't remove / delete user object if it's
* the target of an iterator */
user->queued_for_delete = true;
} else {
/* Remove the user from the object and destroy it */
error = llcache_object_remove_user(object, user);
if (error == NSERROR_OK) {
error = llcache_object_user_destroy(user);
}
}
return error;
}
/* Exported interface documented in content/llcache.h */
nserror llcache_handle_clone(llcache_handle *handle, llcache_handle **result)
{
nserror error;
llcache_object_user *newuser;
error = llcache_object_user_new(handle->cb, handle->pw, &newuser);
if (error == NSERROR_OK) {
llcache_object_add_user(handle->object, newuser);
newuser->handle->state = handle->state;
*result = newuser->handle;
}
return error;
}
/* See llcache.h for documentation */
nserror llcache_handle_abort(llcache_handle *handle)
{
llcache_object_user *user = llcache_object_find_user(handle);
llcache_object *object = handle->object, *newobject;
nserror error = NSERROR_OK;
bool all_alone = true;
/* Determine if we are the only user */
if (user->prev != NULL)
all_alone = false;
if (user->next != NULL)
all_alone = false;
if (all_alone == false) {
/* We must snapshot this object */
error = llcache_object_snapshot(object, &newobject);
if (error != NSERROR_OK)
return error;
/* Move across to the new object */
if (user->iterator_target) {
/* User is current iterator target, clone it */
llcache_object_user *newuser =
calloc(1, sizeof(llcache_object_user));
if (newuser == NULL) {
llcache_object_destroy(newobject);
return NSERROR_NOMEM;
}
/* Move handle across to clone */
newuser->handle = user->handle;
user->handle = NULL;
/* Mark user as needing deletion */
user->queued_for_delete = true;
llcache_object_add_user(newobject, newuser);
} else {
llcache_object_remove_user(object, user);
llcache_object_add_user(newobject, user);
}
/* Add new object to uncached list */
llcache_object_add_to_list(newobject,
&llcache->uncached_objects);
} else {
/* We're the only user, so abort any fetch in progress */
if (object->fetch.fetch != NULL) {
fetch_abort(object->fetch.fetch);
object->fetch.fetch = NULL;
}
object->fetch.state = LLCACHE_FETCH_COMPLETE;
/* Invalidate cache control data */
llcache_invalidate_cache_control_data(object);
}
return error;
}
/* See llcache.h for documentation */
nserror llcache_handle_force_stream(llcache_handle *handle)
{
llcache_object_user *user = llcache_object_find_user(handle);
llcache_object *object = handle->object;
/* Cannot stream if there are multiple users */
if (user->prev != NULL || user->next != NULL)
return NSERROR_OK;
/* Forcibly uncache this object */
if (llcache_object_in_list(object, llcache->cached_objects)) {
llcache_object_remove_from_list(object,
&llcache->cached_objects);
llcache_object_add_to_list(object, &llcache->uncached_objects);
}
object->fetch.flags |= LLCACHE_RETRIEVE_STREAM_DATA;
return NSERROR_OK;
}
/* See llcache.h for documentation */
nserror llcache_handle_invalidate_cache_data(llcache_handle *handle)
{
if ((handle->object != NULL) &&
(handle->object->fetch.fetch == NULL) &&
(handle->object->cache.no_cache == LLCACHE_VALIDATE_FRESH)) {
/* mark the cached object as requiring validation */
handle->object->cache.no_cache = LLCACHE_VALIDATE_ONCE;
}
return NSERROR_OK;
}
/* See llcache.h for documentation */
nsurl *llcache_handle_get_url(const llcache_handle *handle)
{
return handle->object != NULL ? handle->object->url : NULL;
}
/* See llcache.h for documentation */
const uint8_t *llcache_handle_get_source_data(const llcache_handle *handle,
size_t *size)
{
*size = handle->object != NULL ? handle->object->source_len : 0;
return handle->object != NULL ? handle->object->source_data : NULL;
}
/* See llcache.h for documentation */
const char *llcache_handle_get_header(const llcache_handle *handle,
const char *key)
{
const llcache_object *object = handle->object;
size_t i;
if (object == NULL)
return NULL;
/* About as trivial as possible */
for (i = 0; i < object->num_headers; i++) {
if (strcasecmp(key, object->headers[i].name) == 0)
return object->headers[i].value;
}
return NULL;
}
/* See llcache.h for documentation */
bool llcache_handle_references_same_object(const llcache_handle *a,
const llcache_handle *b)
{
return a->object == b->object;
}
Reference in New Issue View Git Blame Copy Permalink