mirror of
https://github.com/netsurf-browser/netsurf
synced 2024-11-25 07:49:38 +03:00
3855 lines
100 KiB
C
3855 lines
100 KiB
C
/*
|
|
* Copyright 2009 John-Mark Bell <jmb@netsurf-browser.org>
|
|
*
|
|
* This file is part of NetSurf, http://www.netsurf-browser.org/
|
|
*
|
|
* NetSurf is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation; version 2 of the License.
|
|
*
|
|
* NetSurf is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
/**
|
|
* \file
|
|
*
|
|
* Low-level resource cache implementation
|
|
*
|
|
* This is the implementation of the low level cache. This cache
|
|
* stores source objects in memory and may use a persistent backing
|
|
* store to extend their lifetime.
|
|
*
|
|
* \todo fix writeout conditions and ordering.
|
|
*
|
|
* \todo instrument and (auto)tune
|
|
*
|
|
*/
|
|
|
|
#include <stdlib.h>
|
|
#include <stdint.h>
|
|
#include <string.h>
|
|
#include <strings.h>
|
|
#include <nsutils/time.h>
|
|
|
|
#include "netsurf/inttypes.h"
|
|
#include "utils/config.h"
|
|
#include "utils/corestrings.h"
|
|
#include "utils/log.h"
|
|
#include "utils/messages.h"
|
|
#include "utils/nsurl.h"
|
|
#include "utils/utils.h"
|
|
#include "utils/time.h"
|
|
#include "utils/http.h"
|
|
#include "netsurf/misc.h"
|
|
#include "desktop/gui_internal.h"
|
|
|
|
#include "content/fetch.h"
|
|
#include "content/backing_store.h"
|
|
#include "content/urldb.h"
|
|
|
|
/**
|
|
* State of a low-level cache object fetch.
|
|
*/
|
|
typedef enum {
|
|
LLCACHE_FETCH_INIT, /**< Initial state, before fetch */
|
|
LLCACHE_FETCH_HEADERS, /**< Fetching headers */
|
|
LLCACHE_FETCH_DATA, /**< Fetching object data */
|
|
LLCACHE_FETCH_COMPLETE /**< Fetch completed */
|
|
} llcache_fetch_state;
|
|
|
|
/**
|
|
* Type of low-level cache object.
|
|
*/
|
|
typedef struct llcache_object llcache_object;
|
|
|
|
/**
|
|
* Handle to low-level cache object.
|
|
*/
|
|
struct llcache_handle {
|
|
llcache_object *object; /**< Pointer to associated object */
|
|
|
|
llcache_handle_callback cb; /**< Client callback */
|
|
void *pw; /**< Client data */
|
|
|
|
llcache_fetch_state state; /**< Last known state of object fetch */
|
|
size_t bytes; /**< Last reported byte count */
|
|
};
|
|
|
|
/**
|
|
* Low-level cache object user record.
|
|
*/
|
|
typedef struct llcache_object_user {
|
|
llcache_handle *handle; /**< Handle data for client */
|
|
|
|
bool iterator_target; /**< This is the iterator target */
|
|
bool queued_for_delete; /**< This user is queued for deletion */
|
|
|
|
struct llcache_object_user *prev; /**< Previous in list */
|
|
struct llcache_object_user *next; /**< Next in list */
|
|
} llcache_object_user;
|
|
|
|
/**
|
|
* Low-level cache object fetch context.
|
|
*/
|
|
typedef struct {
|
|
uint32_t flags; /**< Fetch flags */
|
|
nsurl *referer; /**< Referring URL, or NULL if none */
|
|
llcache_post_data *post; /**< POST data, or NULL for GET */
|
|
|
|
struct fetch *fetch; /**< Fetch handle for this object */
|
|
|
|
llcache_fetch_state state; /**< Current state of object fetch */
|
|
|
|
uint32_t redirect_count; /**< Count of redirects followed */
|
|
|
|
uint32_t retries_remaining; /**< Number of times to retry on timeout */
|
|
|
|
bool hsts_in_use; /**< Whether HSTS applies to this fetch */
|
|
|
|
bool tried_with_auth; /**< Whether we've tried with auth */
|
|
|
|
bool tried_with_tls_downgrade; /**< Whether we've tried TLS <= 1.0 */
|
|
|
|
bool tainted_tls; /**< Whether the TLS transport is tainted */
|
|
} llcache_fetch_ctx;
|
|
|
|
/**
|
|
* Validation control.
|
|
*/
|
|
typedef enum {
|
|
LLCACHE_VALIDATE_FRESH, /**< Only revalidate if not fresh */
|
|
LLCACHE_VALIDATE_ALWAYS, /**< Always revalidate */
|
|
LLCACHE_VALIDATE_ONCE /**< Revalidate once only */
|
|
} llcache_validate;
|
|
|
|
/**
|
|
* Cache control value for invalid age.
|
|
*/
|
|
#define INVALID_AGE -1
|
|
|
|
/** Cache control data */
|
|
typedef struct {
|
|
time_t req_time; /**< Time of request */
|
|
time_t res_time; /**< Time of response */
|
|
time_t fin_time; /**< Time of request completion */
|
|
time_t date; /**< Date: response header */
|
|
time_t expires; /**< Expires: response header */
|
|
int age; /**< Age: response header */
|
|
int max_age; /**< Max-Age Cache-control parameter */
|
|
llcache_validate no_cache; /**< No-Cache Cache-control parameter */
|
|
char *etag; /**< Etag: response header */
|
|
time_t last_modified; /**< Last-Modified: response header */
|
|
} llcache_cache_control;
|
|
|
|
/** Representation of a fetch header */
|
|
typedef struct {
|
|
char *name; /**< Header name */
|
|
char *value; /**< Header value */
|
|
} llcache_header;
|
|
|
|
/** Current status of an object's data */
|
|
typedef enum {
|
|
LLCACHE_STATE_RAM = 0, /**< source data is stored in RAM only */
|
|
LLCACHE_STATE_DISC, /**< source data is stored on disc */
|
|
} llcache_store_state;
|
|
|
|
/**
|
|
* Low-level cache object
|
|
*
|
|
* \todo Consider whether a list is a sane container.
|
|
*/
|
|
struct llcache_object {
|
|
llcache_object *prev; /**< Previous in list */
|
|
llcache_object *next; /**< Next in list */
|
|
|
|
nsurl *url; /**< Post-redirect URL for object */
|
|
|
|
/** \todo We need a generic dynamic buffer object */
|
|
uint8_t *source_data; /**< Source data for object */
|
|
size_t source_len; /**< Byte length of source data */
|
|
size_t source_alloc; /**< Allocated size of source buffer */
|
|
|
|
llcache_store_state store_state; /**< where the data for the object is stored */
|
|
|
|
llcache_object_user *users; /**< List of users */
|
|
|
|
llcache_fetch_ctx fetch; /**< Fetch context for object */
|
|
|
|
llcache_cache_control cache; /**< Cache control data for object */
|
|
llcache_object *candidate; /**< Object to use, if fetch determines
|
|
* that it is still fresh
|
|
*/
|
|
uint32_t candidate_count; /**< Count of objects this is a
|
|
* candidate for
|
|
*/
|
|
|
|
llcache_header *headers; /**< Fetch headers */
|
|
size_t num_headers; /**< Number of fetch headers */
|
|
|
|
/* Instrumentation. These elements are strictly for information
|
|
* to improve the cache performance and to provide performance
|
|
* metrics. The values are non-authoritative and must not be used to
|
|
* determine object lifetime etc.
|
|
*/
|
|
time_t last_used; /**< time the last user was removed from the object */
|
|
};
|
|
|
|
/**
|
|
* Core llcache control context.
|
|
*/
|
|
struct llcache_s {
|
|
/** Head of the low-level cached object list */
|
|
llcache_object *cached_objects;
|
|
|
|
/** Head of the low-level uncached object list */
|
|
llcache_object *uncached_objects;
|
|
|
|
/** The target upper bound for the RAM cache size */
|
|
uint32_t limit;
|
|
|
|
/** The number of fetch attempts we make when timing out */
|
|
uint32_t fetch_attempts;
|
|
|
|
/** Whether or not our users are caught up */
|
|
bool all_caught_up;
|
|
|
|
|
|
/* backing store elements */
|
|
|
|
|
|
/**
|
|
* The minimum lifetime to consider sending objects to backing
|
|
* store.
|
|
*/
|
|
int minimum_lifetime;
|
|
|
|
/**
|
|
* The time over which to apply the bandwidth calculations in ms
|
|
*/
|
|
unsigned long time_quantum;
|
|
|
|
/**
|
|
* The minimum bandwidth to allow the backing store to use in
|
|
* bytes/second. Below this the backing store will be
|
|
* disabled.
|
|
*/
|
|
size_t minimum_bandwidth;
|
|
|
|
/**
|
|
* The maximum bandwidth to allow the backing store to use in
|
|
* bytes/second
|
|
*/
|
|
size_t maximum_bandwidth;
|
|
|
|
/**
|
|
* Total number of bytes written to backing store.
|
|
*/
|
|
uint64_t total_written;
|
|
|
|
/**
|
|
* Total number of milliseconds taken to write to backing store.
|
|
*/
|
|
uint64_t total_elapsed;
|
|
|
|
};
|
|
|
|
/** low level cache state */
|
|
static struct llcache_s *llcache = NULL;
|
|
|
|
/* forward referenced callback function */
|
|
static void llcache_fetch_callback(const fetch_msg *msg, void *p);
|
|
|
|
/* forward referenced catch up function */
|
|
static void llcache_users_not_caught_up(void);
|
|
|
|
|
|
/******************************************************************************
|
|
* Low-level cache internals *
|
|
******************************************************************************/
|
|
|
|
/**
|
|
* Create a new object user.
|
|
*
|
|
* \param cb Callback routine.
|
|
* \param pw Private data for callback.
|
|
* \param user Pointer to location to receive result.
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*/
|
|
static nserror llcache_object_user_new(llcache_handle_callback cb, void *pw,
|
|
llcache_object_user **user)
|
|
{
|
|
llcache_handle *h;
|
|
llcache_object_user *u;
|
|
|
|
h = calloc(1, sizeof(llcache_handle));
|
|
if (h == NULL) {
|
|
return NSERROR_NOMEM;
|
|
}
|
|
|
|
u = calloc(1, sizeof(llcache_object_user));
|
|
if (u == NULL) {
|
|
free(h);
|
|
return NSERROR_NOMEM;
|
|
}
|
|
|
|
h->cb = cb;
|
|
h->pw = pw;
|
|
|
|
u->handle = h;
|
|
|
|
NSLOG(llcache, DEBUG,
|
|
"Created user %p (%p, %p, %p)", u, h, (void *) cb, pw);
|
|
|
|
*user = u;
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Destroy an object user
|
|
*
|
|
* \param user User to destroy
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*
|
|
* \pre User is not attached to an object
|
|
*/
|
|
static nserror llcache_object_user_destroy(llcache_object_user *user)
|
|
{
|
|
NSLOG(llcache, DEBUG, "Destroyed user %p", user);
|
|
|
|
assert(user->next == NULL);
|
|
assert(user->prev == NULL);
|
|
|
|
if (user->handle != NULL)
|
|
free(user->handle);
|
|
|
|
free(user);
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Remove a user from a low-level cache object
|
|
*
|
|
* \param object Object to remove user from
|
|
* \param user User to remove
|
|
* \return NSERROR_OK.
|
|
*/
|
|
static nserror llcache_object_remove_user(llcache_object *object,
|
|
llcache_object_user *user)
|
|
{
|
|
assert(user != NULL);
|
|
assert(object != NULL);
|
|
assert(object->users != NULL);
|
|
assert(user->handle == NULL || user->handle->object == object);
|
|
assert((user->prev != NULL) || (object->users == user));
|
|
|
|
if (user == object->users)
|
|
object->users = user->next;
|
|
else
|
|
user->prev->next = user->next;
|
|
|
|
if (user->next != NULL)
|
|
user->next->prev = user->prev;
|
|
|
|
user->next = user->prev = NULL;
|
|
|
|
/* record the time the last user was removed from the object */
|
|
if (object->users == NULL) {
|
|
object->last_used = time(NULL);
|
|
}
|
|
|
|
NSLOG(llcache, DEBUG, "Removing user %p from %p", user, object);
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Iterate the users of an object, calling their callbacks.
|
|
*
|
|
* \param object The object to iterate
|
|
* \param event The event to pass to the callback.
|
|
* \return NSERROR_OK on success, appropriate error otherwise.
|
|
*/
|
|
static nserror llcache_send_event_to_users(llcache_object *object,
|
|
llcache_event *event)
|
|
{
|
|
nserror error = NSERROR_OK;
|
|
llcache_object_user *user, *next_user;
|
|
|
|
user = object->users;
|
|
while (user != NULL) {
|
|
bool was_target = user->iterator_target;
|
|
user->iterator_target = true;
|
|
|
|
error = user->handle->cb(user->handle, event,
|
|
user->handle->pw);
|
|
|
|
next_user = user->next;
|
|
|
|
user->iterator_target = was_target;
|
|
|
|
if (user->queued_for_delete && !was_target) {
|
|
llcache_object_remove_user(object, user);
|
|
llcache_object_user_destroy(user);
|
|
}
|
|
|
|
if (error != NSERROR_OK)
|
|
break;
|
|
|
|
user = next_user;
|
|
}
|
|
|
|
return error;
|
|
}
|
|
|
|
/**
|
|
* Create a new low-level cache object
|
|
*
|
|
* \param url URL of object to create
|
|
* \param result Pointer to location to receive result
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*/
|
|
static nserror llcache_object_new(nsurl *url, llcache_object **result)
|
|
{
|
|
llcache_object *obj = calloc(1, sizeof(llcache_object));
|
|
if (obj == NULL)
|
|
return NSERROR_NOMEM;
|
|
|
|
NSLOG(llcache, DEBUG, "Created object %p (%s)", obj, nsurl_access(url));
|
|
|
|
obj->url = nsurl_ref(url);
|
|
|
|
*result = obj;
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Clone a POST data object
|
|
*
|
|
* \param orig Object to clone
|
|
* \param clone Pointer to location to receive clone
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*/
|
|
static nserror llcache_post_data_clone(const llcache_post_data *orig,
|
|
llcache_post_data **clone)
|
|
{
|
|
llcache_post_data *post_clone;
|
|
|
|
post_clone = calloc(1, sizeof(llcache_post_data));
|
|
if (post_clone == NULL)
|
|
return NSERROR_NOMEM;
|
|
|
|
post_clone->type = orig->type;
|
|
|
|
/* Deep-copy the type-specific data */
|
|
if (orig->type == LLCACHE_POST_URL_ENCODED) {
|
|
post_clone->data.urlenc = strdup(orig->data.urlenc);
|
|
if (post_clone->data.urlenc == NULL) {
|
|
free(post_clone);
|
|
|
|
return NSERROR_NOMEM;
|
|
}
|
|
} else {
|
|
post_clone->data.multipart = fetch_multipart_data_clone(
|
|
orig->data.multipart);
|
|
if (post_clone->data.multipart == NULL) {
|
|
free(post_clone);
|
|
|
|
return NSERROR_NOMEM;
|
|
}
|
|
}
|
|
|
|
*clone = post_clone;
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Split a fetch header into name and value
|
|
*
|
|
* HTTP header splitting according to grammar defined in RFC7230 section 3.2
|
|
* https://tools.ietf.org/html/rfc7230#section-3.2
|
|
*
|
|
* This implementation is non conformant in that it:
|
|
* - includes carrige return and newline in whitespace (3.2.3)
|
|
* - allows whitespace before and after the field-name token (3.2.4)
|
|
* - does not handle obsolete line folding (3.2.4)
|
|
*
|
|
* \param data Header string
|
|
* \param len Byte length of header
|
|
* \param name Pointer to location to receive header name
|
|
* \param value Pointer to location to receive header value
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*/
|
|
static nserror llcache_fetch_split_header(const uint8_t *data, size_t len,
|
|
char **name, char **value)
|
|
{
|
|
char *n, *v;
|
|
const uint8_t *colon;
|
|
|
|
/* Strip leading whitespace from name */
|
|
while (data[0] == ' ' || data[0] == '\t' ||
|
|
data[0] == '\r' || data[0] == '\n') {
|
|
data++;
|
|
}
|
|
|
|
/* Find colon */
|
|
colon = (const uint8_t *) strchr((const char *) data, ':');
|
|
if (colon == NULL) {
|
|
/* Failed, assume a key with no value */
|
|
colon = data + strlen((const char *)data);
|
|
|
|
/* Strip trailing whitespace from name */
|
|
while ((colon > data) &&
|
|
(colon[-1] == ' ' || colon[-1] == '\t' ||
|
|
colon[-1] == '\r' || colon[-1] == '\n')) {
|
|
colon--;
|
|
}
|
|
n = strndup((const char *) data, colon - data);
|
|
if (n == NULL)
|
|
return NSERROR_NOMEM;
|
|
|
|
v = strdup("");
|
|
if (v == NULL) {
|
|
free(n);
|
|
return NSERROR_NOMEM;
|
|
}
|
|
} else {
|
|
/* Split header into name & value */
|
|
|
|
/* Strip trailing whitespace from name */
|
|
while (colon > data && (colon[-1] == ' ' ||
|
|
colon[-1] == '\t' || colon[-1] == '\r' ||
|
|
colon[-1] == '\n'))
|
|
colon--;
|
|
|
|
n = strndup((const char *) data, colon - data);
|
|
if (n == NULL)
|
|
return NSERROR_NOMEM;
|
|
|
|
/* Find colon again */
|
|
while (*colon != ':') {
|
|
colon++;
|
|
}
|
|
|
|
/* Skip over colon and any subsequent whitespace */
|
|
do {
|
|
colon++;
|
|
} while (*colon == ' ' || *colon == '\t' ||
|
|
*colon == '\r' || *colon == '\n');
|
|
|
|
/* Strip trailing whitespace from value */
|
|
while (len > 0 && (data[len - 1] == ' ' ||
|
|
data[len - 1] == '\t' ||
|
|
data[len - 1] == '\r' ||
|
|
data[len - 1] == '\n')) {
|
|
len--;
|
|
}
|
|
|
|
v = strndup((const char *) colon, len - (colon - data));
|
|
if (v == NULL) {
|
|
free(n);
|
|
return NSERROR_NOMEM;
|
|
}
|
|
}
|
|
|
|
*name = n;
|
|
*value = v;
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* parse cache control header value
|
|
*
|
|
* \param object Object to parse header for
|
|
* \param value header value
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*/
|
|
static nserror
|
|
llcache_fetch_parse_cache_control(llcache_object *object, char *value)
|
|
{
|
|
http_cache_control *cc;
|
|
nserror error;
|
|
|
|
error = http_parse_cache_control(value, &cc);
|
|
if (error != NSERROR_OK) {
|
|
/* Ignore parse errors */
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
if (http_cache_control_no_cache(cc) ||
|
|
http_cache_control_no_store(cc)) {
|
|
/**
|
|
* \todo When we get a disk cache we should
|
|
* distinguish between these two.
|
|
*/
|
|
object->cache.no_cache = LLCACHE_VALIDATE_ALWAYS;
|
|
}
|
|
|
|
if (http_cache_control_has_max_age(cc)) {
|
|
object->cache.max_age = http_cache_control_max_age(cc);
|
|
}
|
|
|
|
http_cache_control_destroy(cc);
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Update cache control from appropriate header
|
|
*
|
|
* \param object Object to parse header for
|
|
* \param name header name
|
|
* \param value header value
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*/
|
|
static nserror
|
|
llcache_fetch_header_cache_control(llcache_object *object,
|
|
char *name,
|
|
char *value)
|
|
{
|
|
nserror res;
|
|
size_t name_len;
|
|
|
|
/* Parse cache headers to populate cache control data */
|
|
name_len = strlen(name);
|
|
|
|
switch (name_len) {
|
|
case 3:
|
|
if (strcasecmp(name, "Age") == 0) {
|
|
/* extract Age header */
|
|
if ('0' <= *value && *value <= '9') {
|
|
object->cache.age = atoi(value);
|
|
}
|
|
|
|
}
|
|
break;
|
|
|
|
case 4:
|
|
if (strcasecmp(name, "Date") == 0) {
|
|
/* extract Date header */
|
|
res = nsc_strntimet(value,
|
|
strlen(value),
|
|
&object->cache.date);
|
|
if (res != NSERROR_OK) {
|
|
NSLOG(llcache, INFO,
|
|
"Processing Date header value \"%s\" returned %d",
|
|
value, res);
|
|
}
|
|
} else if (strcasecmp(name, "ETag") == 0) {
|
|
/* extract ETag header */
|
|
free(object->cache.etag);
|
|
object->cache.etag = strdup(value);
|
|
if (object->cache.etag == NULL) {
|
|
NSLOG(llcache, INFO,
|
|
"No memory to duplicate ETag");
|
|
return NSERROR_NOMEM;
|
|
}
|
|
}
|
|
break;
|
|
|
|
case 7:
|
|
if (strcasecmp(name, "Expires") == 0) {
|
|
/* process Expires header value */
|
|
res = nsc_strntimet(value,
|
|
strlen(value),
|
|
&object->cache.expires);
|
|
if (res != NSERROR_OK) {
|
|
NSLOG(llcache, INFO,
|
|
"Processing Expires header value \"%s\" returned %d",
|
|
value, res);
|
|
object->cache.expires = (time_t)0x7fffffff;
|
|
}
|
|
}
|
|
break;
|
|
|
|
case 13:
|
|
if (strcasecmp(name, "Cache-Control") == 0) {
|
|
/* parse Cache-Control header value */
|
|
llcache_fetch_parse_cache_control(object,value);
|
|
} else if (strcasecmp(name, "Last-Modified") == 0) {
|
|
/* parse Last-Modified header value */
|
|
nsc_strntimet(value,
|
|
strlen(value),
|
|
&object->cache.last_modified);
|
|
}
|
|
break;
|
|
}
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Destroy headers.
|
|
*
|
|
* \param object The object to destroy headers within.
|
|
*/
|
|
static inline void llcache_destroy_headers(llcache_object *object)
|
|
{
|
|
while (object->num_headers > 0) {
|
|
object->num_headers--;
|
|
|
|
free(object->headers[object->num_headers].name);
|
|
free(object->headers[object->num_headers].value);
|
|
}
|
|
free(object->headers);
|
|
object->headers = NULL;
|
|
}
|
|
|
|
/**
|
|
* Invalidate cache control data.
|
|
*
|
|
* \param object The object to invalidate cache control for.
|
|
*/
|
|
static inline void llcache_invalidate_cache_control_data(llcache_object *object)
|
|
{
|
|
free(object->cache.etag);
|
|
memset(&(object->cache), 0, sizeof(llcache_cache_control));
|
|
|
|
object->cache.age = INVALID_AGE;
|
|
object->cache.max_age = INVALID_AGE;
|
|
}
|
|
|
|
/**
|
|
* Process a fetch header
|
|
*
|
|
* \param object Object being fetched
|
|
* \param data Header string
|
|
* \param len Byte length of header
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*/
|
|
static nserror llcache_fetch_process_header(llcache_object *object,
|
|
const uint8_t *data, size_t len)
|
|
{
|
|
nserror res;
|
|
char *name, *value;
|
|
llcache_header *temp;
|
|
|
|
/**
|
|
* \note The headers for multiple HTTP responses may be
|
|
* delivered to us if the fetch layer receives a 401 response
|
|
* for which it has authentication credentials. This will
|
|
* result in a silent re-request after which we'll receive the
|
|
* actual response headers for the object we want to fetch
|
|
* (assuming that the credentials were correct of course)
|
|
*
|
|
* Therefore, if the header is an HTTP response start marker, then we
|
|
* must discard any headers we've read so far, reset the cache data
|
|
* that we might have computed, and start again.
|
|
*/
|
|
/** \todo Properly parse the response line */
|
|
if (strncmp((const char *) data, "HTTP/", SLEN("HTTP/")) == 0) {
|
|
time_t req_time = object->cache.req_time;
|
|
|
|
llcache_invalidate_cache_control_data(object);
|
|
|
|
/* Restore request time, so we compute object's age correctly */
|
|
object->cache.req_time = req_time;
|
|
|
|
llcache_destroy_headers(object);
|
|
}
|
|
|
|
/* Set fetch response time if not already set */
|
|
if (object->cache.res_time == 0) {
|
|
object->cache.res_time = time(NULL);
|
|
}
|
|
|
|
/* Parse header into name-value pair */
|
|
res = llcache_fetch_split_header(data, len, &name, &value);
|
|
if (res != NSERROR_OK) {
|
|
return res;
|
|
}
|
|
|
|
/* deal with empty header */
|
|
if (name[0] == 0) {
|
|
free(name);
|
|
free(value);
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/* update cache control data from header */
|
|
res = llcache_fetch_header_cache_control(object, name, value);
|
|
if (res != NSERROR_OK) {
|
|
free(name);
|
|
free(value);
|
|
return res;
|
|
}
|
|
|
|
/* Append header data to the object's headers array */
|
|
temp = realloc(object->headers,
|
|
(object->num_headers + 1) * sizeof(llcache_header));
|
|
if (temp == NULL) {
|
|
free(name);
|
|
free(value);
|
|
return NSERROR_NOMEM;
|
|
}
|
|
|
|
object->headers = temp;
|
|
|
|
object->headers[object->num_headers].name = name;
|
|
object->headers[object->num_headers].value = value;
|
|
|
|
object->num_headers++;
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* (Re)fetch an object
|
|
*
|
|
* Sets up headers and attempts to start an actual fetch from the
|
|
* fetchers system updating the llcache object with the new fetch on
|
|
* successful start.
|
|
*
|
|
* \pre The fetch parameters in object->fetch must be populated
|
|
*
|
|
* \param object Object to refetch
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*/
|
|
static nserror llcache_object_refetch(llcache_object *object)
|
|
{
|
|
const char *urlenc = NULL;
|
|
struct fetch_multipart_data *multipart = NULL;
|
|
char **headers = NULL;
|
|
int header_idx = 0;
|
|
nserror res;
|
|
|
|
if (object->fetch.post != NULL) {
|
|
if (object->fetch.post->type == LLCACHE_POST_URL_ENCODED) {
|
|
urlenc = object->fetch.post->data.urlenc;
|
|
} else {
|
|
multipart = object->fetch.post->data.multipart;
|
|
}
|
|
}
|
|
|
|
/* Generate cache-control headers */
|
|
headers = malloc(3 * sizeof(char *));
|
|
if (headers == NULL) {
|
|
return NSERROR_NOMEM;
|
|
}
|
|
|
|
if (object->cache.etag != NULL) {
|
|
const size_t len = SLEN("If-None-Match: ") +
|
|
strlen(object->cache.etag) + 1;
|
|
|
|
headers[header_idx] = malloc(len);
|
|
if (headers[header_idx] == NULL) {
|
|
free(headers);
|
|
return NSERROR_NOMEM;
|
|
}
|
|
|
|
snprintf(headers[header_idx], len, "If-None-Match: %s",
|
|
object->cache.etag);
|
|
|
|
header_idx++;
|
|
}
|
|
|
|
if (object->cache.last_modified != 0) {
|
|
/* Maximum length of an RFC 1123 date is 29 bytes */
|
|
const size_t len = SLEN("If-Modified-Since: ") + 29 + 1;
|
|
|
|
headers[header_idx] = malloc(len);
|
|
if (headers[header_idx] == NULL) {
|
|
while (--header_idx >= 0)
|
|
free(headers[header_idx]);
|
|
free(headers);
|
|
return NSERROR_NOMEM;
|
|
}
|
|
|
|
snprintf(headers[header_idx], len, "If-Modified-Since: %s",
|
|
rfc1123_date(object->cache.last_modified));
|
|
|
|
header_idx++;
|
|
}
|
|
headers[header_idx] = NULL;
|
|
|
|
/* Reset cache control data */
|
|
llcache_invalidate_cache_control_data(object);
|
|
object->cache.req_time = time(NULL);
|
|
object->cache.fin_time = object->cache.req_time;
|
|
|
|
/* Reset fetch state */
|
|
object->fetch.state = LLCACHE_FETCH_INIT;
|
|
|
|
NSLOG(llcache, DEBUG, "Re-fetching %p", object);
|
|
|
|
/* Kick off fetch */
|
|
res = fetch_start(object->url,
|
|
object->fetch.referer,
|
|
llcache_fetch_callback,
|
|
object,
|
|
object->fetch.flags & LLCACHE_RETRIEVE_NO_ERROR_PAGES,
|
|
urlenc,
|
|
multipart,
|
|
object->fetch.flags & LLCACHE_RETRIEVE_VERIFIABLE,
|
|
object->fetch.tried_with_tls_downgrade,
|
|
(const char **)headers,
|
|
&object->fetch.fetch);
|
|
|
|
/* Clean up cache-control headers */
|
|
while (--header_idx >= 0) {
|
|
free(headers[header_idx]);
|
|
}
|
|
free(headers);
|
|
|
|
return res;
|
|
}
|
|
|
|
/**
|
|
* Kick-off a fetch for an object
|
|
*
|
|
* \pre object::url must contain the URL to fetch
|
|
* \pre If there is a freshness validation candidate,
|
|
* object::candidate and object::cache must be filled in
|
|
* \pre There must not be a fetch in progress for \a object
|
|
*
|
|
* \param object Object to fetch
|
|
* \param flags Fetch flags
|
|
* \param referer Referring URL, or NULL for none
|
|
* \param post POST data, or NULL for GET
|
|
* \param redirect_count Number of redirects followed so far
|
|
* \param hsts_in_use Whether HSTS applies to this fetch
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*/
|
|
static nserror llcache_object_fetch(llcache_object *object, uint32_t flags,
|
|
nsurl *referer, const llcache_post_data *post,
|
|
uint32_t redirect_count, bool hsts_in_use)
|
|
{
|
|
nserror error;
|
|
nsurl *referer_clone = NULL;
|
|
llcache_post_data *post_clone = NULL;
|
|
|
|
NSLOG(llcache, DEBUG, "Starting fetch for %p", object);
|
|
|
|
if (post != NULL) {
|
|
error = llcache_post_data_clone(post, &post_clone);
|
|
if (error != NSERROR_OK)
|
|
return error;
|
|
}
|
|
|
|
if (referer != NULL)
|
|
referer_clone = nsurl_ref(referer);
|
|
|
|
object->fetch.flags = flags;
|
|
object->fetch.referer = referer_clone;
|
|
object->fetch.post = post_clone;
|
|
object->fetch.redirect_count = redirect_count;
|
|
object->fetch.retries_remaining = llcache->fetch_attempts;
|
|
object->fetch.hsts_in_use = hsts_in_use;
|
|
|
|
return llcache_object_refetch(object);
|
|
}
|
|
|
|
/**
|
|
* Destroy a low-level cache object
|
|
*
|
|
* \pre Object is detached from cache list
|
|
* \pre Object has no users
|
|
* \pre Object is not a candidate (i.e. object::candidate_count == 0)
|
|
*
|
|
* \param object Object to destroy
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*/
|
|
static nserror llcache_object_destroy(llcache_object *object)
|
|
{
|
|
size_t i;
|
|
|
|
NSLOG(llcache, DEBUG, "Destroying object %p, %s", object,
|
|
nsurl_access(object->url));
|
|
|
|
if (object->source_data != NULL) {
|
|
if (object->store_state == LLCACHE_STATE_DISC) {
|
|
guit->llcache->release(object->url, BACKING_STORE_NONE);
|
|
} else {
|
|
free(object->source_data);
|
|
}
|
|
}
|
|
|
|
nsurl_unref(object->url);
|
|
|
|
if (object->fetch.fetch != NULL) {
|
|
fetch_abort(object->fetch.fetch);
|
|
object->fetch.fetch = NULL;
|
|
}
|
|
|
|
if (object->fetch.referer != NULL)
|
|
nsurl_unref(object->fetch.referer);
|
|
|
|
if (object->fetch.post != NULL) {
|
|
if (object->fetch.post->type == LLCACHE_POST_URL_ENCODED) {
|
|
free(object->fetch.post->data.urlenc);
|
|
} else {
|
|
fetch_multipart_data_destroy(
|
|
object->fetch.post->data.multipart);
|
|
}
|
|
|
|
free(object->fetch.post);
|
|
}
|
|
|
|
free(object->cache.etag);
|
|
|
|
for (i = 0; i < object->num_headers; i++) {
|
|
free(object->headers[i].name);
|
|
free(object->headers[i].value);
|
|
}
|
|
free(object->headers);
|
|
|
|
free(object);
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Add a low-level cache object to a cache list
|
|
*
|
|
* \param object Object to add
|
|
* \param list List to add to
|
|
* \return NSERROR_OK
|
|
*/
|
|
static nserror llcache_object_add_to_list(llcache_object *object,
|
|
llcache_object **list)
|
|
{
|
|
object->prev = NULL;
|
|
object->next = *list;
|
|
|
|
if (*list != NULL)
|
|
(*list)->prev = object;
|
|
*list = object;
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Determine the remaining lifetime of a cache object using the
|
|
*
|
|
* \param cd cache control data.
|
|
* \return The length of time remaining for the object or 0 if expired.
|
|
*/
|
|
static int
|
|
llcache_object_rfc2616_remaining_lifetime(const llcache_cache_control *cd)
|
|
{
|
|
int current_age, freshness_lifetime;
|
|
time_t now = time(NULL);
|
|
|
|
/* Calculate staleness of cached object as per RFC 2616 13.2.3/13.2.4 */
|
|
current_age = max(0, (cd->res_time - cd->date));
|
|
current_age = max(current_age, (cd->age == INVALID_AGE) ? 0 : cd->age);
|
|
current_age += cd->res_time - cd->req_time + now - cd->res_time;
|
|
|
|
/* Determine freshness lifetime of this object */
|
|
if (cd->max_age != INVALID_AGE) {
|
|
freshness_lifetime = cd->max_age;
|
|
} else if (cd->expires != 0) {
|
|
freshness_lifetime = cd->expires - cd->date;
|
|
} else if (cd->last_modified != 0) {
|
|
freshness_lifetime = (now - cd->last_modified) / 10;
|
|
} else {
|
|
freshness_lifetime = 0;
|
|
}
|
|
|
|
NSLOG(llcache, DEBUG, "%d:%d", freshness_lifetime, current_age);
|
|
|
|
if ((cd->no_cache == LLCACHE_VALIDATE_FRESH) &&
|
|
(freshness_lifetime > current_age)) {
|
|
/* object was not forbidden from being returned from
|
|
* the cache unvalidated (i.e. the response contained
|
|
* a no-cache directive)
|
|
*
|
|
* The object current age is within the freshness lifetime.
|
|
*/
|
|
return freshness_lifetime - current_age;
|
|
}
|
|
|
|
return 0; /* object has no remaining lifetime */
|
|
}
|
|
|
|
/**
|
|
* Determine if an object is still fresh
|
|
*
|
|
* \param object Object to consider
|
|
* \return True if object is still fresh, false otherwise
|
|
*/
|
|
static bool llcache_object_is_fresh(const llcache_object *object)
|
|
{
|
|
int remaining_lifetime;
|
|
const llcache_cache_control *cd = &object->cache;
|
|
|
|
remaining_lifetime = llcache_object_rfc2616_remaining_lifetime(cd);
|
|
|
|
NSLOG(llcache, DEBUG, "%p: (%d > 0 || %d != %d)", object,
|
|
remaining_lifetime,
|
|
object->fetch.state, LLCACHE_FETCH_COMPLETE);
|
|
|
|
/* The object is fresh if:
|
|
* - it was not forbidden from being returned from the cache
|
|
* unvalidated.
|
|
*
|
|
* - it has remaining lifetime or still being fetched.
|
|
*/
|
|
return ((cd->no_cache == LLCACHE_VALIDATE_FRESH) &&
|
|
((remaining_lifetime > 0) ||
|
|
(object->fetch.state != LLCACHE_FETCH_COMPLETE)));
|
|
}
|
|
|
|
/**
|
|
* Clone an object's cache data
|
|
*
|
|
* \post If \a deep is false, then any pointers in \a source will be set to NULL
|
|
*
|
|
* \param source Source object containing cache data to clone
|
|
* \param destination Destination object to clone cache data into
|
|
* \param deep Whether to deep-copy the data or not
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*/
|
|
static nserror llcache_object_clone_cache_data(llcache_object *source,
|
|
llcache_object *destination, bool deep)
|
|
{
|
|
/* ETag must be first, as it can fail when deep cloning */
|
|
if (source->cache.etag != NULL) {
|
|
char *etag = source->cache.etag;
|
|
|
|
if (deep) {
|
|
/* Copy the etag */
|
|
etag = strdup(source->cache.etag);
|
|
if (etag == NULL)
|
|
return NSERROR_NOMEM;
|
|
} else {
|
|
/* Destination takes ownership */
|
|
source->cache.etag = NULL;
|
|
}
|
|
|
|
if (destination->cache.etag != NULL)
|
|
free(destination->cache.etag);
|
|
|
|
destination->cache.etag = etag;
|
|
}
|
|
|
|
destination->cache.req_time = source->cache.req_time;
|
|
destination->cache.res_time = source->cache.res_time;
|
|
destination->cache.fin_time = source->cache.fin_time;
|
|
|
|
if (source->cache.date != 0)
|
|
destination->cache.date = source->cache.date;
|
|
|
|
if (source->cache.expires != 0)
|
|
destination->cache.expires = source->cache.expires;
|
|
|
|
if (source->cache.age != INVALID_AGE)
|
|
destination->cache.age = source->cache.age;
|
|
|
|
if (source->cache.max_age != INVALID_AGE)
|
|
destination->cache.max_age = source->cache.max_age;
|
|
|
|
if (source->cache.no_cache != LLCACHE_VALIDATE_FRESH)
|
|
destination->cache.no_cache = source->cache.no_cache;
|
|
|
|
if (source->cache.last_modified != 0)
|
|
destination->cache.last_modified = source->cache.last_modified;
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Remove a low-level cache object from a cache list
|
|
*
|
|
* \param object Object to remove
|
|
* \param list List to remove from
|
|
* \return NSERROR_OK
|
|
*/
|
|
static nserror
|
|
llcache_object_remove_from_list(llcache_object *object, llcache_object **list)
|
|
{
|
|
if (object == *list)
|
|
*list = object->next;
|
|
else
|
|
object->prev->next = object->next;
|
|
|
|
if (object->next != NULL)
|
|
object->next->prev = object->prev;
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Retrieve source data for an object from persistent store if necessary.
|
|
*
|
|
* If an object's source data has been placed in the persistent store
|
|
* and there is no in-memory copy, then attempt to retrieve the source
|
|
* data.
|
|
*
|
|
* \param object the object to operate on.
|
|
* \return appropriate error code.
|
|
*/
|
|
static nserror llcache_retrieve_persisted_data(llcache_object *object)
|
|
{
|
|
/* ensure the source data is present if necessary */
|
|
if ((object->source_data != NULL) ||
|
|
(object->store_state != LLCACHE_STATE_DISC)) {
|
|
/* source data does not require retrieving from
|
|
* persistent store.
|
|
*/
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/* Source data for the object may be in the persistent store */
|
|
return guit->llcache->fetch(object->url,
|
|
BACKING_STORE_NONE,
|
|
&object->source_data,
|
|
&object->source_len);
|
|
}
|
|
|
|
/**
|
|
* Generate a serialised version of an object's metadata
|
|
*
|
|
* The metadata includes object headers.
|
|
*
|
|
* \param object The cache object to serialise the metadata of.
|
|
* \param data_out Where the serialised buffer will be placed.
|
|
* \param datasize_out The size of the serialised data.
|
|
* \return NSERROR_OK on success with \a data_out and \a datasize_out
|
|
* updated, NSERROR_NOMEM on memory exhaustion or
|
|
* NSERROR_INVALID if there was an error serialising the
|
|
* stream.
|
|
*/
|
|
static nserror
|
|
llcache_serialise_metadata(llcache_object *object,
|
|
uint8_t **data_out,
|
|
size_t *datasize_out)
|
|
{
|
|
size_t allocsize;
|
|
int datasize;
|
|
uint8_t *data;
|
|
char *op;
|
|
unsigned int hloop;
|
|
int use;
|
|
|
|
allocsize = 10 + 1; /* object length */
|
|
|
|
allocsize += 10 + 1; /* request time */
|
|
|
|
allocsize += 10 + 1; /* response time */
|
|
|
|
allocsize += 10 + 1; /* completion time */
|
|
|
|
allocsize += 10 + 1; /* space for number of header entries */
|
|
|
|
allocsize += nsurl_length(object->url) + 1;
|
|
|
|
for (hloop = 0 ; hloop < object->num_headers ; hloop++) {
|
|
allocsize += strlen(object->headers[hloop].name) + 1;
|
|
allocsize += strlen(object->headers[hloop].value) + 1;
|
|
}
|
|
|
|
data = malloc(allocsize);
|
|
if (data == NULL) {
|
|
return NSERROR_NOMEM;
|
|
}
|
|
|
|
op = (char *)data;
|
|
datasize = allocsize;
|
|
|
|
/* the url, used for checking for collisions */
|
|
use = snprintf(op, datasize, "%s", nsurl_access(object->url));
|
|
if (use < 0) {
|
|
goto operror;
|
|
}
|
|
use++; /* does not count the null */
|
|
if (use > datasize) {
|
|
goto overflow;
|
|
}
|
|
op += use;
|
|
datasize -= use;
|
|
|
|
/* object size */
|
|
use = snprintf(op, datasize, "%" PRIsizet, object->source_len);
|
|
if (use < 0) {
|
|
goto operror;
|
|
}
|
|
use++; /* does not count the null */
|
|
if (use > datasize)
|
|
goto overflow;
|
|
op += use;
|
|
datasize -= use;
|
|
|
|
/* Time of request */
|
|
use = nsc_sntimet(op, datasize, &object->cache.req_time);
|
|
if (use == 0)
|
|
goto overflow;
|
|
use++; /* does not count the null */
|
|
op += use;
|
|
datasize -= use;
|
|
|
|
/* Time of response */
|
|
use = nsc_sntimet(op, datasize, &object->cache.res_time);
|
|
if (use == 0)
|
|
goto overflow;
|
|
use++; /* does not count the null */
|
|
op += use;
|
|
datasize -= use;
|
|
|
|
/* Time of completion */
|
|
use = nsc_sntimet(op, datasize, &object->cache.fin_time);
|
|
if (use == 0)
|
|
goto overflow;
|
|
use++; /* does not count the null */
|
|
op += use;
|
|
datasize -= use;
|
|
|
|
/* number of headers */
|
|
use = snprintf(op, datasize, "%" PRIsizet, object->num_headers);
|
|
if (use < 0) {
|
|
goto operror;
|
|
}
|
|
use++; /* does not count the null */
|
|
if (use > datasize)
|
|
goto overflow;
|
|
op += use;
|
|
datasize -= use;
|
|
|
|
/* headers */
|
|
for (hloop = 0 ; hloop < object->num_headers ; hloop++) {
|
|
use = snprintf(op, datasize,
|
|
"%s:%s",
|
|
object->headers[hloop].name,
|
|
object->headers[hloop].value);
|
|
if (use < 0) {
|
|
goto operror;
|
|
}
|
|
use++; /* does not count the null */
|
|
if (use > datasize)
|
|
goto overflow;
|
|
op += use;
|
|
datasize -= use;
|
|
}
|
|
|
|
NSLOG(llcache, DEBUG, "Filled buffer with %d spare", datasize);
|
|
|
|
*data_out = data;
|
|
*datasize_out = allocsize - datasize;
|
|
|
|
return NSERROR_OK;
|
|
|
|
overflow:
|
|
/* somehow we overflowed the buffer - hth? */
|
|
NSLOG(llcache, INFO, "Overflowed metadata buffer");
|
|
free(data);
|
|
return NSERROR_INVALID;
|
|
|
|
operror:
|
|
/* output error */
|
|
NSLOG(llcache, INFO, "Output error");
|
|
free(data);
|
|
return NSERROR_INVALID;
|
|
}
|
|
|
|
/**
|
|
* Deserialisation of an object's metadata.
|
|
*
|
|
* Attempt to retrieve and deserialise the metadata for an object from
|
|
* the backing store.
|
|
*
|
|
* This must only update object if it is successful otherwise difficult
|
|
* to debug crashes happen later by using bad leftover object state.
|
|
*
|
|
* \param object The object to retrieve the metadata for.
|
|
* \return NSERROR_OK if the metatdata was retrieved and deserialised
|
|
* or error code if URL is not in persistent storage or in
|
|
* event of deserialisation error.
|
|
*/
|
|
static nserror
|
|
llcache_process_metadata(llcache_object *object)
|
|
{
|
|
nserror res;
|
|
uint8_t *metadata = NULL;
|
|
size_t metadatalen = 0;
|
|
nsurl *metadataurl;
|
|
unsigned int line;
|
|
char *ln;
|
|
int lnsize;
|
|
|
|
size_t source_length;
|
|
time_t request_time;
|
|
time_t response_time;
|
|
time_t completion_time;
|
|
size_t num_headers;
|
|
size_t hloop;
|
|
|
|
NSLOG(llcache, INFO, "Retrieving metadata");
|
|
|
|
/* attempt to retrieve object metadata from the backing store */
|
|
res = guit->llcache->fetch(object->url,
|
|
BACKING_STORE_META,
|
|
&metadata,
|
|
&metadatalen);
|
|
if (res != NSERROR_OK) {
|
|
return res;
|
|
}
|
|
|
|
NSLOG(llcache, INFO, "Processing retrieved data");
|
|
|
|
/* metadata line 1 is the url the metadata referrs to */
|
|
line = 1;
|
|
ln = (char *)metadata;
|
|
lnsize = strlen(ln);
|
|
|
|
if (lnsize < 7) {
|
|
res = NSERROR_INVALID;
|
|
goto format_error;
|
|
}
|
|
|
|
res = nsurl_create(ln, &metadataurl);
|
|
if (res != NSERROR_OK)
|
|
goto format_error;
|
|
|
|
if (nsurl_compare(object->url, metadataurl, NSURL_COMPLETE) != true) {
|
|
/* backing store returned the wrong object for the
|
|
* request. This may occur if the backing store had
|
|
* a collision in its storage method. We cope with this
|
|
* by simply skipping caching of this object.
|
|
*/
|
|
|
|
NSLOG(llcache, INFO, "Got metadata for %s instead of %s",
|
|
nsurl_access(metadataurl), nsurl_access(object->url));
|
|
|
|
nsurl_unref(metadataurl);
|
|
|
|
guit->llcache->release(object->url, BACKING_STORE_META);
|
|
|
|
return NSERROR_BAD_URL;
|
|
}
|
|
nsurl_unref(metadataurl);
|
|
|
|
|
|
/* metadata line 2 is the object's length */
|
|
line = 2;
|
|
ln += lnsize + 1;
|
|
lnsize = strlen(ln);
|
|
|
|
if ((lnsize < 1) || (sscanf(ln, "%" PRIsizet, &source_length) != 1)) {
|
|
res = NSERROR_INVALID;
|
|
goto format_error;
|
|
}
|
|
|
|
|
|
/* metadata line 3 is the time of request */
|
|
line = 3;
|
|
ln += lnsize + 1;
|
|
lnsize = strlen(ln);
|
|
|
|
res = nsc_snptimet(ln, lnsize, &request_time);
|
|
if (res != NSERROR_OK)
|
|
goto format_error;
|
|
|
|
|
|
/* metadata line 4 is the time of response */
|
|
line = 4;
|
|
ln += lnsize + 1;
|
|
lnsize = strlen(ln);
|
|
|
|
res = nsc_snptimet(ln, lnsize, &response_time);
|
|
if (res != NSERROR_OK)
|
|
goto format_error;
|
|
|
|
|
|
/* metadata line 5 is the time of request completion */
|
|
line = 5;
|
|
ln += lnsize + 1;
|
|
lnsize = strlen(ln);
|
|
|
|
res = nsc_snptimet(ln, lnsize, &completion_time);
|
|
if (res != NSERROR_OK)
|
|
goto format_error;
|
|
|
|
|
|
/* metadata line 6 is the number of headers */
|
|
line = 6;
|
|
ln += lnsize + 1;
|
|
lnsize = strlen(ln);
|
|
|
|
if ((lnsize < 1) || (sscanf(ln, "%" PRIsizet, &num_headers) != 1)) {
|
|
res = NSERROR_INVALID;
|
|
goto format_error;
|
|
}
|
|
|
|
/* read headers */
|
|
for (hloop = 0 ; hloop < num_headers; hloop++) {
|
|
line++;
|
|
ln += lnsize + 1;
|
|
lnsize = strlen(ln);
|
|
|
|
res = llcache_fetch_process_header(object,
|
|
(uint8_t *)ln,
|
|
lnsize);
|
|
if (res != NSERROR_OK)
|
|
goto format_error;
|
|
}
|
|
|
|
guit->llcache->release(object->url, BACKING_STORE_META);
|
|
|
|
/* update object on successful parse of metadata */
|
|
object->source_len = source_length;
|
|
|
|
/** \todo really not sure this is right, nothing is allocated here? */
|
|
object->source_alloc = metadatalen;
|
|
|
|
object->cache.req_time = request_time;
|
|
object->cache.res_time = response_time;
|
|
object->cache.fin_time = completion_time;
|
|
|
|
/* object stored in backing store */
|
|
object->store_state = LLCACHE_STATE_DISC;
|
|
|
|
return NSERROR_OK;
|
|
|
|
format_error:
|
|
NSLOG(llcache, INFO,
|
|
"metadata error on line %d error code %d\n",
|
|
line, res);
|
|
guit->llcache->release(object->url, BACKING_STORE_META);
|
|
|
|
return res;
|
|
}
|
|
|
|
/**
|
|
* Attempt to retrieve an object from persistent storage.
|
|
*
|
|
* \param object The object to populate from persistent store.
|
|
* \param flags Fetch flags.
|
|
* \param referer The referring url.
|
|
* \param post Post data for fetch.
|
|
* \param redirect_count how many times this fetch has been redirected.
|
|
* \return NSERROR_OK if the object was successfully retrieved from the
|
|
* cache else appropriate error code.
|
|
*/
|
|
static nserror
|
|
llcache_object_fetch_persistent(llcache_object *object,
|
|
uint32_t flags,
|
|
nsurl *referer,
|
|
const llcache_post_data *post,
|
|
uint32_t redirect_count)
|
|
{
|
|
nserror error;
|
|
nsurl *referer_clone = NULL;
|
|
llcache_post_data *post_clone = NULL;
|
|
|
|
object->cache.req_time = time(NULL);
|
|
object->cache.fin_time = object->cache.req_time;
|
|
|
|
/* retrieve and process metadata */
|
|
error = llcache_process_metadata(object);
|
|
if (error != NSERROR_OK) {
|
|
return error;
|
|
}
|
|
|
|
/* entry came out of cache - need to setup object state */
|
|
if (post != NULL) {
|
|
error = llcache_post_data_clone(post, &post_clone);
|
|
if (error != NSERROR_OK)
|
|
return error;
|
|
}
|
|
|
|
if (referer != NULL) {
|
|
referer_clone = nsurl_ref(referer);
|
|
}
|
|
|
|
object->fetch.flags = flags;
|
|
object->fetch.referer = referer_clone;
|
|
object->fetch.post = post_clone;
|
|
object->fetch.redirect_count = redirect_count;
|
|
|
|
/* fetch is "finished" */
|
|
object->fetch.state = LLCACHE_FETCH_COMPLETE;
|
|
object->fetch.fetch = NULL;
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Retrieve a potentially cached object
|
|
*
|
|
* \param url URL of object to retrieve
|
|
* \param flags Fetch flags
|
|
* \param referer Referring URL, or NULL if none
|
|
* \param post POST data, or NULL for a GET request
|
|
* \param redirect_count Number of redirects followed so far
|
|
* \param hsts_in_use Whether HSTS applies to this fetch
|
|
* \param result Pointer to location to receive retrieved object
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*/
|
|
static nserror
|
|
llcache_object_retrieve_from_cache(nsurl *url,
|
|
uint32_t flags,
|
|
nsurl *referer,
|
|
const llcache_post_data *post,
|
|
uint32_t redirect_count,
|
|
bool hsts_in_use,
|
|
llcache_object **result)
|
|
{
|
|
nserror error;
|
|
llcache_object *obj, *newest = NULL;
|
|
|
|
NSLOG(llcache, DEBUG,
|
|
"Searching cache for %s flags:%x referer:%s post:%p",
|
|
nsurl_access(url), flags,
|
|
referer==NULL?"":nsurl_access(referer),
|
|
post);
|
|
|
|
/* Search for the most recently fetched matching object */
|
|
for (obj = llcache->cached_objects; obj != NULL; obj = obj->next) {
|
|
|
|
if ((newest == NULL ||
|
|
obj->cache.req_time > newest->cache.req_time) &&
|
|
nsurl_compare(obj->url, url,
|
|
NSURL_COMPLETE) == true) {
|
|
newest = obj;
|
|
}
|
|
}
|
|
|
|
/* No viable object found in cache create one and attempt to
|
|
* pull from persistent store.
|
|
*/
|
|
if (newest == NULL) {
|
|
NSLOG(llcache, DEBUG, "No viable object found in llcache");
|
|
|
|
error = llcache_object_new(url, &obj);
|
|
if (error != NSERROR_OK)
|
|
return error;
|
|
|
|
/* attempt to retrieve object from persistent store */
|
|
error = llcache_object_fetch_persistent(obj, flags, referer, post, redirect_count);
|
|
if (error == NSERROR_OK) {
|
|
NSLOG(llcache, DEBUG, "retrieved object from persistent store");
|
|
|
|
/* set newest object from persistent store which
|
|
* will cause the normal object handling to be used.
|
|
*/
|
|
newest = obj;
|
|
|
|
/* Add new object to cached object list */
|
|
llcache_object_add_to_list(obj, &llcache->cached_objects);
|
|
|
|
}
|
|
/* else no object found and irretrievable from cache,
|
|
* fall through with newest unset to start fetch
|
|
*/
|
|
}
|
|
|
|
if ((newest != NULL) && (llcache_object_is_fresh(newest))) {
|
|
/* Found a suitable object, and it's still fresh */
|
|
NSLOG(llcache, DEBUG, "Found fresh %p", newest);
|
|
|
|
/* The client needs to catch up with the object's state.
|
|
* This will occur the next time that llcache_poll is called.
|
|
*/
|
|
|
|
/* ensure the source data is present */
|
|
error = llcache_retrieve_persisted_data(newest);
|
|
if (error == NSERROR_OK) {
|
|
/* source data was successfully retrieved from
|
|
* persistent store
|
|
*/
|
|
*result = newest;
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/* retrieval of source data from persistent store
|
|
* failed, destroy cache object and fall though to
|
|
* cache miss to re-fetch
|
|
*/
|
|
NSLOG(llcache, DEBUG, "Persistent retrieval failed for %p", newest);
|
|
|
|
llcache_object_remove_from_list(newest, &llcache->cached_objects);
|
|
llcache_object_destroy(newest);
|
|
|
|
error = llcache_object_new(url, &obj);
|
|
if (error != NSERROR_OK) {
|
|
return error;
|
|
}
|
|
} else if (newest != NULL) {
|
|
/* Found a candidate object but it needs freshness validation */
|
|
|
|
/* ensure the source data is present */
|
|
error = llcache_retrieve_persisted_data(newest);
|
|
if (error == NSERROR_OK) {
|
|
|
|
/* Create a new object */
|
|
error = llcache_object_new(url, &obj);
|
|
if (error != NSERROR_OK)
|
|
return error;
|
|
|
|
NSLOG(llcache, DEBUG, "Found candidate %p (%p)", obj, newest);
|
|
|
|
/* Clone candidate's cache data */
|
|
error = llcache_object_clone_cache_data(newest, obj, true);
|
|
if (error != NSERROR_OK) {
|
|
llcache_object_destroy(obj);
|
|
return error;
|
|
}
|
|
|
|
/* Record candidate, so we can fall back if it is still fresh */
|
|
newest->candidate_count++;
|
|
obj->candidate = newest;
|
|
|
|
/* Attempt to kick-off fetch */
|
|
error = llcache_object_fetch(obj, flags, referer, post,
|
|
redirect_count, hsts_in_use);
|
|
if (error != NSERROR_OK) {
|
|
newest->candidate_count--;
|
|
llcache_object_destroy(obj);
|
|
return error;
|
|
}
|
|
|
|
/* Add new object to cache */
|
|
llcache_object_add_to_list(obj, &llcache->cached_objects);
|
|
|
|
*result = obj;
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
NSLOG(llcache, DEBUG, "Persistent retrieval failed for %p", newest);
|
|
|
|
/* retrieval of source data from persistent store
|
|
* failed, destroy cache object and fall though to
|
|
* cache miss to re-retch
|
|
*/
|
|
llcache_object_remove_from_list(newest,
|
|
&llcache->cached_objects);
|
|
llcache_object_destroy(newest);
|
|
|
|
error = llcache_object_new(url, &obj);
|
|
if (error != NSERROR_OK) {
|
|
return error;
|
|
}
|
|
}
|
|
|
|
/* Attempt to kick-off fetch */
|
|
error = llcache_object_fetch(obj, flags, referer, post,
|
|
redirect_count, hsts_in_use);
|
|
if (error != NSERROR_OK) {
|
|
llcache_object_destroy(obj);
|
|
return error;
|
|
}
|
|
|
|
/* Add new object to cache */
|
|
llcache_object_add_to_list(obj, &llcache->cached_objects);
|
|
|
|
*result = obj;
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Retrieve an object from the cache, fetching it if necessary.
|
|
*
|
|
* \param url URL of object to retrieve
|
|
* \param flags Fetch flags
|
|
* \param referer Referring URL, or NULL if none
|
|
* \param post POST data, or NULL for a GET request
|
|
* \param redirect_count Number of redirects followed so far
|
|
* \param hsts_in_use Whether HSTS applies to this fetch
|
|
* \param result Pointer to location to receive retrieved object
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*/
|
|
static nserror
|
|
llcache_object_retrieve(nsurl *url,
|
|
uint32_t flags,
|
|
nsurl *referer,
|
|
const llcache_post_data *post,
|
|
uint32_t redirect_count,
|
|
bool hsts_in_use,
|
|
llcache_object **result)
|
|
{
|
|
nserror error;
|
|
llcache_object *obj;
|
|
nsurl *defragmented_url;
|
|
bool uncachable = false;
|
|
|
|
NSLOG(llcache, DEBUG, "Retrieve %s (%x, %s, %p)", nsurl_access(url), flags,
|
|
referer==NULL?"":nsurl_access(referer), post);
|
|
|
|
|
|
/* Get rid of any url fragment */
|
|
error = nsurl_defragment(url, &defragmented_url);
|
|
if (error != NSERROR_OK)
|
|
return error;
|
|
|
|
/* determine if content is cachable */
|
|
if ((flags & LLCACHE_RETRIEVE_FORCE_FETCH) != 0) {
|
|
/* Forced fetches are never cached */
|
|
uncachable = true;
|
|
} else if (post != NULL) {
|
|
/* POST requests are never cached */
|
|
uncachable = true;
|
|
} else {
|
|
/* only http(s), resource, and file schemes are cached */
|
|
lwc_string *scheme;
|
|
bool match;
|
|
|
|
scheme = nsurl_get_component(defragmented_url, NSURL_SCHEME);
|
|
|
|
/* nsurl ensures lower case schemes, and corestrings are lower
|
|
* case, so it's safe to use case-sensitive comparison. */
|
|
if (lwc_string_isequal(scheme, corestring_lwc_http,
|
|
&match) == lwc_error_ok &&
|
|
(match == false) &&
|
|
lwc_string_isequal(scheme, corestring_lwc_https,
|
|
&match) == lwc_error_ok &&
|
|
(match == false) &&
|
|
lwc_string_isequal(scheme, corestring_lwc_resource,
|
|
&match) == lwc_error_ok &&
|
|
(match == false) &&
|
|
lwc_string_isequal(scheme, corestring_lwc_file,
|
|
&match) == lwc_error_ok &&
|
|
(match == false)) {
|
|
uncachable = true;
|
|
}
|
|
lwc_string_unref(scheme);
|
|
}
|
|
|
|
|
|
if (uncachable) {
|
|
/* Create new object */
|
|
error = llcache_object_new(defragmented_url, &obj);
|
|
if (error != NSERROR_OK) {
|
|
nsurl_unref(defragmented_url);
|
|
return error;
|
|
}
|
|
|
|
/* Attempt to kick-off fetch */
|
|
error = llcache_object_fetch(obj, flags, referer, post,
|
|
redirect_count, hsts_in_use);
|
|
if (error != NSERROR_OK) {
|
|
llcache_object_destroy(obj);
|
|
nsurl_unref(defragmented_url);
|
|
return error;
|
|
}
|
|
|
|
/* Add new object to uncached list */
|
|
llcache_object_add_to_list(obj, &llcache->uncached_objects);
|
|
} else {
|
|
error = llcache_object_retrieve_from_cache(defragmented_url,
|
|
flags, referer, post, redirect_count,
|
|
hsts_in_use, &obj);
|
|
if (error != NSERROR_OK) {
|
|
nsurl_unref(defragmented_url);
|
|
return error;
|
|
}
|
|
|
|
/* Returned object is already in the cached list */
|
|
}
|
|
|
|
NSLOG(llcache, DEBUG, "Retrieved %p", obj);
|
|
|
|
*result = obj;
|
|
|
|
nsurl_unref(defragmented_url);
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
|
|
/**
|
|
* Add a user to a low-level cache object
|
|
*
|
|
* \param object Object to add user to
|
|
* \param user User to add
|
|
* \return NSERROR_OK.
|
|
*/
|
|
static nserror llcache_object_add_user(llcache_object *object,
|
|
llcache_object_user *user)
|
|
{
|
|
assert(user->next == NULL);
|
|
assert(user->prev == NULL);
|
|
assert(user->handle != NULL);
|
|
|
|
user->handle->object = object;
|
|
|
|
user->prev = NULL;
|
|
user->next = object->users;
|
|
|
|
if (object->users != NULL)
|
|
object->users->prev = user;
|
|
object->users = user;
|
|
|
|
NSLOG(llcache, DEBUG, "Adding user %p to %p", user, object);
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Transform a request-URI based on HSTS policy
|
|
*
|
|
* \param url URL to transform
|
|
* \param result Pointer to location to receive transformed URL
|
|
* \param hsts_in_use Pointer to location to receive HSTS in-use flag
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*/
|
|
static nserror llcache_hsts_transform_url(nsurl *url, nsurl **result,
|
|
bool *hsts_in_use)
|
|
{
|
|
lwc_string *scheme = NULL;
|
|
bool match;
|
|
nserror error = NSERROR_OK;
|
|
|
|
scheme = nsurl_get_component(url, NSURL_SCHEME);
|
|
if (lwc_string_caseless_isequal(scheme, corestring_lwc_http,
|
|
&match) != lwc_error_ok || match == false) {
|
|
/* Non-HTTP fetch: ignore */
|
|
lwc_string_unref(scheme);
|
|
*result = nsurl_ref(url);
|
|
*hsts_in_use = false;
|
|
return error;
|
|
}
|
|
lwc_string_unref(scheme);
|
|
|
|
if (urldb_get_hsts_enabled(url)) {
|
|
/* Only need to force HTTPS. If original port was explicitly
|
|
* specified as 80, nsurl_create/join will remove it (as
|
|
* it's redundant) */
|
|
error = nsurl_replace_scheme(url, corestring_lwc_https,
|
|
result);
|
|
*hsts_in_use = (error == NSERROR_OK);
|
|
} else {
|
|
*result = nsurl_ref(url);
|
|
*hsts_in_use = false;
|
|
}
|
|
|
|
return error;
|
|
}
|
|
|
|
/**
|
|
* Update HSTS policy for target domain.
|
|
*
|
|
* \param object Newly-fetched cache object
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*/
|
|
static nserror llcache_hsts_update_policy(llcache_object *object)
|
|
{
|
|
size_t i;
|
|
lwc_string *scheme = NULL;
|
|
bool match = false;
|
|
|
|
scheme = nsurl_get_component(object->url, NSURL_SCHEME);
|
|
if (lwc_string_caseless_isequal(scheme, corestring_lwc_https,
|
|
&match) != lwc_error_ok || match == false) {
|
|
/* Non-HTTPS fetch: ignore */
|
|
lwc_string_unref(scheme);
|
|
return NSERROR_OK;
|
|
}
|
|
lwc_string_unref(scheme);
|
|
|
|
if (object->fetch.tainted_tls) {
|
|
/* Transport is tainted: ignore */
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
for (i = 0; i < object->num_headers; i++) {
|
|
if (strcasecmp("Strict-Transport-Security",
|
|
object->headers[i].name) == 0) {
|
|
urldb_set_hsts_policy(object->url,
|
|
object->headers[i].value);
|
|
/* Only process the first one we find */
|
|
break;
|
|
}
|
|
}
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Handle FETCH_REDIRECT event
|
|
*
|
|
* \param object Object being redirected
|
|
* \param target Target of redirect (may be relative)
|
|
* \param replacement Pointer to location to receive replacement object
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*/
|
|
static nserror llcache_fetch_redirect(llcache_object *object,
|
|
const char *target, llcache_object **replacement)
|
|
{
|
|
nserror error;
|
|
llcache_object *dest;
|
|
llcache_object_user *user, *next;
|
|
const llcache_post_data *post = object->fetch.post;
|
|
nsurl *url, *hsts_url;
|
|
lwc_string *scheme;
|
|
lwc_string *object_scheme;
|
|
bool match, hsts_in_use;
|
|
/* Extract HTTP response code from the fetch object */
|
|
long http_code = fetch_http_code(object->fetch.fetch);
|
|
llcache_event event;
|
|
|
|
/* Abort fetch for this object */
|
|
fetch_abort(object->fetch.fetch);
|
|
object->fetch.fetch = NULL;
|
|
|
|
/* Invalidate the cache control data */
|
|
llcache_invalidate_cache_control_data(object);
|
|
|
|
/* And mark it complete */
|
|
object->fetch.state = LLCACHE_FETCH_COMPLETE;
|
|
|
|
(void) llcache_hsts_update_policy(object);
|
|
|
|
/* Forcibly stop redirecting if we've followed too many redirects */
|
|
#define REDIRECT_LIMIT 10
|
|
if (object->fetch.redirect_count > REDIRECT_LIMIT) {
|
|
NSLOG(llcache, INFO, "Too many nested redirects");
|
|
|
|
event.type = LLCACHE_EVENT_ERROR;
|
|
event.data.error.code = NSERROR_BAD_REDIRECT;
|
|
event.data.error.msg = messages_get("BadRedirect");
|
|
|
|
return llcache_send_event_to_users(object, &event);
|
|
}
|
|
#undef REDIRECT_LIMIT
|
|
|
|
/* Make target absolute */
|
|
error = nsurl_join(object->url, target, &url);
|
|
if (error != NSERROR_OK)
|
|
return error;
|
|
|
|
/* Perform HSTS transform */
|
|
error = llcache_hsts_transform_url(url, &hsts_url, &hsts_in_use);
|
|
if (error != NSERROR_OK) {
|
|
nsurl_unref(url);
|
|
return error;
|
|
}
|
|
nsurl_unref(url);
|
|
|
|
/* Inform users of redirect */
|
|
event.type = LLCACHE_EVENT_REDIRECT;
|
|
event.data.redirect.from = object->url;
|
|
event.data.redirect.to = hsts_url;
|
|
|
|
error = llcache_send_event_to_users(object, &event);
|
|
|
|
if (error != NSERROR_OK) {
|
|
nsurl_unref(hsts_url);
|
|
return error;
|
|
}
|
|
|
|
/* Reject attempts to redirect from unvalidated to validated schemes
|
|
* A "validated" scheme is one over which we have some guarantee that
|
|
* the source is trustworthy. */
|
|
object_scheme = nsurl_get_component(object->url, NSURL_SCHEME);
|
|
scheme = nsurl_get_component(hsts_url, NSURL_SCHEME);
|
|
|
|
/* resource: and about: are allowed to redirect anywhere */
|
|
if ((lwc_string_isequal(object_scheme, corestring_lwc_resource,
|
|
&match) == lwc_error_ok && match == false) &&
|
|
(lwc_string_isequal(object_scheme, corestring_lwc_about,
|
|
&match) == lwc_error_ok && match == false)) {
|
|
/* file, about and resource are not valid redirect targets */
|
|
if ((lwc_string_isequal(object_scheme, corestring_lwc_file,
|
|
&match) == lwc_error_ok && match == true) ||
|
|
(lwc_string_isequal(object_scheme, corestring_lwc_about,
|
|
&match) == lwc_error_ok && match == true) ||
|
|
(lwc_string_isequal(object_scheme, corestring_lwc_resource,
|
|
&match) == lwc_error_ok && match == true)) {
|
|
lwc_string_unref(object_scheme);
|
|
lwc_string_unref(scheme);
|
|
nsurl_unref(hsts_url);
|
|
return NSERROR_OK;
|
|
}
|
|
}
|
|
|
|
lwc_string_unref(scheme);
|
|
lwc_string_unref(object_scheme);
|
|
|
|
/* Bail out if we've no way of handling this URL */
|
|
if (fetch_can_fetch(hsts_url) == false) {
|
|
nsurl_unref(hsts_url);
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
if (http_code == 301 || http_code == 302 || http_code == 303) {
|
|
/* 301, 302, 303 redirects are all unconditional GET requests */
|
|
post = NULL;
|
|
} else if (http_code != 307 || post != NULL) {
|
|
/** \todo 300, 305, 307 with POST */
|
|
nsurl_unref(hsts_url);
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/* Attempt to fetch target URL */
|
|
error = llcache_object_retrieve(hsts_url, object->fetch.flags,
|
|
object->fetch.referer, post,
|
|
object->fetch.redirect_count + 1,
|
|
hsts_in_use, &dest);
|
|
|
|
/* No longer require url */
|
|
nsurl_unref(hsts_url);
|
|
|
|
if (error != NSERROR_OK)
|
|
return error;
|
|
|
|
/* Move user(s) to replacement object */
|
|
for (user = object->users; user != NULL; user = next) {
|
|
next = user->next;
|
|
|
|
llcache_object_remove_user(object, user);
|
|
llcache_object_add_user(dest, user);
|
|
}
|
|
|
|
/* Dest is now our object */
|
|
*replacement = dest;
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Update an object's cache state
|
|
*
|
|
* \param object Object to update cache for
|
|
* \return NSERROR_OK.
|
|
*/
|
|
static nserror llcache_object_cache_update(llcache_object *object)
|
|
{
|
|
if (object->cache.date == 0)
|
|
object->cache.date = time(NULL);
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Handle FETCH_NOTMODIFIED event
|
|
*
|
|
* \param object Object to process
|
|
* \param replacement Pointer to location to receive replacement object
|
|
* \return NSERROR_OK.
|
|
*/
|
|
static nserror llcache_fetch_notmodified(llcache_object *object,
|
|
llcache_object **replacement)
|
|
{
|
|
/* There may be no candidate if the server erroneously responded
|
|
* to an unconditional request with a 304 Not Modified response.
|
|
* In this case, we simply retain the initial object, having
|
|
* invalidated it and marked it as complete.
|
|
*/
|
|
if (object->candidate != NULL) {
|
|
llcache_object_user *user, *next;
|
|
|
|
/* Move user(s) to candidate content */
|
|
for (user = object->users; user != NULL; user = next) {
|
|
next = user->next;
|
|
|
|
llcache_object_remove_user(object, user);
|
|
llcache_object_add_user(object->candidate, user);
|
|
}
|
|
|
|
/* Candidate is no longer a candidate for us */
|
|
object->candidate->candidate_count--;
|
|
|
|
/* Clone our cache control data into the candidate */
|
|
llcache_object_clone_cache_data(object, object->candidate,
|
|
false);
|
|
/* Bring candidate's cache data up to date */
|
|
llcache_object_cache_update(object->candidate);
|
|
/* Revert no-cache to normal, if required */
|
|
if (object->candidate->cache.no_cache ==
|
|
LLCACHE_VALIDATE_ONCE) {
|
|
object->candidate->cache.no_cache =
|
|
LLCACHE_VALIDATE_FRESH;
|
|
}
|
|
|
|
/* Candidate is now our object */
|
|
*replacement = object->candidate;
|
|
object->candidate = NULL;
|
|
} else {
|
|
/* There was no candidate: retain object */
|
|
*replacement = object;
|
|
}
|
|
|
|
/* Ensure fetch has stopped */
|
|
fetch_abort(object->fetch.fetch);
|
|
object->fetch.fetch = NULL;
|
|
|
|
/* Invalidate our cache-control data */
|
|
llcache_invalidate_cache_control_data(object);
|
|
|
|
/* Mark it complete */
|
|
object->fetch.state = LLCACHE_FETCH_COMPLETE;
|
|
|
|
(void) llcache_hsts_update_policy(object);
|
|
|
|
/* Old object will be flushed from the cache on the next poll */
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Process a chunk of fetched data
|
|
*
|
|
* \param object Object being fetched
|
|
* \param data Data to process
|
|
* \param len Byte length of data
|
|
* \return NSERROR_OK on success, appropriate error otherwise.
|
|
*/
|
|
static nserror
|
|
llcache_fetch_process_data(llcache_object *object,
|
|
const uint8_t *data,
|
|
size_t len)
|
|
{
|
|
if (object->fetch.state != LLCACHE_FETCH_DATA) {
|
|
/**
|
|
* \note
|
|
* On entry into this state, check if we need to
|
|
* invalidate the cache control data. We are guaranteed
|
|
* to have received all response headers.
|
|
*
|
|
* There are two cases in which we want to suppress
|
|
* cacheing of an object:
|
|
*
|
|
* 1) The HTTP response code is not 200 or 203
|
|
* 2) The request URI had a query string and the
|
|
* response headers did not provide an explicit
|
|
* object expiration time.
|
|
*/
|
|
long http_code = fetch_http_code(object->fetch.fetch);
|
|
|
|
if ((http_code != 200 && http_code != 203) ||
|
|
(nsurl_has_component(object->url, NSURL_QUERY) &&
|
|
(object->cache.max_age == INVALID_AGE &&
|
|
object->cache.expires == 0))) {
|
|
/* Invalidate cache control data */
|
|
llcache_invalidate_cache_control_data(object);
|
|
}
|
|
|
|
/* Release candidate, if any */
|
|
if (object->candidate != NULL) {
|
|
object->candidate->candidate_count--;
|
|
object->candidate = NULL;
|
|
}
|
|
|
|
object->fetch.state = LLCACHE_FETCH_DATA;
|
|
}
|
|
|
|
/* Resize source buffer if it's too small */
|
|
if (object->source_len + len >= object->source_alloc) {
|
|
const size_t new_len = object->source_len + len + 64 * 1024;
|
|
uint8_t *temp = realloc(object->source_data, new_len);
|
|
if (temp == NULL)
|
|
return NSERROR_NOMEM;
|
|
|
|
object->source_data = temp;
|
|
object->source_alloc = new_len;
|
|
}
|
|
|
|
/* Append this data chunk to source buffer */
|
|
memcpy(object->source_data + object->source_len, data, len);
|
|
object->source_len += len;
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
|
|
/**
|
|
* Handle an authentication request
|
|
*
|
|
* \param object Object being fetched
|
|
* \param realm Authentication realm
|
|
* \return NSERROR_OK on success, appropriate error otherwise.
|
|
*/
|
|
static nserror llcache_fetch_auth(llcache_object *object, const char *realm)
|
|
{
|
|
const char *auth;
|
|
nserror error = NSERROR_OK;
|
|
|
|
/* Abort fetch for this object */
|
|
fetch_abort(object->fetch.fetch);
|
|
object->fetch.fetch = NULL;
|
|
|
|
/* Invalidate cache-control data */
|
|
llcache_invalidate_cache_control_data(object);
|
|
|
|
/* Destroy headers */
|
|
llcache_destroy_headers(object);
|
|
|
|
/* If there was no realm, then default to the URL */
|
|
/** \todo If there was no WWW-Authenticate header, use response body */
|
|
if (realm == NULL)
|
|
realm = nsurl_access(object->url);
|
|
|
|
auth = urldb_get_auth_details(object->url, realm);
|
|
|
|
if (auth == NULL || object->fetch.tried_with_auth == true) {
|
|
llcache_event event;
|
|
/* No authentication details, or tried what we had, so ask */
|
|
object->fetch.tried_with_auth = false;
|
|
|
|
/* Mark object complete */
|
|
object->fetch.state = LLCACHE_FETCH_COMPLETE;
|
|
|
|
/* Inform client(s) that object fetch failed */
|
|
event.type = LLCACHE_EVENT_ERROR;
|
|
/** \todo More appropriate error message */
|
|
event.data.error.code = NSERROR_BAD_AUTH;
|
|
event.data.error.msg = realm;
|
|
|
|
error = llcache_send_event_to_users(object, &event);
|
|
} else {
|
|
/* Flag that we've tried to refetch with credentials, so
|
|
* that if the fetch fails again, we ask the user again */
|
|
object->fetch.tried_with_auth = true;
|
|
error = llcache_object_refetch(object);
|
|
}
|
|
|
|
return error;
|
|
}
|
|
|
|
/**
|
|
* Handle a TLS certificate verification failure
|
|
*
|
|
* \param object Object being fetched
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*/
|
|
static nserror llcache_fetch_cert_error(llcache_object *object)
|
|
{
|
|
nserror error = NSERROR_OK;
|
|
|
|
/* Fetch has been stopped, and destroyed. Invalidate object's pointer */
|
|
object->fetch.fetch = NULL;
|
|
|
|
/* Invalidate cache-control data */
|
|
llcache_invalidate_cache_control_data(object);
|
|
|
|
/* Consider the TLS transport tainted */
|
|
object->fetch.tainted_tls = true;
|
|
|
|
/* Only give the user a chance if HSTS isn't in use for this fetch */
|
|
if (object->fetch.hsts_in_use == false) {
|
|
llcache_event event;
|
|
|
|
/* Mark object complete */
|
|
object->fetch.state = LLCACHE_FETCH_COMPLETE;
|
|
|
|
/* Inform client(s) that object fetch failed */
|
|
event.type = LLCACHE_EVENT_ERROR;
|
|
/** \todo More appropriate error message */
|
|
event.data.error.code = NSERROR_BAD_CERTS;
|
|
event.data.error.msg = messages_get("FetchFailed");
|
|
|
|
error = llcache_send_event_to_users(object, &event);
|
|
} else {
|
|
llcache_event event;
|
|
|
|
/* Mark object complete */
|
|
object->fetch.state = LLCACHE_FETCH_COMPLETE;
|
|
|
|
/* Inform client(s) that object fetch failed */
|
|
event.type = LLCACHE_EVENT_ERROR;
|
|
/** \todo More appropriate error message */
|
|
event.data.error.code = NSERROR_UNKNOWN;
|
|
event.data.error.msg = messages_get("FetchFailed");
|
|
|
|
error = llcache_send_event_to_users(object, &event);
|
|
}
|
|
|
|
return error;
|
|
}
|
|
|
|
|
|
/**
|
|
* Handle a TLS connection setup failure
|
|
*
|
|
* \param object Object being fetched
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*/
|
|
static nserror llcache_fetch_ssl_error(llcache_object *object)
|
|
{
|
|
nserror error = NSERROR_OK;
|
|
|
|
/* Fetch has been stopped, and destroyed. Invalidate object's pointer */
|
|
object->fetch.fetch = NULL;
|
|
|
|
/* Invalidate cache-control data */
|
|
llcache_invalidate_cache_control_data(object);
|
|
|
|
/* Consider the TLS transport tainted */
|
|
object->fetch.tainted_tls = true;
|
|
|
|
/* Make no attempt to downgrade if HSTS is in use
|
|
* (i.e. assume server does TLS properly) */
|
|
if (object->fetch.hsts_in_use ||
|
|
object->fetch.tried_with_tls_downgrade) {
|
|
/* Have already tried to downgrade, so give up */
|
|
llcache_event event;
|
|
|
|
/* Mark object complete */
|
|
object->fetch.state = LLCACHE_FETCH_COMPLETE;
|
|
|
|
/* Inform client(s) that object fetch failed */
|
|
event.type = LLCACHE_EVENT_ERROR;
|
|
/** \todo More appropriate error message */
|
|
event.data.error.code = NSERROR_UNKNOWN;
|
|
event.data.error.msg = messages_get("FetchFailed");
|
|
|
|
error = llcache_send_event_to_users(object, &event);
|
|
} else {
|
|
/* Flag that we've tried to downgrade, so that if the
|
|
* fetch fails again, we give up */
|
|
object->fetch.tried_with_tls_downgrade = true;
|
|
error = llcache_object_refetch(object);
|
|
}
|
|
|
|
return error;
|
|
}
|
|
|
|
|
|
/**
|
|
* handle time out while trying to fetch.
|
|
*
|
|
* \param object Object being fetched
|
|
* \return NSERROR_OK on success otherwise error code
|
|
*/
|
|
static nserror llcache_fetch_timeout(llcache_object *object)
|
|
{
|
|
llcache_event event;
|
|
|
|
/* The fetch has already been cleaned up by the fetcher but
|
|
* we would like to retry if we can.
|
|
*/
|
|
if (object->fetch.retries_remaining > 1) {
|
|
object->fetch.retries_remaining--;
|
|
return llcache_object_refetch(object);
|
|
}
|
|
|
|
/* The fetch has has already been cleaned up by the fetcher */
|
|
object->fetch.state = LLCACHE_FETCH_COMPLETE;
|
|
object->fetch.fetch = NULL;
|
|
|
|
/* Release candidate, if any */
|
|
if (object->candidate != NULL) {
|
|
object->candidate->candidate_count--;
|
|
object->candidate = NULL;
|
|
}
|
|
|
|
/* Invalidate cache control data */
|
|
llcache_invalidate_cache_control_data(object);
|
|
|
|
event.type = LLCACHE_EVENT_ERROR;
|
|
event.data.error.code = NSERROR_TIMEOUT;
|
|
event.data.error.msg = NULL;
|
|
|
|
return llcache_send_event_to_users(object, &event);
|
|
}
|
|
|
|
|
|
/**
|
|
* Construct a sorted list of objects available for writeout operation.
|
|
*
|
|
* The list contains fresh cacheable objects held in RAM with no
|
|
* pending fetches. Any objects with a remaining lifetime less than
|
|
* the configured minimum lifetime are simply not considered, they will
|
|
* become stale before pushing to backing store is worth the cost.
|
|
*
|
|
* \todo calculate useful cost metrics to improve sorting.
|
|
*
|
|
* \param[out] lst_out list of candidate objects.
|
|
* \param[out] lst_len_out Number of candidate objects in result.
|
|
* \return NSERROR_OK with \a lst_out and \a lst_len_out updated or
|
|
* error code.
|
|
*/
|
|
static nserror
|
|
build_candidate_list(struct llcache_object ***lst_out, int *lst_len_out)
|
|
{
|
|
llcache_object *object, *next;
|
|
struct llcache_object **lst;
|
|
int lst_len = 0;
|
|
int remaining_lifetime;
|
|
|
|
#define MAX_PERSIST_PER_RUN 128
|
|
|
|
lst = calloc(MAX_PERSIST_PER_RUN, sizeof(struct llcache_object *));
|
|
if (lst == NULL) {
|
|
return NSERROR_NOMEM;
|
|
}
|
|
|
|
for (object = llcache->cached_objects; object != NULL; object = next) {
|
|
next = object->next;
|
|
|
|
remaining_lifetime = llcache_object_rfc2616_remaining_lifetime(
|
|
&object->cache);
|
|
|
|
/* cacehable objects with no pending fetches, not
|
|
* already on disc and with sufficient lifetime to
|
|
* make disc cache worthwhile
|
|
*/
|
|
if ((object->candidate_count == 0) &&
|
|
(object->fetch.fetch == NULL) &&
|
|
(object->store_state == LLCACHE_STATE_RAM) &&
|
|
(remaining_lifetime > llcache->minimum_lifetime)) {
|
|
lst[lst_len] = object;
|
|
lst_len++;
|
|
if (lst_len == MAX_PERSIST_PER_RUN)
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (lst_len == 0) {
|
|
free(lst);
|
|
return NSERROR_NOT_FOUND;
|
|
}
|
|
|
|
/** \todo sort list here */
|
|
|
|
*lst_len_out = lst_len;
|
|
*lst_out = lst;
|
|
|
|
#undef MAX_PERSIST_PER_RUN
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Write an object to the backing store.
|
|
*
|
|
* \param object The object to put in the backing store.
|
|
* \param written_out The amount of data written out.
|
|
* \param elapsed The time in ms it took to complete the write to backing store.
|
|
* \return NSERROR_OK on success or appropriate error code.
|
|
*/
|
|
static nserror
|
|
write_backing_store(struct llcache_object *object,
|
|
size_t *written_out,
|
|
unsigned long *elapsed)
|
|
{
|
|
nserror ret;
|
|
uint8_t *metadata;
|
|
size_t metadatasize;
|
|
uint64_t startms = 0;
|
|
uint64_t endms = 1000;
|
|
|
|
nsu_getmonotonic_ms(&startms);
|
|
|
|
/* put object data in backing store */
|
|
ret = guit->llcache->store(object->url,
|
|
BACKING_STORE_NONE,
|
|
object->source_data,
|
|
object->source_len);
|
|
if (ret != NSERROR_OK) {
|
|
/* unable to put source data in backing store */
|
|
return ret;
|
|
}
|
|
|
|
ret = llcache_serialise_metadata(object, &metadata, &metadatasize);
|
|
if (ret != NSERROR_OK) {
|
|
/* There has been a metadata serialisation error. Ensure the
|
|
* already written data object is invalidated.
|
|
*/
|
|
guit->llcache->invalidate(object->url);
|
|
return ret;
|
|
}
|
|
|
|
ret = guit->llcache->store(object->url,
|
|
BACKING_STORE_META,
|
|
metadata,
|
|
metadatasize);
|
|
guit->llcache->release(object->url, BACKING_STORE_META);
|
|
if (ret != NSERROR_OK) {
|
|
/* There has been an error putting the metadata in the
|
|
* backing store. Ensure the data object is invalidated.
|
|
*/
|
|
guit->llcache->invalidate(object->url);
|
|
return ret;
|
|
}
|
|
nsu_getmonotonic_ms(&endms);
|
|
|
|
object->store_state = LLCACHE_STATE_DISC;
|
|
|
|
*written_out = object->source_len + metadatasize;
|
|
|
|
/* by ignoring the overflow this assumes the writeout took
|
|
* less than 5 weeks.
|
|
*/
|
|
*elapsed = endms - startms;
|
|
|
|
/* ensure the writeout is reported to have taken at least the
|
|
* minimal amount of time
|
|
*/
|
|
if (*elapsed == 0) {
|
|
*elapsed = 1;
|
|
}
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Check for overall write performance.
|
|
*
|
|
* If the overall write bandwidth has fallen below a useful level for
|
|
* the backing store to be effective disable it.
|
|
*
|
|
* It is important to ensure a useful amount of data has been written
|
|
* before calculating bandwidths otherwise tiny files taking a
|
|
* disproportionately long time to write might trigger this erroneously.
|
|
*
|
|
* \param p The context pointer passed to the callback.
|
|
*/
|
|
static void llcache_persist_slowcheck(void *p)
|
|
{
|
|
uint64_t total_bandwidth; /* total bandwidth */
|
|
|
|
if (llcache->total_written > (2 * llcache->minimum_bandwidth)) {
|
|
|
|
total_bandwidth = (llcache->total_written * 1000) / llcache->total_elapsed;
|
|
|
|
if (total_bandwidth < llcache->minimum_bandwidth) {
|
|
NSLOG(llcache, INFO,
|
|
"Current bandwidth %"PRIu64" less than minimum %"PRIsizet,
|
|
total_bandwidth,
|
|
llcache->minimum_bandwidth);
|
|
guit->llcache->finalise();
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Possibly write objects data to backing store.
|
|
*
|
|
* \param p The context pointer passed to the callback.
|
|
*/
|
|
static void llcache_persist(void *p)
|
|
{
|
|
nserror ret;
|
|
struct llcache_object **lst; /* candidate object list */
|
|
int lst_count; /* number of candidates in list */
|
|
int idx; /* current candidate object index in list */
|
|
int next = -1; /* when the next run should be scheduled for */
|
|
|
|
unsigned long write_limit; /* max number of bytes to write in this run*/
|
|
|
|
size_t written; /* all bytes written for a single object */
|
|
unsigned long elapsed; /* how long writing an object took */
|
|
|
|
size_t total_written = 0; /* total bytes written in this run */
|
|
unsigned long total_elapsed = 1; /* total ms used to write bytes */
|
|
unsigned long total_bandwidth = 0; /* total bandwidth */
|
|
|
|
ret = build_candidate_list(&lst, &lst_count);
|
|
if (ret != NSERROR_OK) {
|
|
NSLOG(llcache, DEBUG, "Unable to construct candidate list for persistent writeout");
|
|
return;
|
|
}
|
|
|
|
write_limit = (llcache->maximum_bandwidth * llcache->time_quantum) / 1000;
|
|
|
|
/* obtained a candidate list, make each object persistent in turn */
|
|
for (idx = 0; idx < lst_count; idx++) {
|
|
ret = write_backing_store(lst[idx], &written, &elapsed);
|
|
if (ret != NSERROR_OK) {
|
|
continue;
|
|
}
|
|
|
|
/* successfully wrote object to backing store */
|
|
total_written += written;
|
|
total_elapsed += elapsed;
|
|
total_bandwidth = (total_written * 1000) / total_elapsed;
|
|
|
|
NSLOG(llcache, DEBUG,
|
|
"Wrote %"PRIssizet" bytes in %lums bw:%lu %s",
|
|
written, elapsed, (written * 1000) / elapsed,
|
|
nsurl_access(lst[idx]->url) );
|
|
|
|
/* check to for the time quantum or the size
|
|
* (bandwidth) for this run being exceeded.
|
|
*/
|
|
if (total_elapsed > llcache->time_quantum) {
|
|
NSLOG(llcache, INFO, "Overran timeslot");
|
|
/* writeout has exhausted the available time.
|
|
* Either the writeout is slow or the last
|
|
* object was very large.
|
|
*/
|
|
if (total_bandwidth < llcache->minimum_bandwidth) {
|
|
/* Writeout was slow in this time quantum.
|
|
* Schedule a check in the future to see if
|
|
* overall performance is too slow to be useful.
|
|
*/
|
|
guit->misc->schedule(
|
|
llcache->time_quantum * 100,
|
|
llcache_persist_slowcheck,
|
|
NULL);
|
|
break;
|
|
} else {
|
|
if (total_bandwidth > llcache->maximum_bandwidth) {
|
|
/* fast writeout of large file
|
|
* so calculate delay as if
|
|
* write happened only at max
|
|
* limit
|
|
*/
|
|
next = ((total_written * llcache->time_quantum) / write_limit) - total_elapsed;
|
|
} else {
|
|
next = llcache->time_quantum;
|
|
}
|
|
break;
|
|
}
|
|
} else if (total_written > write_limit) {
|
|
/* The bandwidth limit has been reached. */
|
|
|
|
if (total_bandwidth > llcache->maximum_bandwidth) {
|
|
/* fast writeout of large file so
|
|
* calculate delay as if write
|
|
* happened only at max limit
|
|
*/
|
|
next = ((total_written * llcache->time_quantum) / write_limit) - total_elapsed;
|
|
} else {
|
|
next = llcache->time_quantum - total_elapsed;
|
|
}
|
|
break;
|
|
}
|
|
|
|
}
|
|
free(lst);
|
|
|
|
/* Completed list without running out of allowed bytes or time */
|
|
if (idx == lst_count) {
|
|
/* only reschedule if writing is making any progress at all */
|
|
if (total_written > 0) {
|
|
next = llcache->time_quantum - total_elapsed;
|
|
} else {
|
|
next = -1;
|
|
}
|
|
}
|
|
|
|
llcache->total_written += total_written;
|
|
llcache->total_elapsed += total_elapsed;
|
|
|
|
NSLOG(llcache, DEBUG,
|
|
"writeout size:%"PRIssizet" time:%lu bandwidth:%lubytes/s",
|
|
total_written, total_elapsed, total_bandwidth);
|
|
|
|
NSLOG(llcache, DEBUG, "Rescheduling writeout in %dms", next);
|
|
guit->misc->schedule(next, llcache_persist, NULL);
|
|
}
|
|
|
|
|
|
/**
|
|
* Handler for fetch events
|
|
*
|
|
* \param msg Fetch event
|
|
* \param p Our private data
|
|
*/
|
|
static void llcache_fetch_callback(const fetch_msg *msg, void *p)
|
|
{
|
|
nserror error = NSERROR_OK;
|
|
llcache_object *object = p;
|
|
llcache_event event;
|
|
|
|
if (llcache == NULL) {
|
|
NSLOG(llcache, CRITICAL, "Callback happened after llcache finalisation");
|
|
assert(false);
|
|
/* In case assertions are off, return here */
|
|
return;
|
|
}
|
|
|
|
NSLOG(llcache, DEBUG, "Fetch event %d for %p", msg->type, object);
|
|
|
|
switch (msg->type) {
|
|
case FETCH_HEADER:
|
|
/* Received a fetch header */
|
|
object->fetch.state = LLCACHE_FETCH_HEADERS;
|
|
|
|
error = llcache_fetch_process_header(object,
|
|
msg->data.header_or_data.buf,
|
|
msg->data.header_or_data.len);
|
|
break;
|
|
|
|
/* 3xx responses */
|
|
case FETCH_REDIRECT:
|
|
/* Request resulted in a redirect */
|
|
|
|
/* Release candidate, if any */
|
|
if (object->candidate != NULL) {
|
|
object->candidate->candidate_count--;
|
|
object->candidate = NULL;
|
|
}
|
|
|
|
error = llcache_fetch_redirect(object,
|
|
msg->data.redirect, &object);
|
|
break;
|
|
|
|
case FETCH_NOTMODIFIED:
|
|
/* Conditional request determined that cached object is fresh */
|
|
error = llcache_fetch_notmodified(object, &object);
|
|
break;
|
|
|
|
/* Normal 2xx state machine */
|
|
case FETCH_DATA:
|
|
/* Received some data */
|
|
error = llcache_fetch_process_data(object,
|
|
msg->data.header_or_data.buf,
|
|
msg->data.header_or_data.len);
|
|
break;
|
|
|
|
case FETCH_FINISHED:
|
|
/* Finished fetching */
|
|
{
|
|
uint8_t *temp;
|
|
|
|
object->fetch.state = LLCACHE_FETCH_COMPLETE;
|
|
object->fetch.fetch = NULL;
|
|
|
|
/* Shrink source buffer to required size */
|
|
temp = realloc(object->source_data,
|
|
object->source_len);
|
|
/* If source_len is 0, then temp may be NULL */
|
|
if (temp != NULL || object->source_len == 0) {
|
|
object->source_data = temp;
|
|
object->source_alloc = object->source_len;
|
|
}
|
|
|
|
llcache_object_cache_update(object);
|
|
|
|
/* record when the fetch finished */
|
|
object->cache.fin_time = time(NULL);
|
|
|
|
(void) llcache_hsts_update_policy(object);
|
|
|
|
guit->misc->schedule(5000, llcache_persist, NULL);
|
|
}
|
|
break;
|
|
|
|
/* Out-of-band information */
|
|
case FETCH_TIMEDOUT:
|
|
/* Timed out while trying to fetch. */
|
|
error = llcache_fetch_timeout(object);
|
|
break;
|
|
|
|
case FETCH_ERROR:
|
|
/* An error occurred while fetching */
|
|
/* The fetch has has already been cleaned up by the fetcher */
|
|
object->fetch.state = LLCACHE_FETCH_COMPLETE;
|
|
object->fetch.fetch = NULL;
|
|
|
|
/* Release candidate, if any */
|
|
if (object->candidate != NULL) {
|
|
object->candidate->candidate_count--;
|
|
object->candidate = NULL;
|
|
}
|
|
|
|
/* Invalidate cache control data */
|
|
llcache_invalidate_cache_control_data(object);
|
|
|
|
/** \todo Consider using errorcode for something */
|
|
|
|
event.type = LLCACHE_EVENT_ERROR;
|
|
event.data.error.code = NSERROR_UNKNOWN;
|
|
event.data.error.msg = msg->data.error;
|
|
|
|
error = llcache_send_event_to_users(object, &event);
|
|
|
|
break;
|
|
|
|
case FETCH_PROGRESS:
|
|
/* Progress update */
|
|
event.type = LLCACHE_EVENT_PROGRESS;
|
|
event.data.progress_msg = msg->data.progress;
|
|
|
|
error = llcache_send_event_to_users(object, &event);
|
|
|
|
break;
|
|
|
|
case FETCH_CERTS:
|
|
/* Certificate information from the fetch */
|
|
/** \todo CERTS - Should we persist this on the object and
|
|
* then catch up new users etc?
|
|
*/
|
|
event.type = LLCACHE_EVENT_GOT_CERTS;
|
|
event.data.certs.certs = msg->data.certs.certs;
|
|
event.data.certs.num = msg->data.certs.num_certs;
|
|
|
|
error = llcache_send_event_to_users(object, &event);
|
|
break;
|
|
|
|
/* Events requiring action */
|
|
case FETCH_AUTH:
|
|
/* Need Authentication */
|
|
|
|
/* Release candidate, if any */
|
|
if (object->candidate != NULL) {
|
|
object->candidate->candidate_count--;
|
|
object->candidate = NULL;
|
|
}
|
|
|
|
error = llcache_fetch_auth(object, msg->data.auth.realm);
|
|
break;
|
|
|
|
case FETCH_CERT_ERR:
|
|
/* Something went wrong when validating TLS certificates */
|
|
|
|
/* Release candidate, if any */
|
|
if (object->candidate != NULL) {
|
|
object->candidate->candidate_count--;
|
|
object->candidate = NULL;
|
|
}
|
|
|
|
error = llcache_fetch_cert_error(object);
|
|
break;
|
|
|
|
case FETCH_SSL_ERR:
|
|
/* TLS connection setup failed */
|
|
|
|
/* Release candidate, if any */
|
|
if (object->candidate != NULL) {
|
|
object->candidate->candidate_count--;
|
|
object->candidate = NULL;
|
|
}
|
|
|
|
error = llcache_fetch_ssl_error(object);
|
|
break;
|
|
}
|
|
|
|
/* Deal with any errors reported by event handlers */
|
|
if (error != NSERROR_OK) {
|
|
if (error == NSERROR_NOMEM) {
|
|
/* attempt to purge the cache to free some
|
|
* memory. will not help this fetch, but may
|
|
* allow the UI to report errors etc.
|
|
*/
|
|
llcache_clean(true);
|
|
}
|
|
|
|
if (object->fetch.fetch != NULL) {
|
|
fetch_abort(object->fetch.fetch);
|
|
object->fetch.fetch = NULL;
|
|
|
|
/* Invalidate cache control data */
|
|
llcache_invalidate_cache_control_data(object);
|
|
|
|
object->fetch.state = LLCACHE_FETCH_COMPLETE;
|
|
}
|
|
}
|
|
|
|
/* There may be users which are not caught up so schedule ourselves */
|
|
llcache_users_not_caught_up();
|
|
}
|
|
|
|
/**
|
|
* Find a user of a low-level cache object
|
|
*
|
|
* \param handle External cache handle to search for
|
|
* \return Pointer to corresponding user, or NULL if not found
|
|
*/
|
|
static llcache_object_user *llcache_object_find_user(
|
|
const llcache_handle *handle)
|
|
{
|
|
llcache_object_user *user;
|
|
|
|
assert(handle->object != NULL);
|
|
|
|
for (user = handle->object->users; user != NULL; user = user->next) {
|
|
if (user->handle == handle)
|
|
break;
|
|
}
|
|
|
|
return user;
|
|
}
|
|
|
|
|
|
/**
|
|
* Determine if a low-level cache object resides in a given list
|
|
*
|
|
* \param object Object to search for
|
|
* \param list List to search in
|
|
* \return True if object resides in list, false otherwise
|
|
*/
|
|
static bool llcache_object_in_list(const llcache_object *object,
|
|
const llcache_object *list)
|
|
{
|
|
while (list != NULL) {
|
|
if (list == object)
|
|
break;
|
|
|
|
list = list->next;
|
|
}
|
|
|
|
return list != NULL;
|
|
}
|
|
|
|
/**
|
|
* Notify users of an object's current state
|
|
*
|
|
* \param object Object to notify users about
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*/
|
|
static nserror llcache_object_notify_users(llcache_object *object)
|
|
{
|
|
nserror error;
|
|
llcache_object_user *user, *next_user;
|
|
llcache_event event;
|
|
bool emitted_notify = false;
|
|
|
|
/**
|
|
* State transitions and event emission for users.
|
|
* Rows: user state. Cols: object state.
|
|
*
|
|
* User - Obj INIT HEADERS DATA COMPLETE
|
|
* INIT - T T* T*
|
|
* HEADERS - - T T*
|
|
* DATA - - M T
|
|
* COMPLETE - - - -
|
|
*
|
|
* T => transition user to object state
|
|
* M => no transition required, but may need to emit event
|
|
*
|
|
* The transitions marked with an asterisk can be removed by moving
|
|
* the user context into the subsequent state and then reevaluating.
|
|
*
|
|
* Events are issued as follows:
|
|
*
|
|
* HAD_HEADERS: on transition from HEADERS -> DATA state
|
|
* HAD_DATA : in DATA state, whenever there's new source data
|
|
* DONE : on transition from DATA -> COMPLETE state
|
|
*/
|
|
|
|
for (user = object->users; user != NULL; user = next_user) {
|
|
/* Emit necessary events to bring the user up-to-date */
|
|
llcache_handle *handle = user->handle;
|
|
const llcache_fetch_state objstate = object->fetch.state;
|
|
|
|
/* Flag that this user is the current iteration target
|
|
* in case the client attempts to destroy it underneath us */
|
|
user->iterator_target = true;
|
|
|
|
/* A note on the computation of next_user:
|
|
*
|
|
* Within this loop, we may make a number of calls to
|
|
* client code. Our contract with clients is that they
|
|
* can do whatever they like from within their callback
|
|
* handlers. This is so that we limit the pain of
|
|
* reentrancy to this module alone.
|
|
*
|
|
* One of the things a client can do from within its
|
|
* callback handler is to remove users from this object's
|
|
* user list. In the common case, the user they attempt
|
|
* to remove is the current iteration target, and we
|
|
* already protect against that causing problems here.
|
|
* However, no such protection exists if the client
|
|
* attempts to remove other users from this object's
|
|
* user list.
|
|
*
|
|
* Therefore, we cannot compute next_user up-front
|
|
* and expect it to remain valid across calls to
|
|
* client code (as the identity of the next user
|
|
* in the list may change underneath us). Instead,
|
|
* we must compute next_user at the point where we
|
|
* are about to cause another iteration of this loop
|
|
* (i.e. at the very end, and also at the points where
|
|
* continue is used)
|
|
*/
|
|
|
|
if (handle->state != objstate) {
|
|
if (emitted_notify == false) {
|
|
NSLOG(llcache, DEBUG,
|
|
"Notifying users of %p",
|
|
object);
|
|
emitted_notify = true;
|
|
}
|
|
|
|
NSLOG(llcache, DEBUG,
|
|
"User %p state: %d Object state: %d",
|
|
user,
|
|
handle->state,
|
|
objstate);
|
|
}
|
|
|
|
/* User: INIT, Obj: HEADERS, DATA, COMPLETE => User->HEADERS */
|
|
if (handle->state == LLCACHE_FETCH_INIT &&
|
|
objstate > LLCACHE_FETCH_INIT) {
|
|
handle->state = LLCACHE_FETCH_HEADERS;
|
|
}
|
|
|
|
/* User: HEADERS, Obj: DATA, COMPLETE => User->DATA */
|
|
if (handle->state == LLCACHE_FETCH_HEADERS &&
|
|
objstate > LLCACHE_FETCH_HEADERS) {
|
|
handle->state = LLCACHE_FETCH_DATA;
|
|
|
|
/* Emit HAD_HEADERS event */
|
|
event.type = LLCACHE_EVENT_HAD_HEADERS;
|
|
|
|
error = handle->cb(handle, &event, handle->pw);
|
|
|
|
if (user->queued_for_delete) {
|
|
next_user = user->next;
|
|
llcache_object_remove_user(object, user);
|
|
llcache_object_user_destroy(user);
|
|
|
|
if (error != NSERROR_OK)
|
|
return error;
|
|
|
|
continue;
|
|
} else if (error == NSERROR_NEED_DATA) {
|
|
/* User requested replay */
|
|
handle->state = LLCACHE_FETCH_HEADERS;
|
|
|
|
/* Continue with the next user -- we'll
|
|
* reemit the event next time round */
|
|
user->iterator_target = false;
|
|
next_user = user->next;
|
|
llcache_users_not_caught_up();
|
|
continue;
|
|
} else if (error != NSERROR_OK) {
|
|
user->iterator_target = false;
|
|
return error;
|
|
}
|
|
}
|
|
|
|
/* User: DATA, Obj: DATA, COMPLETE, more source available */
|
|
if (handle->state == LLCACHE_FETCH_DATA &&
|
|
objstate >= LLCACHE_FETCH_DATA &&
|
|
object->source_len > handle->bytes) {
|
|
size_t orig_handle_read;
|
|
|
|
/* Construct HAD_DATA event */
|
|
event.type = LLCACHE_EVENT_HAD_DATA;
|
|
event.data.data.buf =
|
|
object->source_data + handle->bytes;
|
|
event.data.data.len =
|
|
object->source_len - handle->bytes;
|
|
|
|
/* Update record of last byte emitted */
|
|
if (object->fetch.flags &
|
|
LLCACHE_RETRIEVE_STREAM_DATA) {
|
|
/* Streaming, so reset to zero to
|
|
* minimise amount of cached source data.
|
|
* Additionally, we don't support replay
|
|
* when streaming. */
|
|
orig_handle_read = 0;
|
|
handle->bytes = object->source_len = 0;
|
|
} else {
|
|
orig_handle_read = handle->bytes;
|
|
handle->bytes = object->source_len;
|
|
}
|
|
|
|
/* Emit event */
|
|
error = handle->cb(handle, &event, handle->pw);
|
|
if (user->queued_for_delete) {
|
|
next_user = user->next;
|
|
llcache_object_remove_user(object, user);
|
|
llcache_object_user_destroy(user);
|
|
|
|
if (error != NSERROR_OK)
|
|
return error;
|
|
|
|
continue;
|
|
} else if (error == NSERROR_NEED_DATA) {
|
|
/* User requested replay */
|
|
handle->bytes = orig_handle_read;
|
|
|
|
/* Continue with the next user -- we'll
|
|
* reemit the data next time round */
|
|
user->iterator_target = false;
|
|
next_user = user->next;
|
|
llcache_users_not_caught_up();
|
|
continue;
|
|
} else if (error != NSERROR_OK) {
|
|
user->iterator_target = false;
|
|
return error;
|
|
}
|
|
}
|
|
|
|
/* User: DATA, Obj: COMPLETE => User->COMPLETE */
|
|
if (handle->state == LLCACHE_FETCH_DATA &&
|
|
objstate > LLCACHE_FETCH_DATA) {
|
|
handle->state = LLCACHE_FETCH_COMPLETE;
|
|
|
|
/* Emit DONE event */
|
|
event.type = LLCACHE_EVENT_DONE;
|
|
|
|
error = handle->cb(handle, &event, handle->pw);
|
|
if (user->queued_for_delete) {
|
|
next_user = user->next;
|
|
llcache_object_remove_user(object, user);
|
|
llcache_object_user_destroy(user);
|
|
|
|
if (error != NSERROR_OK)
|
|
return error;
|
|
|
|
continue;
|
|
} else if (error == NSERROR_NEED_DATA) {
|
|
/* User requested replay */
|
|
handle->state = LLCACHE_FETCH_DATA;
|
|
|
|
/* Continue with the next user -- we'll
|
|
* reemit the event next time round */
|
|
user->iterator_target = false;
|
|
next_user = user->next;
|
|
llcache_users_not_caught_up();
|
|
continue;
|
|
} else if (error != NSERROR_OK) {
|
|
user->iterator_target = false;
|
|
return error;
|
|
}
|
|
}
|
|
|
|
/* No longer the target of an iterator */
|
|
user->iterator_target = false;
|
|
|
|
next_user = user->next;
|
|
}
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* Make a snapshot of the current state of an llcache_object.
|
|
*
|
|
* This has the side-effect of the new object being non-cacheable,
|
|
* also not-fetching and not a candidate for any other object.
|
|
*
|
|
* Also note that this new object has no users and at least one
|
|
* should be assigned to it before llcache_clean is entered or it
|
|
* will be immediately cleaned up.
|
|
*
|
|
* \param object The object to take a snapshot of
|
|
* \param snapshot Pointer to receive snapshot of \a object
|
|
* \return NSERROR_OK on success, appropriate error otherwise
|
|
*/
|
|
static nserror
|
|
llcache_object_snapshot(llcache_object *object, llcache_object **snapshot)
|
|
{
|
|
llcache_object *newobj;
|
|
nserror error;
|
|
|
|
error = llcache_object_new(object->url, &newobj);
|
|
|
|
if (error != NSERROR_OK)
|
|
return error;
|
|
|
|
newobj->source_alloc = newobj->source_len = object->source_len;
|
|
|
|
if (object->source_len > 0) {
|
|
newobj->source_data = malloc(newobj->source_alloc);
|
|
if (newobj->source_data == NULL) {
|
|
llcache_object_destroy(newobj);
|
|
return NSERROR_NOMEM;
|
|
}
|
|
memcpy(newobj->source_data, object->source_data,
|
|
newobj->source_len);
|
|
}
|
|
|
|
if (object->num_headers > 0) {
|
|
newobj->headers = calloc(sizeof(llcache_header),
|
|
object->num_headers);
|
|
if (newobj->headers == NULL) {
|
|
llcache_object_destroy(newobj);
|
|
return NSERROR_NOMEM;
|
|
}
|
|
while (newobj->num_headers < object->num_headers) {
|
|
llcache_header *nh =
|
|
&(newobj->headers[newobj->num_headers]);
|
|
llcache_header *oh =
|
|
&(object->headers[newobj->num_headers]);
|
|
newobj->num_headers += 1;
|
|
nh->name = strdup(oh->name);
|
|
nh->value = strdup(oh->value);
|
|
if (nh->name == NULL || nh->value == NULL) {
|
|
llcache_object_destroy(newobj);
|
|
return NSERROR_NOMEM;
|
|
}
|
|
}
|
|
}
|
|
|
|
newobj->fetch.state = LLCACHE_FETCH_COMPLETE;
|
|
|
|
*snapshot = newobj;
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/**
|
|
* total ram usage of object
|
|
*
|
|
* \param object The object to calculate the total RAM usage of.
|
|
* \return The total RAM usage in bytes.
|
|
*/
|
|
static inline uint32_t
|
|
total_object_size(llcache_object *object)
|
|
{
|
|
uint32_t tot;
|
|
size_t hdrc;
|
|
|
|
tot = sizeof(*object);
|
|
tot += nsurl_length(object->url);
|
|
|
|
if (object->source_data != NULL) {
|
|
tot += object->source_len;
|
|
}
|
|
|
|
tot += sizeof(llcache_header) * object->num_headers;
|
|
|
|
for (hdrc = 0; hdrc < object->num_headers; hdrc++) {
|
|
if (object->headers[hdrc].name != NULL) {
|
|
tot += strlen(object->headers[hdrc].name);
|
|
}
|
|
if (object->headers[hdrc].value != NULL) {
|
|
tot += strlen(object->headers[hdrc].value);
|
|
}
|
|
}
|
|
|
|
return tot;
|
|
}
|
|
|
|
/******************************************************************************
|
|
* Public API *
|
|
******************************************************************************/
|
|
|
|
/*
|
|
* Attempt to clean the cache
|
|
*
|
|
* The memory cache cleaning discards objects in order of increasing value.
|
|
*
|
|
* Exported interface documented in llcache.h
|
|
*/
|
|
void llcache_clean(bool purge)
|
|
{
|
|
llcache_object *object, *next;
|
|
uint32_t llcache_size = 0;
|
|
int remaining_lifetime;
|
|
uint32_t limit;
|
|
|
|
NSLOG(llcache, DEBUG, "Attempting cache clean");
|
|
|
|
/* If the cache is being purged set the size limit to zero. */
|
|
if (purge) {
|
|
limit = 0;
|
|
} else {
|
|
limit = llcache->limit;
|
|
}
|
|
|
|
/* Uncacheable objects with no users or fetches */
|
|
for (object = llcache->uncached_objects;
|
|
object != NULL;
|
|
object = next) {
|
|
next = object->next;
|
|
|
|
/* The candidate count of uncacheable objects is always 0 */
|
|
if ((object->users == NULL) &&
|
|
(object->candidate_count == 0) &&
|
|
(object->fetch.fetch == NULL)) {
|
|
NSLOG(llcache, DEBUG, "Discarding uncachable object with no users (%p) %s",
|
|
object, nsurl_access(object->url));
|
|
|
|
llcache_object_remove_from_list(object,
|
|
&llcache->uncached_objects);
|
|
llcache_object_destroy(object);
|
|
} else {
|
|
llcache_size += total_object_size(object);
|
|
}
|
|
}
|
|
|
|
|
|
/* Stale cacheable objects with no users or pending fetches */
|
|
for (object = llcache->cached_objects;
|
|
object != NULL;
|
|
object = next) {
|
|
next = object->next;
|
|
|
|
remaining_lifetime = llcache_object_rfc2616_remaining_lifetime(
|
|
&object->cache);
|
|
|
|
if ((object->users == NULL) &&
|
|
(object->candidate_count == 0) &&
|
|
(object->fetch.fetch == NULL) &&
|
|
(remaining_lifetime <= 0)) {
|
|
/* object is stale */
|
|
NSLOG(llcache, DEBUG, "discarding stale cacheable object with no "
|
|
"users or pending fetches (%p) %s",
|
|
object, nsurl_access(object->url));
|
|
|
|
llcache_object_remove_from_list(object,
|
|
&llcache->cached_objects);
|
|
|
|
if (object->store_state == LLCACHE_STATE_DISC) {
|
|
guit->llcache->invalidate(object->url);
|
|
}
|
|
|
|
llcache_object_destroy(object);
|
|
|
|
} else {
|
|
/* object has users so account for the storage */
|
|
llcache_size += total_object_size(object);
|
|
}
|
|
}
|
|
|
|
/* if the cache limit is exceeded try to make some objects
|
|
* persistent so their RAM can be reclaimed in the next
|
|
* step
|
|
*/
|
|
if (limit < llcache_size) {
|
|
llcache_persist(NULL);
|
|
}
|
|
|
|
/* Source data of fresh cacheable objects with no users, no
|
|
* pending fetches and pushed to persistent store while the
|
|
* cache exceeds the configured size.
|
|
*/
|
|
for (object = llcache->cached_objects;
|
|
((limit < llcache_size) && (object != NULL));
|
|
object = next) {
|
|
next = object->next;
|
|
if ((object->users == NULL) &&
|
|
(object->candidate_count == 0) &&
|
|
(object->fetch.fetch == NULL) &&
|
|
(object->store_state == LLCACHE_STATE_DISC)) {
|
|
guit->llcache->release(object->url, BACKING_STORE_NONE);
|
|
|
|
object->source_data = NULL;
|
|
|
|
llcache_size -= object->source_len;
|
|
|
|
NSLOG(llcache, DEBUG,
|
|
"Freeing source data for %p len:%"PRIssizet,
|
|
object, object->source_len);
|
|
}
|
|
}
|
|
|
|
/* Fresh cacheable objects with no users, no pending fetches
|
|
* and pushed to persistent store while the cache exceeds
|
|
* the configured size. Effectively just the llcache object metadata.
|
|
*/
|
|
for (object = llcache->cached_objects;
|
|
((limit < llcache_size) && (object != NULL));
|
|
object = next) {
|
|
next = object->next;
|
|
if ((object->users == NULL) &&
|
|
(object->candidate_count == 0) &&
|
|
(object->fetch.fetch == NULL) &&
|
|
(object->store_state == LLCACHE_STATE_DISC) &&
|
|
(object->source_data == NULL)) {
|
|
NSLOG(llcache, DEBUG,
|
|
"discarding backed object len:%"PRIssizet" age:%ld (%p) %s",
|
|
object->source_len,
|
|
(long)(time(NULL) - object->last_used),
|
|
object,
|
|
nsurl_access(object->url));
|
|
|
|
llcache_size -= total_object_size(object);
|
|
|
|
llcache_object_remove_from_list(object,
|
|
&llcache->cached_objects);
|
|
llcache_object_destroy(object);
|
|
|
|
}
|
|
}
|
|
|
|
/* Fresh cacheable objects with no users or pending fetches
|
|
* while the cache exceeds the configured size. These are the
|
|
* most valuable objects as replacing them is a full network
|
|
* fetch
|
|
*/
|
|
for (object = llcache->cached_objects;
|
|
((limit < llcache_size) && (object != NULL));
|
|
object = next) {
|
|
next = object->next;
|
|
|
|
if ((object->users == NULL) &&
|
|
(object->candidate_count == 0) &&
|
|
(object->fetch.fetch == NULL) &&
|
|
(object->store_state == LLCACHE_STATE_RAM)) {
|
|
NSLOG(llcache, DEBUG,
|
|
"discarding fresh object len:%"PRIssizet" age:%ld (%p) %s",
|
|
object->source_len,
|
|
(long)(time(NULL) - object->last_used),
|
|
object,
|
|
nsurl_access(object->url));
|
|
|
|
llcache_size -= object->source_len + sizeof(*object);
|
|
|
|
llcache_object_remove_from_list(object,
|
|
&llcache->cached_objects);
|
|
llcache_object_destroy(object);
|
|
}
|
|
}
|
|
|
|
NSLOG(llcache, DEBUG, "Size: %u (limit: %u)", llcache_size, limit);
|
|
}
|
|
|
|
/* Exported interface documented in content/llcache.h */
|
|
nserror
|
|
llcache_initialise(const struct llcache_parameters *prm)
|
|
{
|
|
llcache = calloc(1, sizeof(struct llcache_s));
|
|
if (llcache == NULL) {
|
|
return NSERROR_NOMEM;
|
|
}
|
|
|
|
llcache->limit = prm->limit;
|
|
llcache->minimum_lifetime = prm->minimum_lifetime;
|
|
llcache->minimum_bandwidth = prm->minimum_bandwidth;
|
|
llcache->maximum_bandwidth = prm->maximum_bandwidth;
|
|
llcache->time_quantum = prm->time_quantum;
|
|
llcache->fetch_attempts = prm->fetch_attempts;
|
|
llcache->all_caught_up = true;
|
|
|
|
NSLOG(llcache, INFO,
|
|
"llcache initialising with a limit of %d bytes",
|
|
llcache->limit);
|
|
|
|
/* backing store initialisation */
|
|
return guit->llcache->initialise(&prm->store);
|
|
}
|
|
|
|
|
|
/* Exported interface documented in content/llcache.h */
|
|
void llcache_finalise(void)
|
|
{
|
|
llcache_object *object, *next;
|
|
uint64_t total_bandwidth = 0; /* total bandwidth */
|
|
|
|
/* Clean uncached objects */
|
|
for (object = llcache->uncached_objects; object != NULL; object = next) {
|
|
llcache_object_user *user, *next_user;
|
|
|
|
next = object->next;
|
|
|
|
for (user = object->users; user != NULL; user = next_user) {
|
|
next_user = user->next;
|
|
|
|
if (user->handle != NULL)
|
|
free(user->handle);
|
|
|
|
free(user);
|
|
}
|
|
|
|
/* Fetch system has already been destroyed */
|
|
object->fetch.fetch = NULL;
|
|
|
|
llcache_object_destroy(object);
|
|
}
|
|
|
|
/* Clean cached objects */
|
|
for (object = llcache->cached_objects; object != NULL; object = next) {
|
|
llcache_object_user *user, *next_user;
|
|
|
|
next = object->next;
|
|
|
|
for (user = object->users; user != NULL; user = next_user) {
|
|
next_user = user->next;
|
|
|
|
if (user->handle != NULL)
|
|
free(user->handle);
|
|
|
|
free(user);
|
|
}
|
|
|
|
/* Fetch system has already been destroyed */
|
|
object->fetch.fetch = NULL;
|
|
|
|
llcache_object_destroy(object);
|
|
}
|
|
|
|
/* backing store finalisation */
|
|
guit->llcache->finalise();
|
|
|
|
if (llcache->total_elapsed > 0) {
|
|
total_bandwidth = (llcache->total_written * 1000) /
|
|
llcache->total_elapsed;
|
|
}
|
|
|
|
NSLOG(llcache, INFO,
|
|
"Backing store wrote %"PRIu64" bytes in %"PRIu64" ms ""(average %"PRIu64" bytes/second)",
|
|
llcache->total_written,
|
|
llcache->total_elapsed,
|
|
total_bandwidth);
|
|
|
|
free(llcache);
|
|
llcache = NULL;
|
|
}
|
|
|
|
/**
|
|
* Catch up the cache users with state changes from fetchers.
|
|
*
|
|
* \param ignored We ignore this because all our state comes from llcache.
|
|
*/
|
|
static void llcache_catch_up_all_users(void *ignored)
|
|
{
|
|
llcache_object *object;
|
|
|
|
/* Assume after this we'll be all caught up. If any user of a handle
|
|
* defers then we'll invalidate all_caught_up and reschedule via
|
|
* llcache_users_not_caught_up()
|
|
*/
|
|
llcache->all_caught_up = true;
|
|
|
|
/* Catch new users up with state of objects */
|
|
for (object = llcache->cached_objects; object != NULL;
|
|
object = object->next) {
|
|
llcache_object_notify_users(object);
|
|
}
|
|
|
|
for (object = llcache->uncached_objects; object != NULL;
|
|
object = object->next) {
|
|
llcache_object_notify_users(object);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Ask for ::llcache_catch_up_all_users to be scheduled ASAP to pump the
|
|
* user state machines.
|
|
*/
|
|
static void llcache_users_not_caught_up(void)
|
|
{
|
|
if (llcache->all_caught_up) {
|
|
llcache->all_caught_up = false;
|
|
guit->misc->schedule(0, llcache_catch_up_all_users, NULL);
|
|
}
|
|
}
|
|
|
|
|
|
/* Exported interface documented in content/llcache.h */
|
|
nserror llcache_handle_retrieve(nsurl *url, uint32_t flags,
|
|
nsurl *referer, const llcache_post_data *post,
|
|
llcache_handle_callback cb, void *pw,
|
|
llcache_handle **result)
|
|
{
|
|
nserror error;
|
|
llcache_object_user *user;
|
|
llcache_object *object;
|
|
nsurl *hsts_url;
|
|
bool hsts_in_use;
|
|
|
|
/* Perform HSTS transform */
|
|
error = llcache_hsts_transform_url(url, &hsts_url, &hsts_in_use);
|
|
if (error != NSERROR_OK) {
|
|
return error;
|
|
}
|
|
|
|
/* Can we fetch this URL at all? */
|
|
if (fetch_can_fetch(hsts_url) == false) {
|
|
nsurl_unref(hsts_url);
|
|
return NSERROR_NO_FETCH_HANDLER;
|
|
}
|
|
|
|
/* Create a new object user */
|
|
error = llcache_object_user_new(cb, pw, &user);
|
|
if (error != NSERROR_OK) {
|
|
nsurl_unref(hsts_url);
|
|
return error;
|
|
}
|
|
|
|
/* Retrieve a suitable object from the cache,
|
|
* creating a new one if needed. */
|
|
error = llcache_object_retrieve(hsts_url, flags, referer, post, 0,
|
|
hsts_in_use, &object);
|
|
if (error != NSERROR_OK) {
|
|
llcache_object_user_destroy(user);
|
|
nsurl_unref(hsts_url);
|
|
return error;
|
|
}
|
|
|
|
/* Add user to object */
|
|
llcache_object_add_user(object, user);
|
|
|
|
*result = user->handle;
|
|
|
|
/* Users exist which are now not caught up! */
|
|
llcache_users_not_caught_up();
|
|
|
|
nsurl_unref(hsts_url);
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
|
|
/* Exported interface documented in content/llcache.h */
|
|
nserror llcache_handle_change_callback(llcache_handle *handle,
|
|
llcache_handle_callback cb, void *pw)
|
|
{
|
|
handle->cb = cb;
|
|
handle->pw = pw;
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
|
|
/* Exported interface documented in content/llcache.h */
|
|
nserror llcache_handle_release(llcache_handle *handle)
|
|
{
|
|
nserror error = NSERROR_OK;
|
|
llcache_object *object = handle->object;
|
|
llcache_object_user *user = llcache_object_find_user(handle);
|
|
|
|
assert(user != NULL);
|
|
|
|
if (user->iterator_target) {
|
|
/* Can't remove / delete user object if it's
|
|
* the target of an iterator */
|
|
user->queued_for_delete = true;
|
|
} else {
|
|
/* Remove the user from the object and destroy it */
|
|
error = llcache_object_remove_user(object, user);
|
|
if (error == NSERROR_OK) {
|
|
error = llcache_object_user_destroy(user);
|
|
}
|
|
}
|
|
|
|
return error;
|
|
}
|
|
|
|
/* Exported interface documented in content/llcache.h */
|
|
nserror llcache_handle_clone(llcache_handle *handle, llcache_handle **result)
|
|
{
|
|
nserror error;
|
|
llcache_object_user *newuser;
|
|
|
|
error = llcache_object_user_new(handle->cb, handle->pw, &newuser);
|
|
if (error == NSERROR_OK) {
|
|
llcache_object_add_user(handle->object, newuser);
|
|
newuser->handle->state = handle->state;
|
|
*result = newuser->handle;
|
|
}
|
|
|
|
return error;
|
|
}
|
|
|
|
/* See llcache.h for documentation */
|
|
nserror llcache_handle_abort(llcache_handle *handle)
|
|
{
|
|
llcache_object_user *user = llcache_object_find_user(handle);
|
|
llcache_object *object = handle->object, *newobject;
|
|
nserror error = NSERROR_OK;
|
|
bool all_alone = true;
|
|
|
|
/* Determine if we are the only user */
|
|
if (user->prev != NULL)
|
|
all_alone = false;
|
|
if (user->next != NULL)
|
|
all_alone = false;
|
|
|
|
if (all_alone == false) {
|
|
/* We must snapshot this object */
|
|
error = llcache_object_snapshot(object, &newobject);
|
|
if (error != NSERROR_OK)
|
|
return error;
|
|
|
|
/* Move across to the new object */
|
|
if (user->iterator_target) {
|
|
/* User is current iterator target, clone it */
|
|
llcache_object_user *newuser =
|
|
calloc(1, sizeof(llcache_object_user));
|
|
if (newuser == NULL) {
|
|
llcache_object_destroy(newobject);
|
|
return NSERROR_NOMEM;
|
|
}
|
|
|
|
/* Move handle across to clone */
|
|
newuser->handle = user->handle;
|
|
user->handle = NULL;
|
|
|
|
/* Mark user as needing deletion */
|
|
user->queued_for_delete = true;
|
|
|
|
llcache_object_add_user(newobject, newuser);
|
|
} else {
|
|
llcache_object_remove_user(object, user);
|
|
llcache_object_add_user(newobject, user);
|
|
}
|
|
|
|
/* Add new object to uncached list */
|
|
llcache_object_add_to_list(newobject,
|
|
&llcache->uncached_objects);
|
|
} else {
|
|
/* We're the only user, so abort any fetch in progress */
|
|
if (object->fetch.fetch != NULL) {
|
|
fetch_abort(object->fetch.fetch);
|
|
object->fetch.fetch = NULL;
|
|
}
|
|
|
|
object->fetch.state = LLCACHE_FETCH_COMPLETE;
|
|
|
|
/* Invalidate cache control data */
|
|
llcache_invalidate_cache_control_data(object);
|
|
}
|
|
|
|
return error;
|
|
}
|
|
|
|
/* See llcache.h for documentation */
|
|
nserror llcache_handle_force_stream(llcache_handle *handle)
|
|
{
|
|
llcache_object_user *user = llcache_object_find_user(handle);
|
|
llcache_object *object = handle->object;
|
|
|
|
/* Cannot stream if there are multiple users */
|
|
if (user->prev != NULL || user->next != NULL)
|
|
return NSERROR_OK;
|
|
|
|
/* Forcibly uncache this object */
|
|
if (llcache_object_in_list(object, llcache->cached_objects)) {
|
|
llcache_object_remove_from_list(object,
|
|
&llcache->cached_objects);
|
|
llcache_object_add_to_list(object, &llcache->uncached_objects);
|
|
}
|
|
|
|
object->fetch.flags |= LLCACHE_RETRIEVE_STREAM_DATA;
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/* See llcache.h for documentation */
|
|
nserror llcache_handle_invalidate_cache_data(llcache_handle *handle)
|
|
{
|
|
if ((handle->object != NULL) &&
|
|
(handle->object->fetch.fetch == NULL) &&
|
|
(handle->object->cache.no_cache == LLCACHE_VALIDATE_FRESH)) {
|
|
/* mark the cached object as requiring validation */
|
|
handle->object->cache.no_cache = LLCACHE_VALIDATE_ONCE;
|
|
}
|
|
|
|
return NSERROR_OK;
|
|
}
|
|
|
|
/* See llcache.h for documentation */
|
|
nsurl *llcache_handle_get_url(const llcache_handle *handle)
|
|
{
|
|
return handle->object != NULL ? handle->object->url : NULL;
|
|
}
|
|
|
|
/* See llcache.h for documentation */
|
|
const uint8_t *llcache_handle_get_source_data(const llcache_handle *handle,
|
|
size_t *size)
|
|
{
|
|
*size = handle->object != NULL ? handle->object->source_len : 0;
|
|
|
|
return handle->object != NULL ? handle->object->source_data : NULL;
|
|
}
|
|
|
|
/* See llcache.h for documentation */
|
|
const char *llcache_handle_get_header(const llcache_handle *handle,
|
|
const char *key)
|
|
{
|
|
const llcache_object *object = handle->object;
|
|
size_t i;
|
|
|
|
if (object == NULL)
|
|
return NULL;
|
|
|
|
/* About as trivial as possible */
|
|
for (i = 0; i < object->num_headers; i++) {
|
|
if (strcasecmp(key, object->headers[i].name) == 0)
|
|
return object->headers[i].value;
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
/* See llcache.h for documentation */
|
|
bool llcache_handle_references_same_object(const llcache_handle *a,
|
|
const llcache_handle *b)
|
|
{
|
|
return a->object == b->object;
|
|
}
|