Update to libfetch-2.22:
- Support for conditional fetch - Allow correctly overwriting fetchRestartCalls from signal handlers by making it volatile - Explicitly document that the size of struct url can increase.
This commit is contained in:
joerg
parent
141adf6795
commit
9da2cc5c4b
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: common.c,v 1.1.1.2 2008/10/07 15:55:20 joerg Exp $ */
|
||||
/* $NetBSD: common.c,v 1.1.1.3 2009/02/14 19:36:51 joerg Exp $ */
|
||||
/*-
|
||||
* Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
|
||||
* Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>
|
||||
|
|
@ -728,6 +728,7 @@ fetch_add_entry(struct url_list *ue, struct url *base, const char *name,
|
|||
|
||||
tmp->offset = 0;
|
||||
tmp->length = 0;
|
||||
tmp->last_modified = -1;
|
||||
|
||||
++ue->length;
|
||||
|
||||
|
|
|
|||
|
|
@ -24,9 +24,9 @@
|
|||
.\" SUCH DAMAGE.
|
||||
.\"
|
||||
.\" $FreeBSD: fetch.3,v 1.64 2007/12/18 11:03:26 des Exp $
|
||||
.\" $NetBSD: fetch.3,v 1.1.1.3 2008/10/29 16:18:13 joerg Exp $
|
||||
.\" $NetBSD: fetch.3,v 1.1.1.4 2009/02/14 19:36:56 joerg Exp $
|
||||
.\"
|
||||
.Dd April 25, 2008
|
||||
.Dd February 4, 2009
|
||||
.Dt FETCH 3
|
||||
.Os
|
||||
.Sh NAME
|
||||
|
|
@ -194,6 +194,7 @@ struct url {
|
|||
char *doc;
|
||||
off_t offset;
|
||||
size_t length;
|
||||
time_t last_modified;
|
||||
};
|
||||
.Ed
|
||||
.Pp
|
||||
|
|
@ -204,6 +205,9 @@ and
|
|||
.Fn fetchParseURL
|
||||
should be freed using
|
||||
.Fn fetchFreeURL .
|
||||
The size of
|
||||
.Vt struct URL
|
||||
is not part of the ABI.
|
||||
.Pp
|
||||
.Fn fetchXGetURL ,
|
||||
.Fn fetchGetURL ,
|
||||
|
|
@ -326,6 +330,19 @@ or
|
|||
functions is read-only, and that a stream returned by one of the
|
||||
.Fn fetchPutXXX
|
||||
functions is write-only.
|
||||
.Sh PROTOCOL INDEPENDENT FLAGS
|
||||
If the
|
||||
.Ql i
|
||||
(if-modified-since) flag is specified, the library will try to fetch
|
||||
the content only if it is newer than
|
||||
.Va last_modified .
|
||||
For HTTP an
|
||||
.Li If-Modified-Since
|
||||
HTTP header is sent.
|
||||
For FTP a
|
||||
.Li MTDM
|
||||
command is sent first and compared locally.
|
||||
For FILE the source file is compared.
|
||||
.Sh FILE SCHEME
|
||||
.Fn fetchXGetFile ,
|
||||
.Fn fetchGetFile ,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: fetch.c,v 1.1.1.4 2008/11/14 23:04:30 joerg Exp $ */
|
||||
/* $NetBSD: fetch.c,v 1.1.1.5 2009/02/14 19:36:56 joerg Exp $ */
|
||||
/*-
|
||||
* Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
|
||||
* Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>
|
||||
|
|
@ -50,7 +50,7 @@ auth_t fetchAuthMethod;
|
|||
int fetchLastErrCode;
|
||||
char fetchLastErrString[MAXERRSTRING];
|
||||
int fetchTimeout;
|
||||
int fetchRestartCalls = 1;
|
||||
volatile int fetchRestartCalls = 1;
|
||||
int fetchDebug;
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -157,10 +157,12 @@ DDEESSCCRRIIPPTTIIOONN
|
|||
char *doc;
|
||||
off_t offset;
|
||||
size_t length;
|
||||
time_t last_modified;
|
||||
};
|
||||
|
||||
The pointer returned by ffeettcchhMMaakkeeUURRLL(), ffeettcchhCCooppyyUURRLL(), and
|
||||
ffeettcchhPPaarrsseeUURRLL() should be freed using ffeettcchhFFrreeeeUURRLL().
|
||||
ffeettcchhPPaarrsseeUURRLL() should be freed using ffeettcchhFFrreeeeUURRLL(). The size of _s_t_r_u_c_t
|
||||
_U_R_L is not part of the ABI.
|
||||
|
||||
ffeettcchhXXGGeettUURRLL(), ffeettcchhGGeettUURRLL(), and ffeettcchhPPuuttUURRLL() constitute the recom-
|
||||
mended interface to the ffeettcchh library. They examine the URL passed to
|
||||
|
|
@ -225,6 +227,12 @@ DDEESSCCRRIIPPTTIIOONN
|
|||
ffeettcchhGGeettXXXXXX() functions is read-only, and that a stream returned by one
|
||||
of the ffeettcchhPPuuttXXXXXX() functions is write-only.
|
||||
|
||||
PPRROOTTOOCCOOLL IINNDDEEPPEENNDDEENNTT FFLLAAGGSS
|
||||
If the `i' (if-modified-since) flag is specified, the library will try to
|
||||
fetch the content only if it is newer than _l_a_s_t___m_o_d_i_f_i_e_d. For HTTP an
|
||||
If-Modified-Since HTTP header is sent. For FTP a MTDM command is sent
|
||||
first and compared locally. For FILE the source file is compared.
|
||||
|
||||
FFIILLEE SSCCHHEEMMEE
|
||||
ffeettcchhXXGGeettFFiillee(), ffeettcchhGGeettFFiillee(), and ffeettcchhPPuuttFFiillee() provide access to
|
||||
documents which are files in a locally mounted file system. Only the
|
||||
|
|
@ -495,4 +503,4 @@ BBUUGGSS
|
|||
|
||||
Some parts of the code are not fully reentrant.
|
||||
|
||||
NetBSD 4.0 April 25, 2008 NetBSD 4.0
|
||||
NetBSD 5.0 February 4, 2009 NetBSD 5.0
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* $NetBSD: fetch.h,v 1.1.1.2 2008/10/07 15:55:20 joerg Exp $ */
|
||||
/* $NetBSD: fetch.h,v 1.1.1.3 2009/02/14 19:36:58 joerg Exp $ */
|
||||
/*-
|
||||
* Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
|
||||
* All rights reserved.
|
||||
|
|
@ -54,6 +54,7 @@ struct url {
|
|||
char *doc;
|
||||
off_t offset;
|
||||
size_t length;
|
||||
time_t last_modified;
|
||||
};
|
||||
|
||||
struct url_stat {
|
||||
|
|
@ -94,6 +95,7 @@ struct url_list {
|
|||
#define FETCH_UNKNOWN 17
|
||||
#define FETCH_URL 18
|
||||
#define FETCH_VERBOSE 19
|
||||
#define FETCH_UNCHANGED 20
|
||||
|
||||
#if defined(__cplusplus)
|
||||
extern "C" {
|
||||
|
|
@ -168,7 +170,7 @@ extern char fetchLastErrString[MAXERRSTRING];
|
|||
extern int fetchTimeout;
|
||||
|
||||
/* Restart interrupted syscalls */
|
||||
extern int fetchRestartCalls;
|
||||
extern volatile int fetchRestartCalls;
|
||||
|
||||
/* Extra verbosity */
|
||||
extern int fetchDebug;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
/* $NetBSD: file.c,v 1.1.1.2 2008/10/07 15:55:20 joerg Exp $ */
|
||||
/* $NetBSD: file.c,v 1.1.1.3 2009/02/14 19:36:58 joerg Exp $ */
|
||||
/*-
|
||||
* Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
|
||||
* Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>
|
||||
* Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org>
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
|
|
@ -79,7 +79,12 @@ fetchXGetFile(struct url *u, struct url_stat *us, const char *flags)
|
|||
{
|
||||
char *path;
|
||||
fetchIO *f;
|
||||
int fd, *cookie;
|
||||
struct url_stat local_us;
|
||||
int if_modified_since, fd, *cookie;
|
||||
|
||||
if_modified_since = CHECK_FLAG('i');
|
||||
if (if_modified_since && us == NULL)
|
||||
us = &local_us;
|
||||
|
||||
if ((path = fetchUnquotePath(u)) == NULL) {
|
||||
fetch_syserr();
|
||||
|
|
@ -95,6 +100,14 @@ fetchXGetFile(struct url *u, struct url_stat *us, const char *flags)
|
|||
|
||||
if (us && fetch_stat_file(fd, us) == -1) {
|
||||
close(fd);
|
||||
fetch_syserr();
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (if_modified_since && u->last_modified > 0 &&
|
||||
u->last_modified >= us->mtime) {
|
||||
fetchLastErrCode = FETCH_UNCHANGED;
|
||||
snprintf(fetchLastErrString, MAXERRSTRING, "Unchanged");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
/* $NetBSD: ftp.c,v 1.1.1.3 2008/10/29 16:18:14 joerg Exp $ */
|
||||
/* $NetBSD: ftp.c,v 1.1.1.4 2009/02/14 19:37:00 joerg Exp $ */
|
||||
/*-
|
||||
* Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
|
||||
* Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>
|
||||
* Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org>
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
|
|
@ -57,6 +57,11 @@
|
|||
*
|
||||
*/
|
||||
|
||||
#ifdef __linux__
|
||||
/* Keep this down to Linux, it can create surprises else where. */
|
||||
#define _GNU_SOURCE
|
||||
#endif
|
||||
|
||||
#if HAVE_CONFIG_H
|
||||
#include "config.h"
|
||||
#endif
|
||||
|
|
@ -1122,7 +1127,8 @@ ftp_request(struct url *url, const char *op, const char *op_arg,
|
|||
fetchIO *f;
|
||||
char *path;
|
||||
conn_t *conn;
|
||||
int oflag;
|
||||
int if_modified_since, oflag;
|
||||
struct url_stat local_us;
|
||||
|
||||
/* check if we should use HTTP instead */
|
||||
if (purl && strcasecmp(purl->scheme, SCHEME_HTTP) == 0) {
|
||||
|
|
@ -1154,6 +1160,10 @@ ftp_request(struct url *url, const char *op, const char *op_arg,
|
|||
return (NULL);
|
||||
}
|
||||
|
||||
if_modified_since = CHECK_FLAG('i');
|
||||
if (if_modified_since && us == NULL)
|
||||
us = &local_us;
|
||||
|
||||
/* stat file */
|
||||
if (us && ftp_stat(conn, path, us) == -1
|
||||
&& fetchLastErrCode != FETCH_PROTO
|
||||
|
|
@ -1162,6 +1172,13 @@ ftp_request(struct url *url, const char *op, const char *op_arg,
|
|||
return (NULL);
|
||||
}
|
||||
|
||||
if (if_modified_since && url->last_modified > 0 &&
|
||||
url->last_modified >= us->mtime) {
|
||||
fetchLastErrCode = FETCH_UNCHANGED;
|
||||
snprintf(fetchLastErrString, MAXERRSTRING, "Unchanged");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* just a stat */
|
||||
if (strcmp(op, "STAT") == 0)
|
||||
return fetchIO_unopen(NULL, NULL, NULL, NULL);
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
/* $NetBSD: http.c,v 1.1.1.2 2008/10/07 15:55:20 joerg Exp $ */
|
||||
/* $NetBSD: http.c,v 1.1.1.3 2009/02/14 19:37:02 joerg Exp $ */
|
||||
/*-
|
||||
* Copyright (c) 2000-2004 Dag-Erling Coïdan Smørgrav
|
||||
* Copyright (c) 2003 Thomas Klausner <wiz@NetBSD.org>
|
||||
* Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>
|
||||
* Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org>
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
|
|
@ -112,6 +112,7 @@
|
|||
#define HTTP_MOVED_PERM 301
|
||||
#define HTTP_MOVED_TEMP 302
|
||||
#define HTTP_SEE_OTHER 303
|
||||
#define HTTP_NOT_MODIFIED 304
|
||||
#define HTTP_TEMP_REDIRECT 307
|
||||
#define HTTP_NEED_AUTH 401
|
||||
#define HTTP_NEED_PROXY_AUTH 407
|
||||
|
|
@ -745,6 +746,21 @@ http_get_proxy(struct url * url, const char *flags)
|
|||
return (NULL);
|
||||
}
|
||||
|
||||
static void
|
||||
set_if_modified_since(conn_t *conn, time_t last_modified)
|
||||
{
|
||||
static const char weekdays[] = "SunMonTueWedThuFriSat";
|
||||
static const char months[] = "JanFebMarAprMayJunJulAugSepOctNovDec";
|
||||
struct tm tm;
|
||||
char buf[80];
|
||||
gmtime_r(&last_modified, &tm);
|
||||
snprintf(buf, sizeof(buf), "%.3s, %02d %.3s %4d %02d:%02d:%02d GMT",
|
||||
weekdays + tm.tm_wday * 3, tm.tm_mday, months + tm.tm_mon * 3,
|
||||
tm.tm_year + 1900, tm.tm_hour, tm.tm_min, tm.tm_sec);
|
||||
http_cmd(conn, "If-Modified-Since: %s", buf);
|
||||
}
|
||||
|
||||
|
||||
/*****************************************************************************
|
||||
* Core
|
||||
*/
|
||||
|
|
@ -761,7 +777,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
|
|||
{
|
||||
conn_t *conn;
|
||||
struct url *url, *new;
|
||||
int chunked, direct, need_auth, noredirect, verbose;
|
||||
int chunked, direct, if_modified_since, need_auth, noredirect, verbose;
|
||||
int e, i, n, val;
|
||||
off_t offset, clength, length, size;
|
||||
time_t mtime;
|
||||
|
|
@ -773,6 +789,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
|
|||
direct = CHECK_FLAG('d');
|
||||
noredirect = CHECK_FLAG('A');
|
||||
verbose = CHECK_FLAG('v');
|
||||
if_modified_since = CHECK_FLAG('i');
|
||||
|
||||
if (direct && purl) {
|
||||
fetchFreeURL(purl);
|
||||
|
|
@ -841,6 +858,9 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
|
|||
op, url->doc);
|
||||
}
|
||||
|
||||
if (if_modified_since && url->last_modified > 0)
|
||||
set_if_modified_since(conn, url->last_modified);
|
||||
|
||||
/* virtual host */
|
||||
http_cmd(conn, "Host: %s", host);
|
||||
|
||||
|
|
@ -904,6 +924,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
|
|||
switch (http_get_reply(conn)) {
|
||||
case HTTP_OK:
|
||||
case HTTP_PARTIAL:
|
||||
case HTTP_NOT_MODIFIED:
|
||||
/* fine */
|
||||
break;
|
||||
case HTTP_MOVED_PERM:
|
||||
|
|
@ -1036,7 +1057,10 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
|
|||
}
|
||||
|
||||
/* we have a hit or an error */
|
||||
if (conn->err == HTTP_OK || conn->err == HTTP_PARTIAL || HTTP_ERROR(conn->err))
|
||||
if (conn->err == HTTP_OK ||
|
||||
conn->err == HTTP_PARTIAL ||
|
||||
conn->err == HTTP_NOT_MODIFIED ||
|
||||
HTTP_ERROR(conn->err))
|
||||
break;
|
||||
|
||||
/* all other cases: we got a redirect */
|
||||
|
|
@ -1089,6 +1113,11 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
|
|||
URL->offset = offset;
|
||||
URL->length = clength;
|
||||
|
||||
if (conn->err == HTTP_NOT_MODIFIED) {
|
||||
http_seterr(HTTP_NOT_MODIFIED);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
/* wrap it up in a fetchIO */
|
||||
if ((f = http_funopen(conn, chunked)) == NULL) {
|
||||
fetch_syserr();
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# $FreeBSD: http.errors,v 1.5 2001/05/23 18:52:02 des Exp $
|
||||
# $NetBSD: http.errors,v 1.1.1.2 2008/10/07 15:55:20 joerg Exp $
|
||||
# $NetBSD: http.errors,v 1.1.1.3 2009/02/14 19:37:02 joerg Exp $
|
||||
#
|
||||
# This list is taken from RFC 2068.
|
||||
#
|
||||
|
|
@ -16,7 +16,7 @@
|
|||
301 MOVED Moved Permanently
|
||||
302 MOVED Moved Temporarily
|
||||
303 MOVED See Other
|
||||
304 OK Not Modified
|
||||
304 UNCHANGED Not Modified
|
||||
305 INFO Use Proxy
|
||||
307 MOVED Temporary Redirect
|
||||
400 PROTO Bad Request
|
||||
|
|
|
|||
Loading…
Reference in New Issue