Move the server's backup manifest code to a separate file.
basebackup.c is already a pretty big and complicated file, so it makes more sense to keep the backup manifest support routines in a separate file, for clarity and ease of maintenance. Discussion: http://postgr.es/m/CA+TgmoavRak5OdP76P8eJExDYhPEKWjMb0sxW7dF01dWFgE=uA@mail.gmail.com
This commit is contained in:
Robert Haas
parent
1e324cb0e7
commit
079ac29d4d
@ -15,6 +15,7 @@ include $(top_builddir)/src/Makefile.global
|
|||||||
override CPPFLAGS := -I. -I$(srcdir) $(CPPFLAGS)
|
override CPPFLAGS := -I. -I$(srcdir) $(CPPFLAGS)
|
||||||
|
|
||||||
OBJS = \
|
OBJS = \
|
||||||
|
backup_manifest.o \
|
||||||
basebackup.o \
|
basebackup.o \
|
||||||
repl_gram.o \
|
repl_gram.o \
|
||||||
slot.o \
|
slot.o \
|
||||||
|
|||||||
375
src/backend/replication/backup_manifest.c
Normal file
375
src/backend/replication/backup_manifest.c
Normal file
@ -0,0 +1,375 @@
|
|||||||
|
/*-------------------------------------------------------------------------
|
||||||
|
*
|
||||||
|
* backup_manifest.c
|
||||||
|
* code for generating and sending a backup manifest
|
||||||
|
*
|
||||||
|
* Portions Copyright (c) 2010-2020, PostgreSQL Global Development Group
|
||||||
|
*
|
||||||
|
* IDENTIFICATION
|
||||||
|
* src/backend/replication/backup_manifest.c
|
||||||
|
*
|
||||||
|
*-------------------------------------------------------------------------
|
||||||
|
*/
|
||||||
|
#include "postgres.h"
|
||||||
|
|
||||||
|
#include "access/timeline.h"
|
||||||
|
#include "libpq/libpq.h"
|
||||||
|
#include "libpq/pqformat.h"
|
||||||
|
#include "mb/pg_wchar.h"
|
||||||
|
#include "replication/backup_manifest.h"
|
||||||
|
#include "utils/builtins.h"
|
||||||
|
#include "utils/json.h"
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Does the user want a backup manifest?
|
||||||
|
*
|
||||||
|
* It's simplest to always have a manifest_info object, so that we don't need
|
||||||
|
* checks for NULL pointers in too many places. However, if the user doesn't
|
||||||
|
* want a manifest, we set manifest->buffile to NULL.
|
||||||
|
*/
|
||||||
|
static inline bool
|
||||||
|
IsManifestEnabled(manifest_info *manifest)
|
||||||
|
{
|
||||||
|
return (manifest->buffile != NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Convenience macro for appending data to the backup manifest.
|
||||||
|
*/
|
||||||
|
#define AppendToManifest(manifest, ...) \
|
||||||
|
{ \
|
||||||
|
char *_manifest_s = psprintf(__VA_ARGS__); \
|
||||||
|
AppendStringToManifest(manifest, _manifest_s); \
|
||||||
|
pfree(_manifest_s); \
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Initialize state so that we can construct a backup manifest.
|
||||||
|
*
|
||||||
|
* NB: Although the checksum type for the data files is configurable, the
|
||||||
|
* checksum for the manifest itself always uses SHA-256. See comments in
|
||||||
|
* SendBackupManifest.
|
||||||
|
*/
|
||||||
|
void
|
||||||
|
InitializeManifest(manifest_info *manifest, manifest_option want_manifest,
|
||||||
|
pg_checksum_type manifest_checksum_type)
|
||||||
|
{
|
||||||
|
if (want_manifest == MANIFEST_OPTION_NO)
|
||||||
|
manifest->buffile = NULL;
|
||||||
|
else
|
||||||
|
manifest->buffile = BufFileCreateTemp(false);
|
||||||
|
manifest->checksum_type = manifest_checksum_type;
|
||||||
|
pg_sha256_init(&manifest->manifest_ctx);
|
||||||
|
manifest->manifest_size = UINT64CONST(0);
|
||||||
|
manifest->force_encode = (want_manifest == MANIFEST_OPTION_FORCE_ENCODE);
|
||||||
|
manifest->first_file = true;
|
||||||
|
manifest->still_checksumming = true;
|
||||||
|
|
||||||
|
if (want_manifest != MANIFEST_OPTION_NO)
|
||||||
|
AppendToManifest(manifest,
|
||||||
|
"{ \"PostgreSQL-Backup-Manifest-Version\": 1,\n"
|
||||||
|
"\"Files\": [");
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Append a cstring to the manifest.
|
||||||
|
*/
|
||||||
|
void
|
||||||
|
AppendStringToManifest(manifest_info *manifest, char *s)
|
||||||
|
{
|
||||||
|
int len = strlen(s);
|
||||||
|
size_t written;
|
||||||
|
|
||||||
|
Assert(manifest != NULL);
|
||||||
|
if (manifest->still_checksumming)
|
||||||
|
pg_sha256_update(&manifest->manifest_ctx, (uint8 *) s, len);
|
||||||
|
written = BufFileWrite(manifest->buffile, s, len);
|
||||||
|
if (written != len)
|
||||||
|
ereport(ERROR,
|
||||||
|
(errcode_for_file_access(),
|
||||||
|
errmsg("could not write to temporary file: %m")));
|
||||||
|
manifest->manifest_size += len;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Add an entry to the backup manifest for a file.
|
||||||
|
*/
|
||||||
|
void
|
||||||
|
AddFileToManifest(manifest_info *manifest, const char *spcoid,
|
||||||
|
const char *pathname, size_t size, pg_time_t mtime,
|
||||||
|
pg_checksum_context *checksum_ctx)
|
||||||
|
{
|
||||||
|
char pathbuf[MAXPGPATH];
|
||||||
|
int pathlen;
|
||||||
|
StringInfoData buf;
|
||||||
|
|
||||||
|
if (!IsManifestEnabled(manifest))
|
||||||
|
return;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* If this file is part of a tablespace, the pathname passed to this
|
||||||
|
* function will be relative to the tar file that contains it. We want the
|
||||||
|
* pathname relative to the data directory (ignoring the intermediate
|
||||||
|
* symlink traversal).
|
||||||
|
*/
|
||||||
|
if (spcoid != NULL)
|
||||||
|
{
|
||||||
|
snprintf(pathbuf, sizeof(pathbuf), "pg_tblspc/%s/%s", spcoid,
|
||||||
|
pathname);
|
||||||
|
pathname = pathbuf;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Each file's entry needs to be separated from any entry that follows by a
|
||||||
|
* comma, but there's no comma before the first one or after the last one.
|
||||||
|
* To make that work, adding a file to the manifest starts by terminating
|
||||||
|
* the most recently added line, with a comma if appropriate, but does not
|
||||||
|
* terminate the line inserted for this file.
|
||||||
|
*/
|
||||||
|
initStringInfo(&buf);
|
||||||
|
if (manifest->first_file)
|
||||||
|
{
|
||||||
|
appendStringInfoString(&buf, "\n");
|
||||||
|
manifest->first_file = false;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
appendStringInfoString(&buf, ",\n");
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Write the relative pathname to this file out to the manifest. The
|
||||||
|
* manifest is always stored in UTF-8, so we have to encode paths that are
|
||||||
|
* not valid in that encoding.
|
||||||
|
*/
|
||||||
|
pathlen = strlen(pathname);
|
||||||
|
if (!manifest->force_encode &&
|
||||||
|
pg_verify_mbstr(PG_UTF8, pathname, pathlen, true))
|
||||||
|
{
|
||||||
|
appendStringInfoString(&buf, "{ \"Path\": ");
|
||||||
|
escape_json(&buf, pathname);
|
||||||
|
appendStringInfoString(&buf, ", ");
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
appendStringInfoString(&buf, "{ \"Encoded-Path\": \"");
|
||||||
|
enlargeStringInfo(&buf, 2 * pathlen);
|
||||||
|
buf.len += hex_encode((char *) pathname, pathlen,
|
||||||
|
&buf.data[buf.len]);
|
||||||
|
appendStringInfoString(&buf, "\", ");
|
||||||
|
}
|
||||||
|
|
||||||
|
appendStringInfo(&buf, "\"Size\": %zu, ", size);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Convert last modification time to a string and append it to the
|
||||||
|
* manifest. Since it's not clear what time zone to use and since time
|
||||||
|
* zone definitions can change, possibly causing confusion, use GMT
|
||||||
|
* always.
|
||||||
|
*/
|
||||||
|
appendStringInfoString(&buf, "\"Last-Modified\": \"");
|
||||||
|
enlargeStringInfo(&buf, 128);
|
||||||
|
buf.len += pg_strftime(&buf.data[buf.len], 128, "%Y-%m-%d %H:%M:%S %Z",
|
||||||
|
pg_gmtime(&mtime));
|
||||||
|
appendStringInfoString(&buf, "\"");
|
||||||
|
|
||||||
|
/* Add checksum information. */
|
||||||
|
if (checksum_ctx->type != CHECKSUM_TYPE_NONE)
|
||||||
|
{
|
||||||
|
uint8 checksumbuf[PG_CHECKSUM_MAX_LENGTH];
|
||||||
|
int checksumlen;
|
||||||
|
|
||||||
|
checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
|
||||||
|
|
||||||
|
appendStringInfo(&buf,
|
||||||
|
", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"",
|
||||||
|
pg_checksum_type_name(checksum_ctx->type));
|
||||||
|
enlargeStringInfo(&buf, 2 * checksumlen);
|
||||||
|
buf.len += hex_encode((char *) checksumbuf, checksumlen,
|
||||||
|
&buf.data[buf.len]);
|
||||||
|
appendStringInfoString(&buf, "\"");
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Close out the object. */
|
||||||
|
appendStringInfoString(&buf, " }");
|
||||||
|
|
||||||
|
/* OK, add it to the manifest. */
|
||||||
|
AppendStringToManifest(manifest, buf.data);
|
||||||
|
|
||||||
|
/* Avoid leaking memory. */
|
||||||
|
pfree(buf.data);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Add information about the WAL that will need to be replayed when restoring
|
||||||
|
* this backup to the manifest.
|
||||||
|
*/
|
||||||
|
void
|
||||||
|
AddWALInfoToManifest(manifest_info *manifest, XLogRecPtr startptr,
|
||||||
|
TimeLineID starttli, XLogRecPtr endptr, TimeLineID endtli)
|
||||||
|
{
|
||||||
|
List *timelines;
|
||||||
|
ListCell *lc;
|
||||||
|
bool first_wal_range = true;
|
||||||
|
bool found_start_timeline = false;
|
||||||
|
|
||||||
|
if (!IsManifestEnabled(manifest))
|
||||||
|
return;
|
||||||
|
|
||||||
|
/* Terminate the list of files. */
|
||||||
|
AppendStringToManifest(manifest, "\n],\n");
|
||||||
|
|
||||||
|
/* Read the timeline history for the ending timeline. */
|
||||||
|
timelines = readTimeLineHistory(endtli);
|
||||||
|
|
||||||
|
/* Start a list of LSN ranges. */
|
||||||
|
AppendStringToManifest(manifest, "\"WAL-Ranges\": [\n");
|
||||||
|
|
||||||
|
foreach(lc, timelines)
|
||||||
|
{
|
||||||
|
TimeLineHistoryEntry *entry = lfirst(lc);
|
||||||
|
XLogRecPtr tl_beginptr;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* We only care about timelines that were active during the backup.
|
||||||
|
* Skip any that ended before the backup started. (Note that if
|
||||||
|
* entry->end is InvalidXLogRecPtr, it means that the timeline has not
|
||||||
|
* yet ended.)
|
||||||
|
*/
|
||||||
|
if (!XLogRecPtrIsInvalid(entry->end) && entry->end < startptr)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Because the timeline history file lists newer timelines before
|
||||||
|
* older ones, the first timeline we encounter that is new enough to
|
||||||
|
* matter ought to match the ending timeline of the backup.
|
||||||
|
*/
|
||||||
|
if (first_wal_range && endtli != entry->tli)
|
||||||
|
ereport(ERROR,
|
||||||
|
errmsg("expected end timeline %u but found timeline %u",
|
||||||
|
starttli, entry->tli));
|
||||||
|
|
||||||
|
if (!XLogRecPtrIsInvalid(entry->begin))
|
||||||
|
tl_beginptr = entry->begin;
|
||||||
|
else
|
||||||
|
{
|
||||||
|
tl_beginptr = startptr;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* If we reach a TLI that has no valid beginning LSN, there can't
|
||||||
|
* be any more timelines in the history after this point, so we'd
|
||||||
|
* better have arrived at the expected starting TLI. If not,
|
||||||
|
* something's gone horribly wrong.
|
||||||
|
*/
|
||||||
|
if (starttli != entry->tli)
|
||||||
|
ereport(ERROR,
|
||||||
|
errmsg("expected start timeline %u but found timeline %u",
|
||||||
|
starttli, entry->tli));
|
||||||
|
}
|
||||||
|
|
||||||
|
AppendToManifest(manifest,
|
||||||
|
"%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }",
|
||||||
|
first_wal_range ? "" : ",\n",
|
||||||
|
entry->tli,
|
||||||
|
(uint32) (tl_beginptr >> 32), (uint32) tl_beginptr,
|
||||||
|
(uint32) (endptr >> 32), (uint32) endptr);
|
||||||
|
|
||||||
|
if (starttli == entry->tli)
|
||||||
|
{
|
||||||
|
found_start_timeline = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
endptr = entry->begin;
|
||||||
|
first_wal_range = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The last entry in the timeline history for the ending timeline should
|
||||||
|
* be the ending timeline itself. Verify that this is what we observed.
|
||||||
|
*/
|
||||||
|
if (!found_start_timeline)
|
||||||
|
ereport(ERROR,
|
||||||
|
errmsg("start timeline %u not found history of timeline %u",
|
||||||
|
starttli, endtli));
|
||||||
|
|
||||||
|
/* Terminate the list of WAL ranges. */
|
||||||
|
AppendStringToManifest(manifest, "\n],\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Finalize the backup manifest, and send it to the client.
|
||||||
|
*/
|
||||||
|
void
|
||||||
|
SendBackupManifest(manifest_info *manifest)
|
||||||
|
{
|
||||||
|
StringInfoData protobuf;
|
||||||
|
uint8 checksumbuf[PG_SHA256_DIGEST_LENGTH];
|
||||||
|
char checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH];
|
||||||
|
size_t manifest_bytes_done = 0;
|
||||||
|
|
||||||
|
if (!IsManifestEnabled(manifest))
|
||||||
|
return;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Append manifest checksum, so that the problems with the manifest itself
|
||||||
|
* can be detected.
|
||||||
|
*
|
||||||
|
* We always use SHA-256 for this, regardless of what algorithm is chosen
|
||||||
|
* for checksumming the files. If we ever want to make the checksum
|
||||||
|
* algorithm used for the manifest file variable, the client will need a
|
||||||
|
* way to figure out which algorithm to use as close to the beginning of
|
||||||
|
* the manifest file as possible, to avoid having to read the whole thing
|
||||||
|
* twice.
|
||||||
|
*/
|
||||||
|
manifest->still_checksumming = false;
|
||||||
|
pg_sha256_final(&manifest->manifest_ctx, checksumbuf);
|
||||||
|
AppendStringToManifest(manifest, "\"Manifest-Checksum\": \"");
|
||||||
|
hex_encode((char *) checksumbuf, sizeof checksumbuf, checksumstringbuf);
|
||||||
|
checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH - 1] = '\0';
|
||||||
|
AppendStringToManifest(manifest, checksumstringbuf);
|
||||||
|
AppendStringToManifest(manifest, "\"}\n");
|
||||||
|
|
||||||
|
/*
|
||||||
|
* We've written all the data to the manifest file. Rewind the file so
|
||||||
|
* that we can read it all back.
|
||||||
|
*/
|
||||||
|
if (BufFileSeek(manifest->buffile, 0, 0L, SEEK_SET))
|
||||||
|
ereport(ERROR,
|
||||||
|
(errcode_for_file_access(),
|
||||||
|
errmsg("could not rewind temporary file: %m")));
|
||||||
|
|
||||||
|
/* Send CopyOutResponse message */
|
||||||
|
pq_beginmessage(&protobuf, 'H');
|
||||||
|
pq_sendbyte(&protobuf, 0); /* overall format */
|
||||||
|
pq_sendint16(&protobuf, 0); /* natts */
|
||||||
|
pq_endmessage(&protobuf);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Send CopyData messages.
|
||||||
|
*
|
||||||
|
* We choose to read back the data from the temporary file in chunks of
|
||||||
|
* size BLCKSZ; this isn't necessary, but buffile.c uses that as the I/O
|
||||||
|
* size, so it seems to make sense to match that value here.
|
||||||
|
*/
|
||||||
|
while (manifest_bytes_done < manifest->manifest_size)
|
||||||
|
{
|
||||||
|
char manifestbuf[BLCKSZ];
|
||||||
|
size_t bytes_to_read;
|
||||||
|
size_t rc;
|
||||||
|
|
||||||
|
bytes_to_read = Min(sizeof(manifestbuf),
|
||||||
|
manifest->manifest_size - manifest_bytes_done);
|
||||||
|
rc = BufFileRead(manifest->buffile, manifestbuf, bytes_to_read);
|
||||||
|
if (rc != bytes_to_read)
|
||||||
|
ereport(ERROR,
|
||||||
|
(errcode_for_file_access(),
|
||||||
|
errmsg("could not read from temporary file: %m")));
|
||||||
|
pq_putmessage('d', manifestbuf, bytes_to_read);
|
||||||
|
manifest_bytes_done += bytes_to_read;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* No more data, so send CopyDone message */
|
||||||
|
pq_putemptymessage('c');
|
||||||
|
|
||||||
|
/* Release resources */
|
||||||
|
BufFileClose(manifest->buffile);
|
||||||
|
}
|
||||||
@ -16,10 +16,8 @@
|
|||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
|
|
||||||
#include "access/timeline.h"
|
|
||||||
#include "access/xlog_internal.h" /* for pg_start/stop_backup */
|
#include "access/xlog_internal.h" /* for pg_start/stop_backup */
|
||||||
#include "catalog/pg_type.h"
|
#include "catalog/pg_type.h"
|
||||||
#include "common/checksum_helper.h"
|
|
||||||
#include "common/file_perm.h"
|
#include "common/file_perm.h"
|
||||||
#include "commands/progress.h"
|
#include "commands/progress.h"
|
||||||
#include "lib/stringinfo.h"
|
#include "lib/stringinfo.h"
|
||||||
@ -32,9 +30,9 @@
|
|||||||
#include "port.h"
|
#include "port.h"
|
||||||
#include "postmaster/syslogger.h"
|
#include "postmaster/syslogger.h"
|
||||||
#include "replication/basebackup.h"
|
#include "replication/basebackup.h"
|
||||||
|
#include "replication/backup_manifest.h"
|
||||||
#include "replication/walsender.h"
|
#include "replication/walsender.h"
|
||||||
#include "replication/walsender_private.h"
|
#include "replication/walsender_private.h"
|
||||||
#include "storage/buffile.h"
|
|
||||||
#include "storage/bufpage.h"
|
#include "storage/bufpage.h"
|
||||||
#include "storage/checksum.h"
|
#include "storage/checksum.h"
|
||||||
#include "storage/dsm_impl.h"
|
#include "storage/dsm_impl.h"
|
||||||
@ -42,19 +40,11 @@
|
|||||||
#include "storage/ipc.h"
|
#include "storage/ipc.h"
|
||||||
#include "storage/reinit.h"
|
#include "storage/reinit.h"
|
||||||
#include "utils/builtins.h"
|
#include "utils/builtins.h"
|
||||||
#include "utils/json.h"
|
|
||||||
#include "utils/ps_status.h"
|
#include "utils/ps_status.h"
|
||||||
#include "utils/relcache.h"
|
#include "utils/relcache.h"
|
||||||
#include "utils/resowner.h"
|
#include "utils/resowner.h"
|
||||||
#include "utils/timestamp.h"
|
#include "utils/timestamp.h"
|
||||||
|
|
||||||
typedef enum manifest_option
|
|
||||||
{
|
|
||||||
MANIFEST_OPTION_YES,
|
|
||||||
MANIFEST_OPTION_NO,
|
|
||||||
MANIFEST_OPTION_FORCE_ENCODE
|
|
||||||
} manifest_option;
|
|
||||||
|
|
||||||
typedef struct
|
typedef struct
|
||||||
{
|
{
|
||||||
const char *label;
|
const char *label;
|
||||||
@ -68,18 +58,6 @@ typedef struct
|
|||||||
pg_checksum_type manifest_checksum_type;
|
pg_checksum_type manifest_checksum_type;
|
||||||
} basebackup_options;
|
} basebackup_options;
|
||||||
|
|
||||||
struct manifest_info
|
|
||||||
{
|
|
||||||
BufFile *buffile;
|
|
||||||
pg_checksum_type checksum_type;
|
|
||||||
pg_sha256_ctx manifest_ctx;
|
|
||||||
uint64 manifest_size;
|
|
||||||
bool force_encode;
|
|
||||||
bool first_file;
|
|
||||||
bool still_checksumming;
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
static int64 sendDir(const char *path, int basepathlen, bool sizeonly,
|
static int64 sendDir(const char *path, int basepathlen, bool sizeonly,
|
||||||
List *tablespaces, bool sendtblspclinks,
|
List *tablespaces, bool sendtblspclinks,
|
||||||
manifest_info *manifest, const char *spcoid);
|
manifest_info *manifest, const char *spcoid);
|
||||||
@ -94,18 +72,6 @@ static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *sta
|
|||||||
bool sizeonly);
|
bool sizeonly);
|
||||||
static void send_int8_string(StringInfoData *buf, int64 intval);
|
static void send_int8_string(StringInfoData *buf, int64 intval);
|
||||||
static void SendBackupHeader(List *tablespaces);
|
static void SendBackupHeader(List *tablespaces);
|
||||||
static bool IsManifestEnabled(manifest_info *manifest);
|
|
||||||
static void InitializeManifest(manifest_info *manifest,
|
|
||||||
basebackup_options *opt);
|
|
||||||
static void AppendStringToManifest(manifest_info *manifest, char *s);
|
|
||||||
static void AddFileToManifest(manifest_info *manifest, const char *spcoid,
|
|
||||||
const char *pathname, size_t size,
|
|
||||||
pg_time_t mtime,
|
|
||||||
pg_checksum_context *checksum_ctx);
|
|
||||||
static void AddWALInfoToManifest(manifest_info *manifest, XLogRecPtr startptr,
|
|
||||||
TimeLineID starttli, XLogRecPtr endptr,
|
|
||||||
TimeLineID endtli);
|
|
||||||
static void SendBackupManifest(manifest_info *manifest);
|
|
||||||
static void perform_base_backup(basebackup_options *opt);
|
static void perform_base_backup(basebackup_options *opt);
|
||||||
static void parse_basebackup_options(List *options, basebackup_options *opt);
|
static void parse_basebackup_options(List *options, basebackup_options *opt);
|
||||||
static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli);
|
static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli);
|
||||||
@ -142,16 +108,6 @@ do { \
|
|||||||
(errmsg("could not read from file \"%s\"", filename))); \
|
(errmsg("could not read from file \"%s\"", filename))); \
|
||||||
} while (0)
|
} while (0)
|
||||||
|
|
||||||
/*
|
|
||||||
* Convenience macro for appending data to the backup manifest.
|
|
||||||
*/
|
|
||||||
#define AppendToManifest(manifest, ...) \
|
|
||||||
{ \
|
|
||||||
char *_manifest_s = psprintf(__VA_ARGS__); \
|
|
||||||
AppendStringToManifest(manifest, _manifest_s); \
|
|
||||||
pfree(_manifest_s); \
|
|
||||||
}
|
|
||||||
|
|
||||||
/* The actual number of bytes, transfer of which may cause sleep. */
|
/* The actual number of bytes, transfer of which may cause sleep. */
|
||||||
static uint64 throttling_sample;
|
static uint64 throttling_sample;
|
||||||
|
|
||||||
@ -342,7 +298,7 @@ perform_base_backup(basebackup_options *opt)
|
|||||||
|
|
||||||
labelfile = makeStringInfo();
|
labelfile = makeStringInfo();
|
||||||
tblspc_map_file = makeStringInfo();
|
tblspc_map_file = makeStringInfo();
|
||||||
InitializeManifest(&manifest, opt);
|
InitializeManifest(&manifest, opt->manifest, opt->manifest_checksum_type);
|
||||||
|
|
||||||
total_checksum_failures = 0;
|
total_checksum_failures = 0;
|
||||||
|
|
||||||
@ -1068,349 +1024,6 @@ SendBackupHeader(List *tablespaces)
|
|||||||
pq_puttextmessage('C', "SELECT");
|
pq_puttextmessage('C', "SELECT");
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
* Does the user want a backup manifest?
|
|
||||||
*
|
|
||||||
* It's simplest to always have a manifest_info object, so that we don't need
|
|
||||||
* checks for NULL pointers in too many places. However, if the user doesn't
|
|
||||||
* want a manifest, we set manifest->buffile to NULL.
|
|
||||||
*/
|
|
||||||
static bool
|
|
||||||
IsManifestEnabled(manifest_info *manifest)
|
|
||||||
{
|
|
||||||
return (manifest->buffile != NULL);
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Initialize state so that we can construct a backup manifest.
|
|
||||||
*
|
|
||||||
* NB: Although the checksum type for the data files is configurable, the
|
|
||||||
* checksum for the manifest itself always uses SHA-256. See comments in
|
|
||||||
* SendBackupManifest.
|
|
||||||
*/
|
|
||||||
static void
|
|
||||||
InitializeManifest(manifest_info *manifest, basebackup_options *opt)
|
|
||||||
{
|
|
||||||
if (opt->manifest == MANIFEST_OPTION_NO)
|
|
||||||
manifest->buffile = NULL;
|
|
||||||
else
|
|
||||||
manifest->buffile = BufFileCreateTemp(false);
|
|
||||||
manifest->checksum_type = opt->manifest_checksum_type;
|
|
||||||
pg_sha256_init(&manifest->manifest_ctx);
|
|
||||||
manifest->manifest_size = UINT64CONST(0);
|
|
||||||
manifest->force_encode = (opt->manifest == MANIFEST_OPTION_FORCE_ENCODE);
|
|
||||||
manifest->first_file = true;
|
|
||||||
manifest->still_checksumming = true;
|
|
||||||
|
|
||||||
if (opt->manifest != MANIFEST_OPTION_NO)
|
|
||||||
AppendToManifest(manifest,
|
|
||||||
"{ \"PostgreSQL-Backup-Manifest-Version\": 1,\n"
|
|
||||||
"\"Files\": [");
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Append a cstring to the manifest.
|
|
||||||
*/
|
|
||||||
static void
|
|
||||||
AppendStringToManifest(manifest_info *manifest, char *s)
|
|
||||||
{
|
|
||||||
int len = strlen(s);
|
|
||||||
size_t written;
|
|
||||||
|
|
||||||
Assert(manifest != NULL);
|
|
||||||
if (manifest->still_checksumming)
|
|
||||||
pg_sha256_update(&manifest->manifest_ctx, (uint8 *) s, len);
|
|
||||||
written = BufFileWrite(manifest->buffile, s, len);
|
|
||||||
if (written != len)
|
|
||||||
ereport(ERROR,
|
|
||||||
(errcode_for_file_access(),
|
|
||||||
errmsg("could not write to temporary file: %m")));
|
|
||||||
manifest->manifest_size += len;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Add an entry to the backup manifest for a file.
|
|
||||||
*/
|
|
||||||
static void
|
|
||||||
AddFileToManifest(manifest_info *manifest, const char *spcoid,
|
|
||||||
const char *pathname, size_t size, pg_time_t mtime,
|
|
||||||
pg_checksum_context *checksum_ctx)
|
|
||||||
{
|
|
||||||
char pathbuf[MAXPGPATH];
|
|
||||||
int pathlen;
|
|
||||||
StringInfoData buf;
|
|
||||||
|
|
||||||
if (!IsManifestEnabled(manifest))
|
|
||||||
return;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* If this file is part of a tablespace, the pathname passed to this
|
|
||||||
* function will be relative to the tar file that contains it. We want the
|
|
||||||
* pathname relative to the data directory (ignoring the intermediate
|
|
||||||
* symlink traversal).
|
|
||||||
*/
|
|
||||||
if (spcoid != NULL)
|
|
||||||
{
|
|
||||||
snprintf(pathbuf, sizeof(pathbuf), "pg_tblspc/%s/%s", spcoid,
|
|
||||||
pathname);
|
|
||||||
pathname = pathbuf;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Each file's entry needs to be separated from any entry that follows by a
|
|
||||||
* comma, but there's no comma before the first one or after the last one.
|
|
||||||
* To make that work, adding a file to the manifest starts by terminating
|
|
||||||
* the most recently added line, with a comma if appropriate, but does not
|
|
||||||
* terminate the line inserted for this file.
|
|
||||||
*/
|
|
||||||
initStringInfo(&buf);
|
|
||||||
if (manifest->first_file)
|
|
||||||
{
|
|
||||||
appendStringInfoString(&buf, "\n");
|
|
||||||
manifest->first_file = false;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
appendStringInfoString(&buf, ",\n");
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Write the relative pathname to this file out to the manifest. The
|
|
||||||
* manifest is always stored in UTF-8, so we have to encode paths that are
|
|
||||||
* not valid in that encoding.
|
|
||||||
*/
|
|
||||||
pathlen = strlen(pathname);
|
|
||||||
if (!manifest->force_encode &&
|
|
||||||
pg_verify_mbstr(PG_UTF8, pathname, pathlen, true))
|
|
||||||
{
|
|
||||||
appendStringInfoString(&buf, "{ \"Path\": ");
|
|
||||||
escape_json(&buf, pathname);
|
|
||||||
appendStringInfoString(&buf, ", ");
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
appendStringInfoString(&buf, "{ \"Encoded-Path\": \"");
|
|
||||||
enlargeStringInfo(&buf, 2 * pathlen);
|
|
||||||
buf.len += hex_encode((char *) pathname, pathlen,
|
|
||||||
&buf.data[buf.len]);
|
|
||||||
appendStringInfoString(&buf, "\", ");
|
|
||||||
}
|
|
||||||
|
|
||||||
appendStringInfo(&buf, "\"Size\": %zu, ", size);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Convert last modification time to a string and append it to the
|
|
||||||
* manifest. Since it's not clear what time zone to use and since time
|
|
||||||
* zone definitions can change, possibly causing confusion, use GMT
|
|
||||||
* always.
|
|
||||||
*/
|
|
||||||
appendStringInfoString(&buf, "\"Last-Modified\": \"");
|
|
||||||
enlargeStringInfo(&buf, 128);
|
|
||||||
buf.len += pg_strftime(&buf.data[buf.len], 128, "%Y-%m-%d %H:%M:%S %Z",
|
|
||||||
pg_gmtime(&mtime));
|
|
||||||
appendStringInfoString(&buf, "\"");
|
|
||||||
|
|
||||||
/* Add checksum information. */
|
|
||||||
if (checksum_ctx->type != CHECKSUM_TYPE_NONE)
|
|
||||||
{
|
|
||||||
uint8 checksumbuf[PG_CHECKSUM_MAX_LENGTH];
|
|
||||||
int checksumlen;
|
|
||||||
|
|
||||||
checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
|
|
||||||
|
|
||||||
appendStringInfo(&buf,
|
|
||||||
", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"",
|
|
||||||
pg_checksum_type_name(checksum_ctx->type));
|
|
||||||
enlargeStringInfo(&buf, 2 * checksumlen);
|
|
||||||
buf.len += hex_encode((char *) checksumbuf, checksumlen,
|
|
||||||
&buf.data[buf.len]);
|
|
||||||
appendStringInfoString(&buf, "\"");
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Close out the object. */
|
|
||||||
appendStringInfoString(&buf, " }");
|
|
||||||
|
|
||||||
/* OK, add it to the manifest. */
|
|
||||||
AppendStringToManifest(manifest, buf.data);
|
|
||||||
|
|
||||||
/* Avoid leaking memory. */
|
|
||||||
pfree(buf.data);
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Add information about the WAL that will need to be replayed when restoring
|
|
||||||
* this backup to the manifest.
|
|
||||||
*/
|
|
||||||
static void
|
|
||||||
AddWALInfoToManifest(manifest_info *manifest, XLogRecPtr startptr,
|
|
||||||
TimeLineID starttli, XLogRecPtr endptr, TimeLineID endtli)
|
|
||||||
{
|
|
||||||
List *timelines;
|
|
||||||
ListCell *lc;
|
|
||||||
bool first_wal_range = true;
|
|
||||||
bool found_start_timeline = false;
|
|
||||||
|
|
||||||
if (!IsManifestEnabled(manifest))
|
|
||||||
return;
|
|
||||||
|
|
||||||
/* Terminate the list of files. */
|
|
||||||
AppendStringToManifest(manifest, "\n],\n");
|
|
||||||
|
|
||||||
/* Read the timeline history for the ending timeline. */
|
|
||||||
timelines = readTimeLineHistory(endtli);
|
|
||||||
|
|
||||||
/* Start a list of LSN ranges. */
|
|
||||||
AppendStringToManifest(manifest, "\"WAL-Ranges\": [\n");
|
|
||||||
|
|
||||||
foreach(lc, timelines)
|
|
||||||
{
|
|
||||||
TimeLineHistoryEntry *entry = lfirst(lc);
|
|
||||||
XLogRecPtr tl_beginptr;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* We only care about timelines that were active during the backup.
|
|
||||||
* Skip any that ended before the backup started. (Note that if
|
|
||||||
* entry->end is InvalidXLogRecPtr, it means that the timeline has not
|
|
||||||
* yet ended.)
|
|
||||||
*/
|
|
||||||
if (!XLogRecPtrIsInvalid(entry->end) && entry->end < startptr)
|
|
||||||
continue;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Because the timeline history file lists newer timelines before
|
|
||||||
* older ones, the first timeline we encounter that is new enough to
|
|
||||||
* matter ought to match the ending timeline of the backup.
|
|
||||||
*/
|
|
||||||
if (first_wal_range && endtli != entry->tli)
|
|
||||||
ereport(ERROR,
|
|
||||||
errmsg("expected end timeline %u but found timeline %u",
|
|
||||||
starttli, entry->tli));
|
|
||||||
|
|
||||||
if (!XLogRecPtrIsInvalid(entry->begin))
|
|
||||||
tl_beginptr = entry->begin;
|
|
||||||
else
|
|
||||||
{
|
|
||||||
tl_beginptr = startptr;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* If we reach a TLI that has no valid beginning LSN, there can't
|
|
||||||
* be any more timelines in the history after this point, so we'd
|
|
||||||
* better have arrived at the expected starting TLI. If not,
|
|
||||||
* something's gone horribly wrong.
|
|
||||||
*/
|
|
||||||
if (starttli != entry->tli)
|
|
||||||
ereport(ERROR,
|
|
||||||
errmsg("expected start timeline %u but found timeline %u",
|
|
||||||
starttli, entry->tli));
|
|
||||||
}
|
|
||||||
|
|
||||||
AppendToManifest(manifest,
|
|
||||||
"%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }",
|
|
||||||
first_wal_range ? "" : ",\n",
|
|
||||||
entry->tli,
|
|
||||||
(uint32) (tl_beginptr >> 32), (uint32) tl_beginptr,
|
|
||||||
(uint32) (endptr >> 32), (uint32) endptr);
|
|
||||||
|
|
||||||
if (starttli == entry->tli)
|
|
||||||
{
|
|
||||||
found_start_timeline = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
endptr = entry->begin;
|
|
||||||
first_wal_range = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* The last entry in the timeline history for the ending timeline should
|
|
||||||
* be the ending timeline itself. Verify that this is what we observed.
|
|
||||||
*/
|
|
||||||
if (!found_start_timeline)
|
|
||||||
ereport(ERROR,
|
|
||||||
errmsg("start timeline %u not found history of timeline %u",
|
|
||||||
starttli, endtli));
|
|
||||||
|
|
||||||
/* Terminate the list of WAL ranges. */
|
|
||||||
AppendStringToManifest(manifest, "\n],\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Finalize the backup manifest, and send it to the client.
|
|
||||||
*/
|
|
||||||
static void
|
|
||||||
SendBackupManifest(manifest_info *manifest)
|
|
||||||
{
|
|
||||||
StringInfoData protobuf;
|
|
||||||
uint8 checksumbuf[PG_SHA256_DIGEST_LENGTH];
|
|
||||||
char checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH];
|
|
||||||
size_t manifest_bytes_done = 0;
|
|
||||||
|
|
||||||
if (!IsManifestEnabled(manifest))
|
|
||||||
return;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Append manifest checksum, so that the problems with the manifest itself
|
|
||||||
* can be detected.
|
|
||||||
*
|
|
||||||
* We always use SHA-256 for this, regardless of what algorithm is chosen
|
|
||||||
* for checksumming the files. If we ever want to make the checksum
|
|
||||||
* algorithm used for the manifest file variable, the client will need a
|
|
||||||
* way to figure out which algorithm to use as close to the beginning of
|
|
||||||
* the manifest file as possible, to avoid having to read the whole thing
|
|
||||||
* twice.
|
|
||||||
*/
|
|
||||||
manifest->still_checksumming = false;
|
|
||||||
pg_sha256_final(&manifest->manifest_ctx, checksumbuf);
|
|
||||||
AppendStringToManifest(manifest, "\"Manifest-Checksum\": \"");
|
|
||||||
hex_encode((char *) checksumbuf, sizeof checksumbuf, checksumstringbuf);
|
|
||||||
checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH - 1] = '\0';
|
|
||||||
AppendStringToManifest(manifest, checksumstringbuf);
|
|
||||||
AppendStringToManifest(manifest, "\"}\n");
|
|
||||||
|
|
||||||
/*
|
|
||||||
* We've written all the data to the manifest file. Rewind the file so
|
|
||||||
* that we can read it all back.
|
|
||||||
*/
|
|
||||||
if (BufFileSeek(manifest->buffile, 0, 0L, SEEK_SET))
|
|
||||||
ereport(ERROR,
|
|
||||||
(errcode_for_file_access(),
|
|
||||||
errmsg("could not rewind temporary file: %m")));
|
|
||||||
|
|
||||||
/* Send CopyOutResponse message */
|
|
||||||
pq_beginmessage(&protobuf, 'H');
|
|
||||||
pq_sendbyte(&protobuf, 0); /* overall format */
|
|
||||||
pq_sendint16(&protobuf, 0); /* natts */
|
|
||||||
pq_endmessage(&protobuf);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Send CopyData messages.
|
|
||||||
*
|
|
||||||
* We choose to read back the data from the temporary file in chunks of
|
|
||||||
* size BLCKSZ; this isn't necessary, but buffile.c uses that as the I/O
|
|
||||||
* size, so it seems to make sense to match that value here.
|
|
||||||
*/
|
|
||||||
while (manifest_bytes_done < manifest->manifest_size)
|
|
||||||
{
|
|
||||||
char manifestbuf[BLCKSZ];
|
|
||||||
size_t bytes_to_read;
|
|
||||||
size_t rc;
|
|
||||||
|
|
||||||
bytes_to_read = Min(sizeof(manifestbuf),
|
|
||||||
manifest->manifest_size - manifest_bytes_done);
|
|
||||||
rc = BufFileRead(manifest->buffile, manifestbuf, bytes_to_read);
|
|
||||||
if (rc != bytes_to_read)
|
|
||||||
ereport(ERROR,
|
|
||||||
(errcode_for_file_access(),
|
|
||||||
errmsg("could not read from temporary file: %m")));
|
|
||||||
pq_putmessage('d', manifestbuf, bytes_to_read);
|
|
||||||
manifest_bytes_done += bytes_to_read;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* No more data, so send CopyDone message */
|
|
||||||
pq_putemptymessage('c');
|
|
||||||
|
|
||||||
/* Release resources */
|
|
||||||
BufFileClose(manifest->buffile);
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Send a single resultset containing just a single
|
* Send a single resultset containing just a single
|
||||||
* XLogRecPtr record (in text format)
|
* XLogRecPtr record (in text format)
|
||||||
|
|||||||
51
src/include/replication/backup_manifest.h
Normal file
51
src/include/replication/backup_manifest.h
Normal file
@ -0,0 +1,51 @@
|
|||||||
|
/*-------------------------------------------------------------------------
|
||||||
|
*
|
||||||
|
* backup_manifest.h
|
||||||
|
* Routines for generating a backup manifest.
|
||||||
|
*
|
||||||
|
* Portions Copyright (c) 2010-2020, PostgreSQL Global Development Group
|
||||||
|
*
|
||||||
|
* src/include/replication/backup_manifest.h
|
||||||
|
*
|
||||||
|
*-------------------------------------------------------------------------
|
||||||
|
*/
|
||||||
|
#ifndef BACKUP_MANIFEST_H
|
||||||
|
#define BACKUP_MANIFEST_H
|
||||||
|
|
||||||
|
#include "access/xlogdefs.h"
|
||||||
|
#include "common/checksum_helper.h"
|
||||||
|
#include "pgtime.h"
|
||||||
|
#include "storage/buffile.h"
|
||||||
|
|
||||||
|
typedef enum manifest_option
|
||||||
|
{
|
||||||
|
MANIFEST_OPTION_YES,
|
||||||
|
MANIFEST_OPTION_NO,
|
||||||
|
MANIFEST_OPTION_FORCE_ENCODE
|
||||||
|
} manifest_option;
|
||||||
|
|
||||||
|
typedef struct manifest_info
|
||||||
|
{
|
||||||
|
BufFile *buffile;
|
||||||
|
pg_checksum_type checksum_type;
|
||||||
|
pg_sha256_ctx manifest_ctx;
|
||||||
|
uint64 manifest_size;
|
||||||
|
bool force_encode;
|
||||||
|
bool first_file;
|
||||||
|
bool still_checksumming;
|
||||||
|
} manifest_info;
|
||||||
|
|
||||||
|
extern void InitializeManifest(manifest_info *manifest,
|
||||||
|
manifest_option want_manifest,
|
||||||
|
pg_checksum_type manifest_checksum_type);
|
||||||
|
extern void AppendStringToManifest(manifest_info *manifest, char *s);
|
||||||
|
extern void AddFileToManifest(manifest_info *manifest, const char *spcoid,
|
||||||
|
const char *pathname, size_t size,
|
||||||
|
pg_time_t mtime,
|
||||||
|
pg_checksum_context *checksum_ctx);
|
||||||
|
extern void AddWALInfoToManifest(manifest_info *manifest, XLogRecPtr startptr,
|
||||||
|
TimeLineID starttli, XLogRecPtr endptr,
|
||||||
|
TimeLineID endtli);
|
||||||
|
extern void SendBackupManifest(manifest_info *manifest);
|
||||||
|
|
||||||
|
#endif
|
||||||
Loading…
x
Reference in New Issue
Block a user