mirrors/postgres
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move the server's backup manifest code to a separate file.

Browse Source 
basebackup.c is already a pretty big and complicated file, so it
makes more sense to keep the backup manifest support routines
in a separate file, for clarity and ease of maintenance.

Discussion: http://postgr.es/m/CA+TgmoavRak5OdP76P8eJExDYhPEKWjMb0sxW7dF01dWFgE=uA@mail.gmail.com
This commit is contained in:
Robert Haas 2020-04-20 14:37:38 -04:00
parent 1e324cb0e7
commit 079ac29d4d
4 changed files with 429 additions and 389 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/backend/replication/Makefile
View File

@ -15,6 +15,7 @@ include $(top_builddir)/src/Makefile.global
override CPPFLAGS := -I. -I$(srcdir) $(CPPFLAGS)
OBJS = \
backup_manifest.o \
basebackup.o \
repl_gram.o \
slot.o \

375
src/backend/replication/backup_manifest.c Normal file
View File

@ -0,0 +1,375 @@
/*-------------------------------------------------------------------------
*
* backup_manifest.c
* code for generating and sending a backup manifest
*
* Portions Copyright (c) 2010-2020, PostgreSQL Global Development Group
*
* IDENTIFICATION
* src/backend/replication/backup_manifest.c
*
*-------------------------------------------------------------------------
*/
#include "postgres.h"
#include "access/timeline.h"
#include "libpq/libpq.h"
#include "libpq/pqformat.h"
#include "mb/pg_wchar.h"
#include "replication/backup_manifest.h"
#include "utils/builtins.h"
#include "utils/json.h"
/*
* Does the user want a backup manifest?
*
* It's simplest to always have a manifest_info object, so that we don't need
* checks for NULL pointers in too many places. However, if the user doesn't
* want a manifest, we set manifest->buffile to NULL.
*/
static inline bool
IsManifestEnabled(manifest_info *manifest)
{
return (manifest->buffile != NULL);
}
/*
* Convenience macro for appending data to the backup manifest.
*/
#define AppendToManifest(manifest, ...) \
{ \
char *_manifest_s = psprintf(__VA_ARGS__); \
AppendStringToManifest(manifest, _manifest_s); \
pfree(_manifest_s); \
}
/*
* Initialize state so that we can construct a backup manifest.
*
* NB: Although the checksum type for the data files is configurable, the
* checksum for the manifest itself always uses SHA-256. See comments in
* SendBackupManifest.
*/
void
InitializeManifest(manifest_info *manifest, manifest_option want_manifest,
pg_checksum_type manifest_checksum_type)
{
if (want_manifest == MANIFEST_OPTION_NO)
manifest->buffile = NULL;
else
manifest->buffile = BufFileCreateTemp(false);
manifest->checksum_type = manifest_checksum_type;
pg_sha256_init(&manifest->manifest_ctx);
manifest->manifest_size = UINT64CONST(0);
manifest->force_encode = (want_manifest == MANIFEST_OPTION_FORCE_ENCODE);
manifest->first_file = true;
manifest->still_checksumming = true;
if (want_manifest != MANIFEST_OPTION_NO)
AppendToManifest(manifest,
"{ \"PostgreSQL-Backup-Manifest-Version\": 1,\n"
"\"Files\": [");
}
/*
* Append a cstring to the manifest.
*/
void
AppendStringToManifest(manifest_info *manifest, char *s)
{
int len = strlen(s);
size_t written;
Assert(manifest != NULL);
if (manifest->still_checksumming)
pg_sha256_update(&manifest->manifest_ctx, (uint8 *) s, len);
written = BufFileWrite(manifest->buffile, s, len);
if (written != len)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not write to temporary file: %m")));
manifest->manifest_size += len;
}
/*
* Add an entry to the backup manifest for a file.
*/
void
AddFileToManifest(manifest_info *manifest, const char *spcoid,
const char *pathname, size_t size, pg_time_t mtime,
pg_checksum_context *checksum_ctx)
{
char pathbuf[MAXPGPATH];
int pathlen;
StringInfoData buf;
if (!IsManifestEnabled(manifest))
return;
/*
* If this file is part of a tablespace, the pathname passed to this
* function will be relative to the tar file that contains it. We want the
* pathname relative to the data directory (ignoring the intermediate
* symlink traversal).
*/
if (spcoid != NULL)
{
snprintf(pathbuf, sizeof(pathbuf), "pg_tblspc/%s/%s", spcoid,
pathname);
pathname = pathbuf;
}
/*
* Each file's entry needs to be separated from any entry that follows by a
* comma, but there's no comma before the first one or after the last one.
* To make that work, adding a file to the manifest starts by terminating
* the most recently added line, with a comma if appropriate, but does not
* terminate the line inserted for this file.
*/
initStringInfo(&buf);
if (manifest->first_file)
{
appendStringInfoString(&buf, "\n");
manifest->first_file = false;
}
else
appendStringInfoString(&buf, ",\n");
/*
* Write the relative pathname to this file out to the manifest. The
* manifest is always stored in UTF-8, so we have to encode paths that are
* not valid in that encoding.
*/
pathlen = strlen(pathname);
if (!manifest->force_encode &&
pg_verify_mbstr(PG_UTF8, pathname, pathlen, true))
{
appendStringInfoString(&buf, "{ \"Path\": ");
escape_json(&buf, pathname);
appendStringInfoString(&buf, ", ");
}
else
{
appendStringInfoString(&buf, "{ \"Encoded-Path\": \"");
enlargeStringInfo(&buf, 2 * pathlen);
buf.len += hex_encode((char *) pathname, pathlen,
&buf.data[buf.len]);
appendStringInfoString(&buf, "\", ");
}
appendStringInfo(&buf, "\"Size\": %zu, ", size);
/*
* Convert last modification time to a string and append it to the
* manifest. Since it's not clear what time zone to use and since time
* zone definitions can change, possibly causing confusion, use GMT
* always.
*/
appendStringInfoString(&buf, "\"Last-Modified\": \"");
enlargeStringInfo(&buf, 128);
buf.len += pg_strftime(&buf.data[buf.len], 128, "%Y-%m-%d %H:%M:%S %Z",
pg_gmtime(&mtime));
appendStringInfoString(&buf, "\"");
/* Add checksum information. */
if (checksum_ctx->type != CHECKSUM_TYPE_NONE)
{
uint8 checksumbuf[PG_CHECKSUM_MAX_LENGTH];
int checksumlen;
checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
appendStringInfo(&buf,
", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"",
pg_checksum_type_name(checksum_ctx->type));
enlargeStringInfo(&buf, 2 * checksumlen);
buf.len += hex_encode((char *) checksumbuf, checksumlen,
&buf.data[buf.len]);
appendStringInfoString(&buf, "\"");
}
/* Close out the object. */
appendStringInfoString(&buf, " }");
/* OK, add it to the manifest. */
AppendStringToManifest(manifest, buf.data);
/* Avoid leaking memory. */
pfree(buf.data);
}
/*
* Add information about the WAL that will need to be replayed when restoring
* this backup to the manifest.
*/
void
AddWALInfoToManifest(manifest_info *manifest, XLogRecPtr startptr,
TimeLineID starttli, XLogRecPtr endptr, TimeLineID endtli)
{
List *timelines;
ListCell *lc;
bool first_wal_range = true;
bool found_start_timeline = false;
if (!IsManifestEnabled(manifest))
return;
/* Terminate the list of files. */
AppendStringToManifest(manifest, "\n],\n");
/* Read the timeline history for the ending timeline. */
timelines = readTimeLineHistory(endtli);
/* Start a list of LSN ranges. */
AppendStringToManifest(manifest, "\"WAL-Ranges\": [\n");
foreach(lc, timelines)
{
TimeLineHistoryEntry *entry = lfirst(lc);
XLogRecPtr tl_beginptr;
/*
* We only care about timelines that were active during the backup.
* Skip any that ended before the backup started. (Note that if
* entry->end is InvalidXLogRecPtr, it means that the timeline has not
* yet ended.)
*/
if (!XLogRecPtrIsInvalid(entry->end) && entry->end < startptr)
continue;
/*
* Because the timeline history file lists newer timelines before
* older ones, the first timeline we encounter that is new enough to
* matter ought to match the ending timeline of the backup.
*/
if (first_wal_range && endtli != entry->tli)
ereport(ERROR,
errmsg("expected end timeline %u but found timeline %u",
starttli, entry->tli));
if (!XLogRecPtrIsInvalid(entry->begin))
tl_beginptr = entry->begin;
else
{
tl_beginptr = startptr;
/*
* If we reach a TLI that has no valid beginning LSN, there can't
* be any more timelines in the history after this point, so we'd
* better have arrived at the expected starting TLI. If not,
* something's gone horribly wrong.
*/
if (starttli != entry->tli)
ereport(ERROR,
errmsg("expected start timeline %u but found timeline %u",
starttli, entry->tli));
}
AppendToManifest(manifest,
"%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }",
first_wal_range ? "" : ",\n",
entry->tli,
(uint32) (tl_beginptr >> 32), (uint32) tl_beginptr,
(uint32) (endptr >> 32), (uint32) endptr);
if (starttli == entry->tli)
{
found_start_timeline = true;
break;
}
endptr = entry->begin;
first_wal_range = false;
}
/*
* The last entry in the timeline history for the ending timeline should
* be the ending timeline itself. Verify that this is what we observed.
*/
if (!found_start_timeline)
ereport(ERROR,
errmsg("start timeline %u not found history of timeline %u",
starttli, endtli));
/* Terminate the list of WAL ranges. */
AppendStringToManifest(manifest, "\n],\n");
}
/*
* Finalize the backup manifest, and send it to the client.
*/
void
SendBackupManifest(manifest_info *manifest)
{
StringInfoData protobuf;
uint8 checksumbuf[PG_SHA256_DIGEST_LENGTH];
char checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH];
size_t manifest_bytes_done = 0;
if (!IsManifestEnabled(manifest))
return;
/*
* Append manifest checksum, so that the problems with the manifest itself
* can be detected.
*
* We always use SHA-256 for this, regardless of what algorithm is chosen
* for checksumming the files. If we ever want to make the checksum
* algorithm used for the manifest file variable, the client will need a
* way to figure out which algorithm to use as close to the beginning of
* the manifest file as possible, to avoid having to read the whole thing
* twice.
*/
manifest->still_checksumming = false;
pg_sha256_final(&manifest->manifest_ctx, checksumbuf);
AppendStringToManifest(manifest, "\"Manifest-Checksum\": \"");
hex_encode((char *) checksumbuf, sizeof checksumbuf, checksumstringbuf);
checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH - 1] = '\0';
AppendStringToManifest(manifest, checksumstringbuf);
AppendStringToManifest(manifest, "\"}\n");
/*
* We've written all the data to the manifest file. Rewind the file so
* that we can read it all back.
*/
if (BufFileSeek(manifest->buffile, 0, 0L, SEEK_SET))
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not rewind temporary file: %m")));
/* Send CopyOutResponse message */
pq_beginmessage(&protobuf, 'H');
pq_sendbyte(&protobuf, 0); /* overall format */
pq_sendint16(&protobuf, 0); /* natts */
pq_endmessage(&protobuf);
/*
* Send CopyData messages.
*
* We choose to read back the data from the temporary file in chunks of
* size BLCKSZ; this isn't necessary, but buffile.c uses that as the I/O
* size, so it seems to make sense to match that value here.
*/
while (manifest_bytes_done < manifest->manifest_size)
{
char manifestbuf[BLCKSZ];
size_t bytes_to_read;
size_t rc;
bytes_to_read = Min(sizeof(manifestbuf),
manifest->manifest_size - manifest_bytes_done);
rc = BufFileRead(manifest->buffile, manifestbuf, bytes_to_read);
if (rc != bytes_to_read)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from temporary file: %m")));
pq_putmessage('d', manifestbuf, bytes_to_read);
manifest_bytes_done += bytes_to_read;
}
/* No more data, so send CopyDone message */
pq_putemptymessage('c');
/* Release resources */
BufFileClose(manifest->buffile);
}

391
src/backend/replication/basebackup.c
View File

@ -16,10 +16,8 @@
#include <unistd.h>
#include <time.h>
#include "access/timeline.h"
#include "access/xlog_internal.h" /* for pg_start/stop_backup */
#include "catalog/pg_type.h"
#include "common/checksum_helper.h"
#include "common/file_perm.h"
#include "commands/progress.h"
#include "lib/stringinfo.h"
@ -32,9 +30,9 @@
#include "port.h"
#include "postmaster/syslogger.h"
#include "replication/basebackup.h"
#include "replication/backup_manifest.h"
#include "replication/walsender.h"
#include "replication/walsender_private.h"
#include "storage/buffile.h"
#include "storage/bufpage.h"
#include "storage/checksum.h"
#include "storage/dsm_impl.h"
@ -42,19 +40,11 @@
#include "storage/ipc.h"
#include "storage/reinit.h"
#include "utils/builtins.h"
#include "utils/json.h"
#include "utils/ps_status.h"
#include "utils/relcache.h"
#include "utils/resowner.h"
#include "utils/timestamp.h"
typedef enum manifest_option
{
MANIFEST_OPTION_YES,
MANIFEST_OPTION_NO,
MANIFEST_OPTION_FORCE_ENCODE
} manifest_option;
typedef struct
{
const char *label;
@ -68,18 +58,6 @@ typedef struct
pg_checksum_type manifest_checksum_type;
} basebackup_options;
struct manifest_info
{
BufFile *buffile;
pg_checksum_type checksum_type;
pg_sha256_ctx manifest_ctx;
uint64 manifest_size;
bool force_encode;
bool first_file;
bool still_checksumming;
};
static int64 sendDir(const char *path, int basepathlen, bool sizeonly,
List *tablespaces, bool sendtblspclinks,
manifest_info *manifest, const char *spcoid);
@ -94,18 +72,6 @@ static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *sta
bool sizeonly);
static void send_int8_string(StringInfoData *buf, int64 intval);
static void SendBackupHeader(List *tablespaces);
static bool IsManifestEnabled(manifest_info *manifest);
static void InitializeManifest(manifest_info *manifest,
basebackup_options *opt);
static void AppendStringToManifest(manifest_info *manifest, char *s);
static void AddFileToManifest(manifest_info *manifest, const char *spcoid,
const char *pathname, size_t size,
pg_time_t mtime,
pg_checksum_context *checksum_ctx);
static void AddWALInfoToManifest(manifest_info *manifest, XLogRecPtr startptr,
TimeLineID starttli, XLogRecPtr endptr,
TimeLineID endtli);
static void SendBackupManifest(manifest_info *manifest);
static void perform_base_backup(basebackup_options *opt);
static void parse_basebackup_options(List *options, basebackup_options *opt);
static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli);
@ -142,16 +108,6 @@ do { \
(errmsg("could not read from file \"%s\"", filename))); \
} while (0)
/*
* Convenience macro for appending data to the backup manifest.
*/
#define AppendToManifest(manifest, ...) \
{ \
char *_manifest_s = psprintf(__VA_ARGS__); \
AppendStringToManifest(manifest, _manifest_s); \
pfree(_manifest_s); \
}
/* The actual number of bytes, transfer of which may cause sleep. */
static uint64 throttling_sample;
@ -342,7 +298,7 @@ perform_base_backup(basebackup_options *opt)
labelfile = makeStringInfo();
tblspc_map_file = makeStringInfo();
InitializeManifest(&manifest, opt);
InitializeManifest(&manifest, opt->manifest, opt->manifest_checksum_type);
total_checksum_failures = 0;
@ -1068,349 +1024,6 @@ SendBackupHeader(List *tablespaces)
pq_puttextmessage('C', "SELECT");
}
/*
* Does the user want a backup manifest?
*
* It's simplest to always have a manifest_info object, so that we don't need
* checks for NULL pointers in too many places. However, if the user doesn't
* want a manifest, we set manifest->buffile to NULL.
*/
static bool
IsManifestEnabled(manifest_info *manifest)
{
return (manifest->buffile != NULL);
}
/*
* Initialize state so that we can construct a backup manifest.
*
* NB: Although the checksum type for the data files is configurable, the
* checksum for the manifest itself always uses SHA-256. See comments in
* SendBackupManifest.
*/
static void
InitializeManifest(manifest_info *manifest, basebackup_options *opt)
{
if (opt->manifest == MANIFEST_OPTION_NO)
manifest->buffile = NULL;
else
manifest->buffile = BufFileCreateTemp(false);
manifest->checksum_type = opt->manifest_checksum_type;
pg_sha256_init(&manifest->manifest_ctx);
manifest->manifest_size = UINT64CONST(0);
manifest->force_encode = (opt->manifest == MANIFEST_OPTION_FORCE_ENCODE);
manifest->first_file = true;
manifest->still_checksumming = true;
if (opt->manifest != MANIFEST_OPTION_NO)
AppendToManifest(manifest,
"{ \"PostgreSQL-Backup-Manifest-Version\": 1,\n"
"\"Files\": [");
}
/*
* Append a cstring to the manifest.
*/
static void
AppendStringToManifest(manifest_info *manifest, char *s)
{
int len = strlen(s);
size_t written;
Assert(manifest != NULL);
if (manifest->still_checksumming)
pg_sha256_update(&manifest->manifest_ctx, (uint8 *) s, len);
written = BufFileWrite(manifest->buffile, s, len);
if (written != len)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not write to temporary file: %m")));
manifest->manifest_size += len;
}
/*
* Add an entry to the backup manifest for a file.
*/
static void
AddFileToManifest(manifest_info *manifest, const char *spcoid,
const char *pathname, size_t size, pg_time_t mtime,
pg_checksum_context *checksum_ctx)
{
char pathbuf[MAXPGPATH];
int pathlen;
StringInfoData buf;
if (!IsManifestEnabled(manifest))
return;
/*
* If this file is part of a tablespace, the pathname passed to this
* function will be relative to the tar file that contains it. We want the
* pathname relative to the data directory (ignoring the intermediate
* symlink traversal).
*/
if (spcoid != NULL)
{
snprintf(pathbuf, sizeof(pathbuf), "pg_tblspc/%s/%s", spcoid,
pathname);
pathname = pathbuf;
}
/*
* Each file's entry needs to be separated from any entry that follows by a
* comma, but there's no comma before the first one or after the last one.
* To make that work, adding a file to the manifest starts by terminating
* the most recently added line, with a comma if appropriate, but does not
* terminate the line inserted for this file.
*/
initStringInfo(&buf);
if (manifest->first_file)
{
appendStringInfoString(&buf, "\n");
manifest->first_file = false;
}
else
appendStringInfoString(&buf, ",\n");
/*
* Write the relative pathname to this file out to the manifest. The
* manifest is always stored in UTF-8, so we have to encode paths that are
* not valid in that encoding.
*/
pathlen = strlen(pathname);
if (!manifest->force_encode &&
pg_verify_mbstr(PG_UTF8, pathname, pathlen, true))
{
appendStringInfoString(&buf, "{ \"Path\": ");
escape_json(&buf, pathname);
appendStringInfoString(&buf, ", ");
}
else
{
appendStringInfoString(&buf, "{ \"Encoded-Path\": \"");
enlargeStringInfo(&buf, 2 * pathlen);
buf.len += hex_encode((char *) pathname, pathlen,
&buf.data[buf.len]);
appendStringInfoString(&buf, "\", ");
}
appendStringInfo(&buf, "\"Size\": %zu, ", size);
/*
* Convert last modification time to a string and append it to the
* manifest. Since it's not clear what time zone to use and since time
* zone definitions can change, possibly causing confusion, use GMT
* always.
*/
appendStringInfoString(&buf, "\"Last-Modified\": \"");
enlargeStringInfo(&buf, 128);
buf.len += pg_strftime(&buf.data[buf.len], 128, "%Y-%m-%d %H:%M:%S %Z",
pg_gmtime(&mtime));
appendStringInfoString(&buf, "\"");
/* Add checksum information. */
if (checksum_ctx->type != CHECKSUM_TYPE_NONE)
{
uint8 checksumbuf[PG_CHECKSUM_MAX_LENGTH];
int checksumlen;
checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
appendStringInfo(&buf,
", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"",
pg_checksum_type_name(checksum_ctx->type));
enlargeStringInfo(&buf, 2 * checksumlen);
buf.len += hex_encode((char *) checksumbuf, checksumlen,
&buf.data[buf.len]);
appendStringInfoString(&buf, "\"");
}
/* Close out the object. */
appendStringInfoString(&buf, " }");
/* OK, add it to the manifest. */
AppendStringToManifest(manifest, buf.data);
/* Avoid leaking memory. */
pfree(buf.data);
}
/*
* Add information about the WAL that will need to be replayed when restoring
* this backup to the manifest.
*/
static void
AddWALInfoToManifest(manifest_info *manifest, XLogRecPtr startptr,
TimeLineID starttli, XLogRecPtr endptr, TimeLineID endtli)
{
List *timelines;
ListCell *lc;
bool first_wal_range = true;
bool found_start_timeline = false;
if (!IsManifestEnabled(manifest))
return;
/* Terminate the list of files. */
AppendStringToManifest(manifest, "\n],\n");
/* Read the timeline history for the ending timeline. */
timelines = readTimeLineHistory(endtli);
/* Start a list of LSN ranges. */
AppendStringToManifest(manifest, "\"WAL-Ranges\": [\n");
foreach(lc, timelines)
{
TimeLineHistoryEntry *entry = lfirst(lc);
XLogRecPtr tl_beginptr;
/*
* We only care about timelines that were active during the backup.
* Skip any that ended before the backup started. (Note that if
* entry->end is InvalidXLogRecPtr, it means that the timeline has not
* yet ended.)
*/
if (!XLogRecPtrIsInvalid(entry->end) && entry->end < startptr)
continue;
/*
* Because the timeline history file lists newer timelines before
* older ones, the first timeline we encounter that is new enough to
* matter ought to match the ending timeline of the backup.
*/
if (first_wal_range && endtli != entry->tli)
ereport(ERROR,
errmsg("expected end timeline %u but found timeline %u",
starttli, entry->tli));
if (!XLogRecPtrIsInvalid(entry->begin))
tl_beginptr = entry->begin;
else
{
tl_beginptr = startptr;
/*
* If we reach a TLI that has no valid beginning LSN, there can't
* be any more timelines in the history after this point, so we'd
* better have arrived at the expected starting TLI. If not,
* something's gone horribly wrong.
*/
if (starttli != entry->tli)
ereport(ERROR,
errmsg("expected start timeline %u but found timeline %u",
starttli, entry->tli));
}
AppendToManifest(manifest,
"%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }",
first_wal_range ? "" : ",\n",
entry->tli,
(uint32) (tl_beginptr >> 32), (uint32) tl_beginptr,
(uint32) (endptr >> 32), (uint32) endptr);
if (starttli == entry->tli)
{
found_start_timeline = true;
break;
}
endptr = entry->begin;
first_wal_range = false;
}
/*
* The last entry in the timeline history for the ending timeline should
* be the ending timeline itself. Verify that this is what we observed.
*/
if (!found_start_timeline)
ereport(ERROR,
errmsg("start timeline %u not found history of timeline %u",
starttli, endtli));
/* Terminate the list of WAL ranges. */
AppendStringToManifest(manifest, "\n],\n");
}
/*
* Finalize the backup manifest, and send it to the client.
*/
static void
SendBackupManifest(manifest_info *manifest)
{
StringInfoData protobuf;
uint8 checksumbuf[PG_SHA256_DIGEST_LENGTH];
char checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH];
size_t manifest_bytes_done = 0;
if (!IsManifestEnabled(manifest))
return;
/*
* Append manifest checksum, so that the problems with the manifest itself
* can be detected.
*
* We always use SHA-256 for this, regardless of what algorithm is chosen
* for checksumming the files. If we ever want to make the checksum
* algorithm used for the manifest file variable, the client will need a
* way to figure out which algorithm to use as close to the beginning of
* the manifest file as possible, to avoid having to read the whole thing
* twice.
*/
manifest->still_checksumming = false;
pg_sha256_final(&manifest->manifest_ctx, checksumbuf);
AppendStringToManifest(manifest, "\"Manifest-Checksum\": \"");
hex_encode((char *) checksumbuf, sizeof checksumbuf, checksumstringbuf);
checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH - 1] = '\0';
AppendStringToManifest(manifest, checksumstringbuf);
AppendStringToManifest(manifest, "\"}\n");
/*
* We've written all the data to the manifest file. Rewind the file so
* that we can read it all back.
*/
if (BufFileSeek(manifest->buffile, 0, 0L, SEEK_SET))
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not rewind temporary file: %m")));
/* Send CopyOutResponse message */
pq_beginmessage(&protobuf, 'H');
pq_sendbyte(&protobuf, 0); /* overall format */
pq_sendint16(&protobuf, 0); /* natts */
pq_endmessage(&protobuf);
/*
* Send CopyData messages.
*
* We choose to read back the data from the temporary file in chunks of
* size BLCKSZ; this isn't necessary, but buffile.c uses that as the I/O
* size, so it seems to make sense to match that value here.
*/
while (manifest_bytes_done < manifest->manifest_size)
{
char manifestbuf[BLCKSZ];
size_t bytes_to_read;
size_t rc;
bytes_to_read = Min(sizeof(manifestbuf),
manifest->manifest_size - manifest_bytes_done);
rc = BufFileRead(manifest->buffile, manifestbuf, bytes_to_read);
if (rc != bytes_to_read)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read from temporary file: %m")));
pq_putmessage('d', manifestbuf, bytes_to_read);
manifest_bytes_done += bytes_to_read;
}
/* No more data, so send CopyDone message */
pq_putemptymessage('c');
/* Release resources */
BufFileClose(manifest->buffile);
}
/*
* Send a single resultset containing just a single
* XLogRecPtr record (in text format)

51
src/include/replication/backup_manifest.h Normal file
View File

@ -0,0 +1,51 @@
/*-------------------------------------------------------------------------
*
* backup_manifest.h
* Routines for generating a backup manifest.
*
* Portions Copyright (c) 2010-2020, PostgreSQL Global Development Group
*
* src/include/replication/backup_manifest.h
*
*-------------------------------------------------------------------------
*/
#ifndef BACKUP_MANIFEST_H
#define BACKUP_MANIFEST_H
#include "access/xlogdefs.h"
#include "common/checksum_helper.h"
#include "pgtime.h"
#include "storage/buffile.h"
typedef enum manifest_option
{
MANIFEST_OPTION_YES,
MANIFEST_OPTION_NO,
MANIFEST_OPTION_FORCE_ENCODE
} manifest_option;
typedef struct manifest_info
{
BufFile *buffile;
pg_checksum_type checksum_type;
pg_sha256_ctx manifest_ctx;
uint64 manifest_size;
bool force_encode;
bool first_file;
bool still_checksumming;
} manifest_info;
extern void InitializeManifest(manifest_info *manifest,
manifest_option want_manifest,
pg_checksum_type manifest_checksum_type);
extern void AppendStringToManifest(manifest_info *manifest, char *s);
extern void AddFileToManifest(manifest_info *manifest, const char *spcoid,
const char *pathname, size_t size,
pg_time_t mtime,
pg_checksum_context *checksum_ctx);
extern void AddWALInfoToManifest(manifest_info *manifest, XLogRecPtr startptr,
TimeLineID starttli, XLogRecPtr endptr,
TimeLineID endtli);
extern void SendBackupManifest(manifest_info *manifest);
#endif