mirrors/postgres
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for progress reporting to pg_verifybackup

Browse Source 
This adds a new option to pg_verifybackup called -P/--progress, showing
every second some information about the progress of the checksum
verification based on the data of a backup manifest.

Similarly to what is done for pg_rewind and pg_basebackup, the
information printed in the progress report consists of the current
amount of data computed and the total amount of data that will be
computed.  Note that files found with an incorrect size do not have
their checksum verified, hence their size is not appended to the total
amount of data estimated during the first scan of the manifest data
(such incorrect sizes could be overly high, for one, falsifying the
progress report).

Author: Masahiko Sawada
Discussion: https://postgr.es/m/CAD21AoC5+JOgMd4o3z_oxw0f8JDSsCYY7zSbhe-O9x7f33rw_A@mail.gmail.com
This commit is contained in:
Michael Paquier 2023-02-06 14:40:31 +09:00
parent 71c37797d7
commit d07c2948bf
3 changed files with 111 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
doc/src/sgml/ref/pg_verifybackup.sgml
View File

@ -178,6 +178,21 @@ PostgreSQL documentation
</listitem>
</varlistentry>
<varlistentry>
<term><option>-P</option></term>
<term><option>--progress</option></term>
<listitem>
<para>
Enable progress reporting. Turning this on will deliver a progress
report while verifying checksums.
</para>
<para>
This option cannot be used together with the option
<option>--quiet</option>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>-q</option></term>
<term><option>--quiet</option></term>

86
src/bin/pg_verifybackup/pg_verifybackup.c
View File

@ -16,12 +16,14 @@
#include <dirent.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <time.h>
#include "common/hashfn.h"
#include "common/logging.h"
#include "fe_utils/simple_list.h"
#include "getopt_long.h"
#include "parse_manifest.h"
#include "pgtime.h"
/*
* For efficiency, we'd like our hash table containing information about the
@ -58,6 +60,9 @@ typedef struct manifest_file
bool bad;
} manifest_file;
#define should_verify_checksum(m) \
(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
/*
* Define a hash table which we can use to store information about the files
* mentioned in the backup manifest.
@ -147,10 +152,19 @@ static void report_fatal_error(const char *pg_restrict fmt,...)
pg_attribute_printf(1, 2) pg_attribute_noreturn();
static bool should_ignore_relpath(verifier_context *context, char *relpath);
static void progress_report(bool finished);
static void usage(void);
static const char *progname;
/* options */
static bool show_progress = false;
static bool skip_checksums = false;
/* Progress indicators */
static uint64 total_size = 0;
static uint64 done_size = 0;
/*
* Main entry point.
*/
@ -162,6 +176,7 @@ main(int argc, char **argv)
{"ignore", required_argument, NULL, 'i'},
{"manifest-path", required_argument, NULL, 'm'},
{"no-parse-wal", no_argument, NULL, 'n'},
{"progress", no_argument, NULL, 'P'},
{"quiet", no_argument, NULL, 'q'},
{"skip-checksums", no_argument, NULL, 's'},
{"wal-directory", required_argument, NULL, 'w'},
@ -174,7 +189,6 @@ main(int argc, char **argv)
char *manifest_path = NULL;
bool no_parse_wal = false;
bool quiet = false;
bool skip_checksums = false;
char *wal_directory = NULL;
char *pg_waldump_path = NULL;
@ -219,7 +233,7 @@ main(int argc, char **argv)
simple_string_list_append(&context.ignore_list, "recovery.signal");
simple_string_list_append(&context.ignore_list, "standby.signal");
while ((c = getopt_long(argc, argv, "ei:m:nqsw:", long_options, NULL)) != -1)
while ((c = getopt_long(argc, argv, "ei:m:nPqsw:", long_options, NULL)) != -1)
{
switch (c)
{
@ -241,6 +255,9 @@ main(int argc, char **argv)
case 'n':
no_parse_wal = true;
break;
case 'P':
show_progress = true;
break;
case 'q':
quiet = true;
break;
@ -277,6 +294,11 @@ main(int argc, char **argv)
exit(1);
}
/* Complain if the specified arguments conflict */
if (show_progress && quiet)
pg_fatal("cannot specify both %s and %s",
"-P/--progress", "-q/--quiet");
/* Unless --no-parse-wal was specified, we will need pg_waldump. */
if (!no_parse_wal)
{
@ -638,6 +660,10 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
m->bad = true;
}
/* Update statistics for progress report, if necessary */
if (show_progress && !skip_checksums && should_verify_checksum(m))
total_size += m->size;
/*
* We don't verify checksums at this stage. We first finish verifying that
* we have the expected set of files with the expected sizes, and only
@ -675,10 +701,12 @@ verify_backup_checksums(verifier_context *context)
manifest_files_iterator it;
manifest_file *m;
progress_report(false);
manifest_files_start_iterate(context->ht, &it);
while ((m = manifest_files_iterate(context->ht, &it)) != NULL)
{
if (m->matched && !m->bad && m->checksum_type != CHECKSUM_TYPE_NONE &&
if (should_verify_checksum(m) &&
!should_ignore_relpath(context, m->pathname))
{
char *fullpath;
@ -694,6 +722,8 @@ verify_backup_checksums(verifier_context *context)
pfree(fullpath);
}
}
progress_report(true);
}
/*
@ -740,6 +770,10 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
close(fd);
return;
}
/* Report progress */
done_size += rc;
progress_report(false);
}
if (rc < 0)
report_backup_error(context, "could not read file \"%s\": %m",
@ -894,6 +928,51 @@ hash_string_pointer(char *s)
return hash_bytes(ss, strlen(s));
}
/*
* Print a progress report based on the global variables.
*
* Progress report is written at maximum once per second, unless the finished
* parameter is set to true.
*
* If finished is set to true, this is the last progress report. The cursor
* is moved to the next line.
*/
static void
progress_report(bool finished)
{
static pg_time_t last_progress_report = 0;
pg_time_t now;
int percent_size = 0;
char totalsize_str[32];
char donesize_str[32];
if (!show_progress)
return;
now = time(NULL);
if (now == last_progress_report && !finished)
return; /* Max once per second */
last_progress_report = now;
percent_size = total_size ? (int) ((done_size * 100 / total_size)) : 0;
snprintf(totalsize_str, sizeof(totalsize_str), UINT64_FORMAT,
total_size / 1024);
snprintf(donesize_str, sizeof(donesize_str), UINT64_FORMAT,
done_size / 1024);
fprintf(stderr,
_("%*s/%s kB (%d%%) verified"),
(int) strlen(totalsize_str),
donesize_str, totalsize_str, percent_size);
/*
* Stay on the same line if reporting to a terminal and we're not done
* yet.
*/
fputc((!finished && isatty(fileno(stderr))) ? '\r' : '\n', stderr);
}
/*
* Print out usage information and exit.
*/
@ -907,6 +986,7 @@ usage(void)
printf(_(" -i, --ignore=RELATIVE_PATH ignore indicated path\n"));
printf(_(" -m, --manifest-path=PATH use specified path for manifest\n"));
printf(_(" -n, --no-parse-wal do not try to parse WAL files\n"));
printf(_(" -P, --progress show progress information\n"));
printf(_(" -q, --quiet do not print any output, except for errors\n"));
printf(_(" -s, --skip-checksums skip checksum verification\n"));
printf(_(" -w, --wal-directory=PATH use specified path for WAL files\n"));

17
src/bin/pg_verifybackup/t/004_options.pl
View File

@ -28,6 +28,12 @@ ok($result, "-q succeeds: exit code 0");
is($stdout, '', "-q succeeds: no stdout");
is($stderr, '', "-q succeeds: no stderr");
# Test invalid options
command_fails_like(
[ 'pg_verifybackup', '--progress', '--quiet', $backup_path ],
qr{cannot specify both -P/--progress and -q/--quiet},
'cannot use --progress and --quiet at the same time');
# Corrupt the PG_VERSION file.
my $version_pathname = "$backup_path/PG_VERSION";
my $version_contents = slurp_file($version_pathname);
@ -48,10 +54,13 @@ command_like(
qr/backup successfully verified/,
'-s skips checksumming');
# Validation should succeed if we ignore the problem file.
command_like(
[ 'pg_verifybackup', '-i', 'PG_VERSION', $backup_path ],
qr/backup successfully verified/,
# Validation should succeed if we ignore the problem file. Also, check
# the progress information.
command_checks_all(
[ 'pg_verifybackup', '--progress', '-i', 'PG_VERSION', $backup_path ],
0,
[qr/backup successfully verified/],
[qr{(\d+/\d+ kB \(\d+%\) verified)+}],
'-i ignores problem file');
# PG_VERSION is already corrupt; let's try also removing all of pg_xact.