diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 9178c779ba..719b947ef4 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -2731,14 +2731,24 @@ The commands accepted in replication mode are: - COMPRESSION_LEVEL level + COMPRESSION_DETAIL detail - Specifies the compression level to be used. This should only be - used in conjunction with the COMPRESSION option. - For gzip the value should be an integer between 1 - and 9, for lz4 between 1 and 12, and for - zstd it should be between 1 and 22. + Specifies details for the chosen compression method. This should only + be used in conjunction with the COMPRESSION + option. If the value is an integer, it specifies the compression + level. Otherwise, it should be a comma-separated list of items, + each of the form keyword or + keyword=value. Currently, the only supported + keyword is level, which sets the compression + level. + + + + For gzip the compression level should be an + integer between 1 and 9, for lz4 an integer + between 1 and 12, and for zstd an integer + between 1 and 22. diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index 4a630b59b7..d9233beb8e 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -399,9 +399,9 @@ PostgreSQL documentation - [:level] + - [:level] + Requests compression of the backup. If client or @@ -419,13 +419,20 @@ PostgreSQL documentation The compression method can be set to gzip, lz4, zstd, or - none for no compression. A compression level can - optionally be specified, by appending the level number after a colon - (:). If no level is specified, the default - compression level will be used. If only a level is specified without - mentioning an algorithm, gzip compression will be - used if the level is greater than 0, and no compression will be used if - the level is 0. + none for no compression. A compression detail + string can optionally be specified. If the detail string is an + integer, it specifies the compression level. Otherwise, it should be + a comma-separated list of items, each of the form + keyword or keyword=value. + Currently, the only supported keyword is level, + which sets the compression level. + + + If no compression level is specified, the default compression level + will be used. If only a level is specified without mentioning an + algorithm, gzip compression will be used if the + level is greater than 0, and no compression will be used if the level + is 0. When the tar format is used with gzip, diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index c2aedc14a2..6884cad2c0 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -17,6 +17,7 @@ #include #include "access/xlog_internal.h" /* for pg_start/stop_backup */ +#include "common/backup_compression.h" #include "common/file_perm.h" #include "commands/defrem.h" #include "lib/stringinfo.h" @@ -54,14 +55,6 @@ */ #define SINK_BUFFER_LENGTH Max(32768, BLCKSZ) -typedef enum -{ - BACKUP_COMPRESSION_NONE, - BACKUP_COMPRESSION_GZIP, - BACKUP_COMPRESSION_LZ4, - BACKUP_COMPRESSION_ZSTD -} basebackup_compression_type; - typedef struct { const char *label; @@ -75,8 +68,8 @@ typedef struct bool use_copytblspc; BaseBackupTargetHandle *target_handle; backup_manifest_option manifest; - basebackup_compression_type compression; - int compression_level; + bc_algorithm compression; + bc_specification compression_specification; pg_checksum_type manifest_checksum_type; } basebackup_options; @@ -713,12 +706,14 @@ parse_basebackup_options(List *options, basebackup_options *opt) char *target_str = NULL; char *target_detail_str = NULL; bool o_compression = false; - bool o_compression_level = false; + bool o_compression_detail = false; + char *compression_detail_str = NULL; MemSet(opt, 0, sizeof(*opt)); opt->manifest = MANIFEST_OPTION_NO; opt->manifest_checksum_type = CHECKSUM_TYPE_CRC32C; opt->compression = BACKUP_COMPRESSION_NONE; + opt->compression_specification.algorithm = BACKUP_COMPRESSION_NONE; foreach(lopt, options) { @@ -885,29 +880,21 @@ parse_basebackup_options(List *options, basebackup_options *opt) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("duplicate option \"%s\"", defel->defname))); - if (strcmp(optval, "none") == 0) - opt->compression = BACKUP_COMPRESSION_NONE; - else if (strcmp(optval, "gzip") == 0) - opt->compression = BACKUP_COMPRESSION_GZIP; - else if (strcmp(optval, "lz4") == 0) - opt->compression = BACKUP_COMPRESSION_LZ4; - else if (strcmp(optval, "zstd") == 0) - opt->compression = BACKUP_COMPRESSION_ZSTD; - else + if (!parse_bc_algorithm(optval, &opt->compression)) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("unrecognized compression algorithm: \"%s\"", + errmsg("unrecognized compression algorithm \"%s\"", optval))); o_compression = true; } - else if (strcmp(defel->defname, "compression_level") == 0) + else if (strcmp(defel->defname, "compression_detail") == 0) { - if (o_compression_level) + if (o_compression_detail) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("duplicate option \"%s\"", defel->defname))); - opt->compression_level = defGetInt32(defel); - o_compression_level = true; + compression_detail_str = defGetString(defel); + o_compression_detail = true; } else ereport(ERROR, @@ -949,10 +936,25 @@ parse_basebackup_options(List *options, basebackup_options *opt) opt->target_handle = BaseBackupGetTargetHandle(target_str, target_detail_str); - if (o_compression_level && !o_compression) + if (o_compression_detail && !o_compression) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("compression level requires compression"))); + errmsg("compression detail requires compression"))); + + if (o_compression) + { + char *error_detail; + + parse_bc_specification(opt->compression, compression_detail_str, + &opt->compression_specification); + error_detail = + validate_bc_specification(&opt->compression_specification); + if (error_detail != NULL) + ereport(ERROR, + errcode(ERRCODE_SYNTAX_ERROR), + errmsg("invalid compression specification: %s", + error_detail)); + } } @@ -998,11 +1000,11 @@ SendBaseBackup(BaseBackupCmd *cmd) /* Set up server-side compression, if client requested it */ if (opt.compression == BACKUP_COMPRESSION_GZIP) - sink = bbsink_gzip_new(sink, opt.compression_level); + sink = bbsink_gzip_new(sink, &opt.compression_specification); else if (opt.compression == BACKUP_COMPRESSION_LZ4) - sink = bbsink_lz4_new(sink, opt.compression_level); + sink = bbsink_lz4_new(sink, &opt.compression_specification); else if (opt.compression == BACKUP_COMPRESSION_ZSTD) - sink = bbsink_zstd_new(sink, opt.compression_level); + sink = bbsink_zstd_new(sink, &opt.compression_specification); /* Set up progress reporting. */ sink = bbsink_progress_new(sink, opt.progress); diff --git a/src/backend/replication/basebackup_gzip.c b/src/backend/replication/basebackup_gzip.c index b66d3da7a3..703a91ba77 100644 --- a/src/backend/replication/basebackup_gzip.c +++ b/src/backend/replication/basebackup_gzip.c @@ -56,12 +56,13 @@ const bbsink_ops bbsink_gzip_ops = { #endif /* - * Create a new basebackup sink that performs gzip compression using the - * designated compression level. + * Create a new basebackup sink that performs gzip compression. */ bbsink * -bbsink_gzip_new(bbsink *next, int compresslevel) +bbsink_gzip_new(bbsink *next, bc_specification *compress) { + int compresslevel; + #ifndef HAVE_LIBZ ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -71,15 +72,14 @@ bbsink_gzip_new(bbsink *next, int compresslevel) bbsink_gzip *sink; Assert(next != NULL); - Assert(compresslevel >= 0 && compresslevel <= 9); - if (compresslevel == 0) + if ((compress->options & BACKUP_COMPRESSION_OPTION_LEVEL) == 0) compresslevel = Z_DEFAULT_COMPRESSION; - else if (compresslevel < 0 || compresslevel > 9) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("gzip compression level %d is out of range", - compresslevel))); + else + { + compresslevel = compress->level; + Assert(compresslevel >= 1 && compresslevel <= 9); + } sink = palloc0(sizeof(bbsink_gzip)); *((const bbsink_ops **) &sink->base.bbs_ops) = &bbsink_gzip_ops; diff --git a/src/backend/replication/basebackup_lz4.c b/src/backend/replication/basebackup_lz4.c index d838f723d0..06c161ddc4 100644 --- a/src/backend/replication/basebackup_lz4.c +++ b/src/backend/replication/basebackup_lz4.c @@ -56,12 +56,13 @@ const bbsink_ops bbsink_lz4_ops = { #endif /* - * Create a new basebackup sink that performs lz4 compression using the - * designated compression level. + * Create a new basebackup sink that performs lz4 compression. */ bbsink * -bbsink_lz4_new(bbsink *next, int compresslevel) +bbsink_lz4_new(bbsink *next, bc_specification *compress) { + int compresslevel; + #ifndef USE_LZ4 ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -72,11 +73,13 @@ bbsink_lz4_new(bbsink *next, int compresslevel) Assert(next != NULL); - if (compresslevel < 0 || compresslevel > 12) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("lz4 compression level %d is out of range", - compresslevel))); + if ((compress->options & BACKUP_COMPRESSION_OPTION_LEVEL) == 0) + compresslevel = 0; + else + { + compresslevel = compress->level; + Assert(compresslevel >= 1 && compresslevel <= 12); + } sink = palloc0(sizeof(bbsink_lz4)); *((const bbsink_ops **) &sink->base.bbs_ops) = &bbsink_lz4_ops; diff --git a/src/backend/replication/basebackup_zstd.c b/src/backend/replication/basebackup_zstd.c index c0e2be6e27..96b7985693 100644 --- a/src/backend/replication/basebackup_zstd.c +++ b/src/backend/replication/basebackup_zstd.c @@ -55,12 +55,13 @@ const bbsink_ops bbsink_zstd_ops = { #endif /* - * Create a new basebackup sink that performs zstd compression using the - * designated compression level. + * Create a new basebackup sink that performs zstd compression. */ bbsink * -bbsink_zstd_new(bbsink *next, int compresslevel) +bbsink_zstd_new(bbsink *next, bc_specification *compress) { + int compresslevel; + #ifndef USE_ZSTD ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -71,11 +72,13 @@ bbsink_zstd_new(bbsink *next, int compresslevel) Assert(next != NULL); - if (compresslevel < 0 || compresslevel > 22) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("zstd compression level %d is out of range", - compresslevel))); + if ((compress->options & BACKUP_COMPRESSION_OPTION_LEVEL) == 0) + compresslevel = 0; + else + { + compresslevel = compress->level; + Assert(compresslevel >= 1 && compresslevel <= 22); + } sink = palloc0(sizeof(bbsink_zstd)); *((const bbsink_ops **) &sink->base.bbs_ops) = &bbsink_zstd_ops; diff --git a/src/bin/pg_basebackup/bbstreamer.h b/src/bin/pg_basebackup/bbstreamer.h index 02d4c05df6..dfa3f77af4 100644 --- a/src/bin/pg_basebackup/bbstreamer.h +++ b/src/bin/pg_basebackup/bbstreamer.h @@ -22,6 +22,7 @@ #ifndef BBSTREAMER_H #define BBSTREAMER_H +#include "common/backup_compression.h" #include "lib/stringinfo.h" #include "pqexpbuffer.h" @@ -200,17 +201,17 @@ bbstreamer_buffer_until(bbstreamer *streamer, const char **data, int *len, */ extern bbstreamer *bbstreamer_plain_writer_new(char *pathname, FILE *file); extern bbstreamer *bbstreamer_gzip_writer_new(char *pathname, FILE *file, - int compresslevel); + bc_specification *compress); extern bbstreamer *bbstreamer_extractor_new(const char *basepath, const char *(*link_map) (const char *), void (*report_output_file) (const char *)); extern bbstreamer *bbstreamer_gzip_decompressor_new(bbstreamer *next); extern bbstreamer *bbstreamer_lz4_compressor_new(bbstreamer *next, - int compresslevel); + bc_specification *compress); extern bbstreamer *bbstreamer_lz4_decompressor_new(bbstreamer *next); extern bbstreamer *bbstreamer_zstd_compressor_new(bbstreamer *next, - int compresslevel); + bc_specification *compress); extern bbstreamer *bbstreamer_zstd_decompressor_new(bbstreamer *next); extern bbstreamer *bbstreamer_tar_parser_new(bbstreamer *next); extern bbstreamer *bbstreamer_tar_terminator_new(bbstreamer *next); diff --git a/src/bin/pg_basebackup/bbstreamer_gzip.c b/src/bin/pg_basebackup/bbstreamer_gzip.c index 894f857103..1979e95639 100644 --- a/src/bin/pg_basebackup/bbstreamer_gzip.c +++ b/src/bin/pg_basebackup/bbstreamer_gzip.c @@ -76,7 +76,8 @@ const bbstreamer_ops bbstreamer_gzip_decompressor_ops = { * closed so that the data may be written there. */ bbstreamer * -bbstreamer_gzip_writer_new(char *pathname, FILE *file, int compresslevel) +bbstreamer_gzip_writer_new(char *pathname, FILE *file, + bc_specification *compress) { #ifdef HAVE_LIBZ bbstreamer_gzip_writer *streamer; @@ -115,11 +116,11 @@ bbstreamer_gzip_writer_new(char *pathname, FILE *file, int compresslevel) } } - if (gzsetparams(streamer->gzfile, compresslevel, + if (gzsetparams(streamer->gzfile, compress->level, Z_DEFAULT_STRATEGY) != Z_OK) { pg_log_error("could not set compression level %d: %s", - compresslevel, get_gz_error(streamer->gzfile)); + compress->level, get_gz_error(streamer->gzfile)); exit(1); } diff --git a/src/bin/pg_basebackup/bbstreamer_lz4.c b/src/bin/pg_basebackup/bbstreamer_lz4.c index 810052e4e3..a6ec317e2b 100644 --- a/src/bin/pg_basebackup/bbstreamer_lz4.c +++ b/src/bin/pg_basebackup/bbstreamer_lz4.c @@ -67,7 +67,7 @@ const bbstreamer_ops bbstreamer_lz4_decompressor_ops = { * blocks. */ bbstreamer * -bbstreamer_lz4_compressor_new(bbstreamer *next, int compresslevel) +bbstreamer_lz4_compressor_new(bbstreamer *next, bc_specification *compress) { #ifdef USE_LZ4 bbstreamer_lz4_frame *streamer; @@ -89,7 +89,7 @@ bbstreamer_lz4_compressor_new(bbstreamer *next, int compresslevel) prefs = &streamer->prefs; memset(prefs, 0, sizeof(LZ4F_preferences_t)); prefs->frameInfo.blockSizeID = LZ4F_max256KB; - prefs->compressionLevel = compresslevel; + prefs->compressionLevel = compress->level; /* * Find out the compression bound, it specifies the minimum destination diff --git a/src/bin/pg_basebackup/bbstreamer_zstd.c b/src/bin/pg_basebackup/bbstreamer_zstd.c index e86749a8fb..caa5edcaf1 100644 --- a/src/bin/pg_basebackup/bbstreamer_zstd.c +++ b/src/bin/pg_basebackup/bbstreamer_zstd.c @@ -63,7 +63,7 @@ const bbstreamer_ops bbstreamer_zstd_decompressor_ops = { * blocks. */ bbstreamer * -bbstreamer_zstd_compressor_new(bbstreamer *next, int compresslevel) +bbstreamer_zstd_compressor_new(bbstreamer *next, bc_specification *compress) { #ifdef USE_ZSTD bbstreamer_zstd_frame *streamer; @@ -85,7 +85,7 @@ bbstreamer_zstd_compressor_new(bbstreamer *next, int compresslevel) /* Initialize stream compression preferences */ ZSTD_CCtx_setParameter(streamer->cctx, ZSTD_c_compressionLevel, - compresslevel); + compress->level); /* Initialize the ZSTD output buffer. */ streamer->zstd_outBuf.dst = streamer->base.bbs_buffer.data; diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index 2943d9ec1a..3e6977df1a 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -29,6 +29,7 @@ #include "access/xlog_internal.h" #include "bbstreamer.h" +#include "common/backup_compression.h" #include "common/file_perm.h" #include "common/file_utils.h" #include "common/logging.h" @@ -57,6 +58,7 @@ typedef struct TablespaceList typedef struct ArchiveStreamState { int tablespacenum; + bc_specification *compress; bbstreamer *streamer; bbstreamer *manifest_inject_streamer; PQExpBuffer manifest_buffer; @@ -132,9 +134,6 @@ static bool checksum_failure = false; static bool showprogress = false; static bool estimatesize = true; static int verbose = 0; -static int compresslevel = 0; -static WalCompressionMethod compressmethod = COMPRESSION_NONE; -static CompressionLocation compressloc = COMPRESS_LOCATION_UNSPECIFIED; static IncludeWal includewal = STREAM_WAL; static bool fastcheckpoint = false; static bool writerecoveryconf = false; @@ -198,7 +197,8 @@ static void progress_report(int tablespacenum, bool force, bool finished); static bbstreamer *CreateBackupStreamer(char *archive_name, char *spclocation, bbstreamer **manifest_inject_streamer_p, bool is_recovery_guc_supported, - bool expect_unterminated_tarfile); + bool expect_unterminated_tarfile, + bc_specification *compress); static void ReceiveArchiveStreamChunk(size_t r, char *copybuf, void *callback_data); static char GetCopyDataByte(size_t r, char *copybuf, size_t *cursor); @@ -207,7 +207,7 @@ static uint64 GetCopyDataUInt64(size_t r, char *copybuf, size_t *cursor); static void GetCopyDataEnd(size_t r, char *copybuf, size_t cursor); static void ReportCopyDataParseError(size_t r, char *copybuf); static void ReceiveTarFile(PGconn *conn, char *archive_name, char *spclocation, - bool tablespacenum); + bool tablespacenum, bc_specification *compress); static void ReceiveTarCopyChunk(size_t r, char *copybuf, void *callback_data); static void ReceiveBackupManifest(PGconn *conn); static void ReceiveBackupManifestChunk(size_t r, char *copybuf, @@ -215,7 +215,9 @@ static void ReceiveBackupManifestChunk(size_t r, char *copybuf, static void ReceiveBackupManifestInMemory(PGconn *conn, PQExpBuffer buf); static void ReceiveBackupManifestInMemoryChunk(size_t r, char *copybuf, void *callback_data); -static void BaseBackup(void); +static void BaseBackup(char *compression_algorithm, char *compression_detail, + CompressionLocation compressloc, + bc_specification *client_compress); static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline, bool segment_finished); @@ -405,8 +407,8 @@ usage(void) printf(_(" -X, --wal-method=none|fetch|stream\n" " include required WAL files with specified method\n")); printf(_(" -z, --gzip compress tar output\n")); - printf(_(" -Z, --compress=[{client|server}-]{gzip|lz4|zstd}[:LEVEL]\n" - " compress tar output with given compression method or level\n")); + printf(_(" -Z, --compress=[{client|server}-]METHOD[:DETAIL]\n" + " compress on client or server as specified\n")); printf(_(" -Z, --compress=none do not compress tar output\n")); printf(_("\nGeneral options:\n")); printf(_(" -c, --checkpoint=fast|spread\n" @@ -542,7 +544,9 @@ typedef struct } logstreamer_param; static int -LogStreamerMain(logstreamer_param *param) +LogStreamerMain(logstreamer_param *param, + WalCompressionMethod wal_compress_method, + int wal_compress_level) { StreamCtl stream; @@ -565,25 +569,14 @@ LogStreamerMain(logstreamer_param *param) stream.mark_done = true; stream.partial_suffix = NULL; stream.replication_slot = replication_slot; - if (format == 'p') stream.walmethod = CreateWalDirectoryMethod(param->xlog, COMPRESSION_NONE, 0, stream.do_sync); - else if (compressloc != COMPRESS_LOCATION_CLIENT) - stream.walmethod = CreateWalTarMethod(param->xlog, - COMPRESSION_NONE, - compresslevel, - stream.do_sync); - else if (compressmethod == COMPRESSION_GZIP) - stream.walmethod = CreateWalTarMethod(param->xlog, - compressmethod, - compresslevel, - stream.do_sync); else stream.walmethod = CreateWalTarMethod(param->xlog, - COMPRESSION_NONE, - compresslevel, + wal_compress_method, + wal_compress_level, stream.do_sync); if (!ReceiveXlogStream(param->bgconn, &stream)) @@ -629,7 +622,9 @@ LogStreamerMain(logstreamer_param *param) * stream the logfile in parallel with the backups. */ static void -StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier) +StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier, + WalCompressionMethod wal_compress_method, + int wal_compress_level) { logstreamer_param *param; uint32 hi, @@ -729,7 +724,7 @@ StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier) int ret; /* in child process */ - ret = LogStreamerMain(param); + ret = LogStreamerMain(param, wal_compress_method, wal_compress_level); /* temp debugging aid to analyze 019_replslot_limit failures */ if (verbose) @@ -1004,136 +999,81 @@ parse_max_rate(char *src) } /* - * Utility wrapper to parse the values specified for -Z/--compress. - * *methodres and *levelres will be optionally filled with values coming - * from the parsed results. + * Basic parsing of a value specified for -Z/--compress. + * + * We're not concerned here with understanding exactly what behavior the + * user wants, but we do need to know whether the user is requesting client + * or server side compression or leaving it unspecified, and we need to + * separate the name of the compression algorithm from the detail string. + * + * For instance, if the user writes --compress client-lz4:6, we want to + * separate that into (a) client-side compression, (b) algorithm "lz4", + * and (c) detail "6". Note, however, that all the client/server prefix is + * optional, and so is the detail. The algorithm name is required, unless + * the whole string is an integer, in which case we assume "gzip" as the + * algorithm and use the integer as the detail. + * + * We're not concerned with validation at this stage, so if the user writes + * --compress client-turkey:sandwich, the requested algorithm is "turkey" + * and the detail string is "sandwich". We'll sort out whether that's legal + * at a later stage. */ static void -parse_compress_options(char *src, WalCompressionMethod *methodres, - CompressionLocation *locationres, int *levelres) +parse_compress_options(char *option, char **algorithm, char **detail, + CompressionLocation *locationres) { char *sep; - int firstlen; - char *firstpart; + char *endp; /* - * clear 'levelres' so that if there are multiple compression options, - * the last one fully overrides the earlier ones + * Check whether the compression specification consists of a bare integer. + * + * If so, for backward compatibility, assume gzip. */ - *levelres = 0; + (void) strtol(option, &endp, 10); + if (*endp == '\0') + { + *locationres = COMPRESS_LOCATION_UNSPECIFIED; + *algorithm = pstrdup("gzip"); + *detail = pstrdup(option); + return; + } - /* check if the option is split in two */ - sep = strchr(src, ':'); + /* Strip off any "client-" or "server-" prefix. */ + if (strncmp(option, "server-", 7) == 0) + { + *locationres = COMPRESS_LOCATION_SERVER; + option += 7; + } + else if (strncmp(option, "client-", 7) == 0) + { + *locationres = COMPRESS_LOCATION_CLIENT; + option += 7; + } + else + *locationres = COMPRESS_LOCATION_UNSPECIFIED; /* - * The first part of the option value could be a method name, or just a - * level value. + * Check whether there is a compression detail following the algorithm + * name. */ - firstlen = (sep != NULL) ? (sep - src) : strlen(src); - firstpart = pg_malloc(firstlen + 1); - memcpy(firstpart, src, firstlen); - firstpart[firstlen] = '\0'; - - /* - * Check if the first part of the string matches with a supported - * compression method. - */ - if (pg_strcasecmp(firstpart, "gzip") == 0) + sep = strchr(option, ':'); + if (sep == NULL) { - *methodres = COMPRESSION_GZIP; - *locationres = COMPRESS_LOCATION_UNSPECIFIED; - } - else if (pg_strcasecmp(firstpart, "client-gzip") == 0) - { - *methodres = COMPRESSION_GZIP; - *locationres = COMPRESS_LOCATION_CLIENT; - } - else if (pg_strcasecmp(firstpart, "server-gzip") == 0) - { - *methodres = COMPRESSION_GZIP; - *locationres = COMPRESS_LOCATION_SERVER; - } - else if (pg_strcasecmp(firstpart, "lz4") == 0) - { - *methodres = COMPRESSION_LZ4; - *locationres = COMPRESS_LOCATION_UNSPECIFIED; - } - else if (pg_strcasecmp(firstpart, "client-lz4") == 0) - { - *methodres = COMPRESSION_LZ4; - *locationres = COMPRESS_LOCATION_CLIENT; - } - else if (pg_strcasecmp(firstpart, "server-lz4") == 0) - { - *methodres = COMPRESSION_LZ4; - *locationres = COMPRESS_LOCATION_SERVER; - } - else if (pg_strcasecmp(firstpart, "zstd") == 0) - { - *methodres = COMPRESSION_ZSTD; - *locationres = COMPRESS_LOCATION_UNSPECIFIED; - } - else if (pg_strcasecmp(firstpart, "client-zstd") == 0) - { - *methodres = COMPRESSION_ZSTD; - *locationres = COMPRESS_LOCATION_CLIENT; - } - else if (pg_strcasecmp(firstpart, "server-zstd") == 0) - { - *methodres = COMPRESSION_ZSTD; - *locationres = COMPRESS_LOCATION_SERVER; - } - else if (pg_strcasecmp(firstpart, "none") == 0) - { - *methodres = COMPRESSION_NONE; - *locationres = COMPRESS_LOCATION_UNSPECIFIED; + *algorithm = pstrdup(option); + *detail = NULL; } else { - /* - * It does not match anything known, so check for the - * backward-compatible case of only an integer where the implied - * compression method changes depending on the level value. - */ - if (!option_parse_int(firstpart, "-Z/--compress", 0, - INT_MAX, levelres)) - exit(1); + char *alg; - *methodres = (*levelres > 0) ? - COMPRESSION_GZIP : COMPRESSION_NONE; - *locationres = COMPRESS_LOCATION_UNSPECIFIED; + alg = palloc((sep - option) + 1); + memcpy(alg, option, sep - option); + alg[sep - option] = '\0'; - free(firstpart); - return; + *algorithm = alg; + *detail = pstrdup(sep + 1); } - - if (sep == NULL) - { - /* - * The caller specified a method without a colon separator, so let any - * subsequent checks assign a default level. - */ - free(firstpart); - return; - } - - /* Check the contents after the colon separator. */ - sep++; - if (*sep == '\0') - { - pg_log_error("no compression level defined for method %s", firstpart); - exit(1); - } - - /* - * For any of the methods currently supported, the data after the - * separator can just be an integer. - */ - if (!option_parse_int(sep, "-Z/--compress", 0, INT_MAX, - levelres)) - exit(1); - - free(firstpart); } /* @@ -1200,7 +1140,8 @@ static bbstreamer * CreateBackupStreamer(char *archive_name, char *spclocation, bbstreamer **manifest_inject_streamer_p, bool is_recovery_guc_supported, - bool expect_unterminated_tarfile) + bool expect_unterminated_tarfile, + bc_specification *compress) { bbstreamer *streamer = NULL; bbstreamer *manifest_inject_streamer = NULL; @@ -1316,32 +1257,28 @@ CreateBackupStreamer(char *archive_name, char *spclocation, archive_file = NULL; } - if (compressmethod == COMPRESSION_NONE || - compressloc != COMPRESS_LOCATION_CLIENT) + if (compress->algorithm == BACKUP_COMPRESSION_NONE) streamer = bbstreamer_plain_writer_new(archive_filename, archive_file); - else if (compressmethod == COMPRESSION_GZIP) + else if (compress->algorithm == BACKUP_COMPRESSION_GZIP) { strlcat(archive_filename, ".gz", sizeof(archive_filename)); streamer = bbstreamer_gzip_writer_new(archive_filename, - archive_file, - compresslevel); + archive_file, compress); } - else if (compressmethod == COMPRESSION_LZ4) + else if (compress->algorithm == BACKUP_COMPRESSION_LZ4) { strlcat(archive_filename, ".lz4", sizeof(archive_filename)); streamer = bbstreamer_plain_writer_new(archive_filename, archive_file); - streamer = bbstreamer_lz4_compressor_new(streamer, - compresslevel); + streamer = bbstreamer_lz4_compressor_new(streamer, compress); } - else if (compressmethod == COMPRESSION_ZSTD) + else if (compress->algorithm == BACKUP_COMPRESSION_ZSTD) { strlcat(archive_filename, ".zst", sizeof(archive_filename)); streamer = bbstreamer_plain_writer_new(archive_filename, archive_file); - streamer = bbstreamer_zstd_compressor_new(streamer, - compresslevel); + streamer = bbstreamer_zstd_compressor_new(streamer, compress); } else { @@ -1395,13 +1332,13 @@ CreateBackupStreamer(char *archive_name, char *spclocation, * If the user has requested a server compressed archive along with archive * extraction at client then we need to decompress it. */ - if (format == 'p' && compressloc == COMPRESS_LOCATION_SERVER) + if (format == 'p') { - if (compressmethod == COMPRESSION_GZIP) + if (is_tar_gz) streamer = bbstreamer_gzip_decompressor_new(streamer); - else if (compressmethod == COMPRESSION_LZ4) + else if (is_tar_lz4) streamer = bbstreamer_lz4_decompressor_new(streamer); - else if (compressmethod == COMPRESSION_ZSTD) + else if (is_tar_zstd) streamer = bbstreamer_zstd_decompressor_new(streamer); } @@ -1415,13 +1352,14 @@ CreateBackupStreamer(char *archive_name, char *spclocation, * manifest if present - as a single COPY stream. */ static void -ReceiveArchiveStream(PGconn *conn) +ReceiveArchiveStream(PGconn *conn, bc_specification *compress) { ArchiveStreamState state; /* Set up initial state. */ memset(&state, 0, sizeof(state)); state.tablespacenum = -1; + state.compress = compress; /* All the real work happens in ReceiveArchiveStreamChunk. */ ReceiveCopyData(conn, ReceiveArchiveStreamChunk, &state); @@ -1542,7 +1480,8 @@ ReceiveArchiveStreamChunk(size_t r, char *copybuf, void *callback_data) CreateBackupStreamer(archive_name, spclocation, &state->manifest_inject_streamer, - true, false); + true, false, + state->compress); } break; } @@ -1743,7 +1682,7 @@ ReportCopyDataParseError(size_t r, char *copybuf) */ static void ReceiveTarFile(PGconn *conn, char *archive_name, char *spclocation, - bool tablespacenum) + bool tablespacenum, bc_specification *compress) { WriteTarState state; bbstreamer *manifest_inject_streamer; @@ -1759,7 +1698,8 @@ ReceiveTarFile(PGconn *conn, char *archive_name, char *spclocation, state.streamer = CreateBackupStreamer(archive_name, spclocation, &manifest_inject_streamer, is_recovery_guc_supported, - expect_unterminated_tarfile); + expect_unterminated_tarfile, + compress); state.tablespacenum = tablespacenum; ReceiveCopyData(conn, ReceiveTarCopyChunk, &state); progress_update_filename(NULL); @@ -1902,7 +1842,8 @@ ReceiveBackupManifestInMemoryChunk(size_t r, char *copybuf, } static void -BaseBackup(void) +BaseBackup(char *compression_algorithm, char *compression_detail, + CompressionLocation compressloc, bc_specification *client_compress) { PGresult *res; char *sysidentifier; @@ -2055,33 +1996,17 @@ BaseBackup(void) if (compressloc == COMPRESS_LOCATION_SERVER) { - char *compressmethodstr = NULL; - if (!use_new_option_syntax) { pg_log_error("server does not support server-side compression"); exit(1); } - switch (compressmethod) - { - case COMPRESSION_GZIP: - compressmethodstr = "gzip"; - break; - case COMPRESSION_LZ4: - compressmethodstr = "lz4"; - break; - case COMPRESSION_ZSTD: - compressmethodstr = "zstd"; - break; - default: - Assert(false); - break; - } AppendStringCommandOption(&buf, use_new_option_syntax, - "COMPRESSION", compressmethodstr); - if (compresslevel >= 1) /* not 0 or Z_DEFAULT_COMPRESSION */ - AppendIntegerCommandOption(&buf, use_new_option_syntax, - "COMPRESSION_LEVEL", compresslevel); + "COMPRESSION", compression_algorithm); + if (compression_detail != NULL) + AppendStringCommandOption(&buf, use_new_option_syntax, + "COMPRESSION_DETAIL", + compression_detail); } if (verbose) @@ -2207,15 +2132,33 @@ BaseBackup(void) */ if (includewal == STREAM_WAL) { + WalCompressionMethod wal_compress_method; + int wal_compress_level; + if (verbose) pg_log_info("starting background WAL receiver"); - StartLogStreamer(xlogstart, starttli, sysidentifier); + + if (client_compress->algorithm == BACKUP_COMPRESSION_GZIP) + { + wal_compress_method = COMPRESSION_GZIP; + wal_compress_level = + (client_compress->options & BACKUP_COMPRESSION_OPTION_LEVEL) + != 0 ? client_compress->level : 0; + } + else + { + wal_compress_method = COMPRESSION_NONE; + wal_compress_level = 0; + } + + StartLogStreamer(xlogstart, starttli, sysidentifier, + wal_compress_method, wal_compress_level); } if (serverMajor >= 1500) { /* Receive a single tar stream with everything. */ - ReceiveArchiveStream(conn); + ReceiveArchiveStream(conn, client_compress); } else { @@ -2244,7 +2187,8 @@ BaseBackup(void) spclocation = PQgetvalue(res, i, 1); } - ReceiveTarFile(conn, archive_name, spclocation, i); + ReceiveTarFile(conn, archive_name, spclocation, i, + client_compress); } /* @@ -2511,6 +2455,10 @@ main(int argc, char **argv) int c; int option_index; + char *compression_algorithm = "none"; + char *compression_detail = NULL; + CompressionLocation compressloc = COMPRESS_LOCATION_UNSPECIFIED; + bc_specification client_compress; pg_logging_init(argv[0]); progname = get_progname(argv[0]); @@ -2616,17 +2564,13 @@ main(int argc, char **argv) do_sync = false; break; case 'z': -#ifdef HAVE_LIBZ - compresslevel = Z_DEFAULT_COMPRESSION; -#else - compresslevel = 1; /* will be rejected below */ -#endif - compressmethod = COMPRESSION_GZIP; + compression_algorithm = "gzip"; + compression_detail = NULL; compressloc = COMPRESS_LOCATION_UNSPECIFIED; break; case 'Z': - parse_compress_options(optarg, &compressmethod, - &compressloc, &compresslevel); + parse_compress_options(optarg, &compression_algorithm, + &compression_detail, &compressloc); break; case 'c': if (pg_strcasecmp(optarg, "fast") == 0) @@ -2753,12 +2697,11 @@ main(int argc, char **argv) } /* - * If we're compressing the backup and the user has not said where to - * perform the compression, do it on the client, unless they specified - * --target, in which case the server is the only choice. + * If the user has not specified where to perform backup compression, + * default to the client, unless the user specified --target, in which case + * the server is the only choice. */ - if (compressmethod != COMPRESSION_NONE && - compressloc == COMPRESS_LOCATION_UNSPECIFIED) + if (compressloc == COMPRESS_LOCATION_UNSPECIFIED) { if (backup_target == NULL) compressloc = COMPRESS_LOCATION_CLIENT; @@ -2766,6 +2709,40 @@ main(int argc, char **argv) compressloc = COMPRESS_LOCATION_SERVER; } + /* + * If any compression that we're doing is happening on the client side, + * we must try to parse the compression algorithm and detail, but if it's + * all on the server side, then we're just going to pass through whatever + * was requested and let the server decide what to do. + */ + if (compressloc == COMPRESS_LOCATION_CLIENT) + { + bc_algorithm alg; + char *error_detail; + + if (!parse_bc_algorithm(compression_algorithm, &alg)) + { + pg_log_error("unrecognized compression algorithm \"%s\"", + compression_algorithm); + exit(1); + } + + parse_bc_specification(alg, compression_detail, &client_compress); + error_detail = validate_bc_specification(&client_compress); + if (error_detail != NULL) + { + pg_log_error("invalid compression specification: %s", + error_detail); + exit(1); + } + } + else + { + Assert(compressloc == COMPRESS_LOCATION_SERVER); + client_compress.algorithm = BACKUP_COMPRESSION_NONE; + client_compress.options = 0; + } + /* * Can't perform client-side compression if the backup is not being * sent to the client. @@ -2779,9 +2756,10 @@ main(int argc, char **argv) } /* - * Compression doesn't make sense unless tar format is in use. + * Client-side compression doesn't make sense unless tar format is in use. */ - if (format == 'p' && compressloc == COMPRESS_LOCATION_CLIENT) + if (format == 'p' && compressloc == COMPRESS_LOCATION_CLIENT && + client_compress.algorithm != BACKUP_COMPRESSION_NONE) { pg_log_error("only tar mode backups can be compressed"); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), @@ -2882,56 +2860,6 @@ main(int argc, char **argv) } } - /* Sanity checks for compression-related options. */ - switch (compressmethod) - { - case COMPRESSION_NONE: - if (compresslevel != 0) - { - pg_log_error("cannot use compression level with method %s", - "none"); - fprintf(stderr, _("Try \"%s --help\" for more information.\n"), - progname); - exit(1); - } - break; - case COMPRESSION_GZIP: - if (compresslevel > 9) - { - pg_log_error("compression level %d of method %s higher than maximum of 9", - compresslevel, "gzip"); - exit(1); - } - if (compressloc == COMPRESS_LOCATION_CLIENT) - { -#ifdef HAVE_LIBZ - if (compresslevel == 0) - compresslevel = Z_DEFAULT_COMPRESSION; -#else - pg_log_error("this build does not support compression with %s", - "gzip"); - exit(1); -#endif - } - break; - case COMPRESSION_LZ4: - if (compresslevel > 12) - { - pg_log_error("compression level %d of method %s higher than maximum of 12", - compresslevel, "lz4"); - exit(1); - } - break; - case COMPRESSION_ZSTD: - if (compresslevel > 22) - { - pg_log_error("compression level %d of method %s higher than maximum of 22", - compresslevel, "zstd"); - exit(1); - } - break; - } - /* * Sanity checks for progress reporting options. */ @@ -3040,7 +2968,8 @@ main(int argc, char **argv) free(linkloc); } - BaseBackup(); + BaseBackup(compression_algorithm, compression_detail, compressloc, + &client_compress); success = true; return 0; diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl index efefe947d9..2869a239e7 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -42,16 +42,12 @@ $node->command_fails(['pg_basebackup'], # Sanity checks for options $node->command_fails_like( [ 'pg_basebackup', '-D', "$tempdir/backup", '--compress', 'none:1' ], - qr/\Qpg_basebackup: error: cannot use compression level with method none/, + qr/\Qcompression algorithm "none" does not accept a compression level/, 'failure if method "none" specified with compression level'); $node->command_fails_like( [ 'pg_basebackup', '-D', "$tempdir/backup", '--compress', 'none+' ], - qr/\Qpg_basebackup: error: invalid value "none+" for option/, + qr/\Qunrecognized compression algorithm "none+"/, 'failure on incorrect separator to define compression level'); -$node->command_fails_like( - [ 'pg_basebackup', '-D', "$tempdir/backup", '--compress', 'none:' ], - qr/\Qpg_basebackup: error: no compression level defined for method none/, - 'failure on missing compression level value'); # Some Windows ANSI code pages may reject this filename, in which case we # quietly proceed without this bit of test coverage. @@ -89,6 +85,70 @@ print $conf "wal_level = replica\n"; close $conf; $node->restart; +# Now that we have a server that supports replication commands, test whether +# certain invalid compression commands fail on the client side with client-side +# compression and on the server side with server-side compression. +my $client_fails = + 'pg_basebackup: error: '; +my $server_fails = + 'pg_basebackup: error: could not initiate base backup: ERROR: '; +my @compression_failure_tests = ( + [ + 'extrasquishy', + 'unrecognized compression algorithm "extrasquishy"', + 'failure on invalid compression algorithm' + ], + [ + 'gzip:', + 'invalid compression specification: found empty string where a compression option was expected', + 'failure on empty compression options list' + ], + [ + 'gzip:thunk', + 'invalid compression specification: unknown compression option "thunk"', + 'failure on unknown compression option' + ], + [ + 'gzip:level', + 'invalid compression specification: compression option "level" requires a value', + 'failure on missing compression level' + ], + [ + 'gzip:level=', + 'invalid compression specification: value for compression option "level" must be an integer', + 'failure on empty compression level' + ], + [ + 'gzip:level=high', + 'invalid compression specification: value for compression option "level" must be an integer', + 'failure on non-numeric compression level' + ], + [ + 'gzip:level=236', + 'invalid compression specification: compression algorithm "gzip" expects a compression level between 1 and 9', + 'failure on out-of-range compression level' + ], + [ + 'gzip:level=9,', + 'invalid compression specification: found empty string where a compression option was expected', + 'failure on extra, empty compression option' + ], +); +for my $cft (@compression_failure_tests) +{ + my $cfail = quotemeta($client_fails . $cft->[1]); + my $sfail = quotemeta($server_fails . $cft->[1]); + $node->command_fails_like( + [ 'pg_basebackup', '-D', "$tempdir/backup", '--compress', $cft->[0] ], + qr/$cfail/, + 'client '. $cft->[2]); + $node->command_fails_like( + [ 'pg_basebackup', '-D', "$tempdir/backup", '--compress', + 'server-' . $cft->[0] ], + qr/$sfail/, + 'server ' . $cft->[2]); +} + # Write some files to test that they are not copied. foreach my $filename ( qw(backup_label tablespace_map postgresql.auto.conf.tmp diff --git a/src/common/Makefile b/src/common/Makefile index 31c0dd366d..f627349835 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -47,6 +47,7 @@ LIBS += $(PTHREAD_LIBS) OBJS_COMMON = \ archive.o \ + backup_compression.o \ base64.o \ checksum_helper.o \ config_info.o \ diff --git a/src/common/backup_compression.c b/src/common/backup_compression.c new file mode 100644 index 0000000000..591b97a60c --- /dev/null +++ b/src/common/backup_compression.c @@ -0,0 +1,269 @@ +/*------------------------------------------------------------------------- + * + * backup_compression.c + * + * Shared code for backup compression methods and specifications. + * + * A compression specification specifies the parameters that should be used + * when performing compression with a specific algorithm. The simplest + * possible compression specification is an integer, which sets the + * compression level. + * + * Otherwise, a compression specification is a comma-separated list of items, + * each having the form keyword or keyword=value. + * + * Currently, the only supported keyword is "level". + * + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/common/backup_compression.c + *------------------------------------------------------------------------- + */ + +#ifndef FRONTEND +#include "postgres.h" +#else +#include "postgres_fe.h" +#endif + +#include "common/backup_compression.h" + +static int expect_integer_value(char *keyword, char *value, + bc_specification *result); + +/* + * Look up a compression algorithm by name. Returns true and sets *algorithm + * if the name is recognized. Otherwise returns false. + */ +bool +parse_bc_algorithm(char *name, bc_algorithm *algorithm) +{ + if (strcmp(name, "none") == 0) + *algorithm = BACKUP_COMPRESSION_NONE; + else if (strcmp(name, "gzip") == 0) + *algorithm = BACKUP_COMPRESSION_GZIP; + else if (strcmp(name, "lz4") == 0) + *algorithm = BACKUP_COMPRESSION_LZ4; + else if (strcmp(name, "zstd") == 0) + *algorithm = BACKUP_COMPRESSION_ZSTD; + else + return false; + return true; +} + +/* + * Get the human-readable name corresponding to a particular compression + * algorithm. + */ +const char * +get_bc_algorithm_name(bc_algorithm algorithm) +{ + switch (algorithm) + { + case BACKUP_COMPRESSION_NONE: + return "none"; + case BACKUP_COMPRESSION_GZIP: + return "gzip"; + case BACKUP_COMPRESSION_LZ4: + return "lz4"; + case BACKUP_COMPRESSION_ZSTD: + return "zstd"; + /* no default, to provoke compiler warnings if values are added */ + } + Assert(false); +} + +/* + * Parse a compression specification for a specified algorithm. + * + * See the file header comments for a brief description of what a compression + * specification is expected to look like. + * + * On return, all fields of the result object will be initialized. + * In particular, result->parse_error will be NULL if no errors occurred + * during parsing, and will otherwise contain an appropriate error message. + * The caller may free this error message string using pfree, if desired. + * Note, however, even if there's no parse error, the string might not make + * sense: e.g. for gzip, level=12 is not sensible, but it does parse OK. + * + * Use validate_bc_specification() to find out whether a compression + * specification is semantically sensible. + */ +void +parse_bc_specification(bc_algorithm algorithm, char *specification, + bc_specification *result) +{ + int bare_level; + char *bare_level_endp; + + /* Initial setup of result object. */ + result->algorithm = algorithm; + result->options = 0; + result->level = -1; + result->parse_error = NULL; + + /* If there is no specification, we're done already. */ + if (specification == NULL) + return; + + /* As a special case, the specification can be a bare integer. */ + bare_level = strtol(specification, &bare_level_endp, 10); + if (specification != bare_level_endp && *bare_level_endp == '\0') + { + result->level = bare_level; + result->options |= BACKUP_COMPRESSION_OPTION_LEVEL; + return; + } + + /* Look for comma-separated keyword or keyword=value entries. */ + while (1) + { + char *kwstart; + char *kwend; + char *vstart; + char *vend; + int kwlen; + int vlen; + bool has_value; + char *keyword; + char *value; + + /* Figure start, end, and length of next keyword and any value. */ + kwstart = kwend = specification; + while (*kwend != '\0' && *kwend != ',' && *kwend != '=') + ++kwend; + kwlen = kwend - kwstart; + if (*kwend != '=') + { + vstart = vend = NULL; + vlen = 0; + has_value = false; + } + else + { + vstart = vend = kwend + 1; + while (*vend != '\0' && *vend != ',') + ++vend; + vlen = vend - vstart; + has_value = true; + } + + /* Reject empty keyword. */ + if (kwlen == 0) + { + result->parse_error = + pstrdup(_("found empty string where a compression option was expected")); + break; + } + + /* Extract keyword and value as separate C strings. */ + keyword = palloc(kwlen + 1); + memcpy(keyword, kwstart, kwlen); + keyword[kwlen] = '\0'; + if (!has_value) + value = NULL; + else + { + value = palloc(vlen + 1); + memcpy(value, vstart, vlen); + value[vlen] = '\0'; + } + + /* Handle whatever keyword we found. */ + if (strcmp(keyword, "level") == 0) + { + result->level = expect_integer_value(keyword, value, result); + result->options |= BACKUP_COMPRESSION_OPTION_LEVEL; + } + else + result->parse_error = + psprintf(_("unknown compression option \"%s\""), keyword); + + /* Release memory, just to be tidy. */ + pfree(keyword); + if (value != NULL) + pfree(value); + + /* If we got an error or have reached the end of the string, stop. */ + if (result->parse_error != NULL || *kwend == '\0' || *vend == '\0') + break; + + /* Advance to next entry and loop around. */ + specification = vend == NULL ? kwend + 1 : vend + 1; + } +} + +/* + * Parse 'value' as an integer and return the result. + * + * If parsing fails, set result->parse_error to an appropriate message + * and return -1. + */ +static int +expect_integer_value(char *keyword, char *value, bc_specification *result) +{ + int ivalue; + char *ivalue_endp; + + if (value == NULL) + { + result->parse_error = + psprintf(_("compression option \"%s\" requires a value"), + keyword); + return -1; + } + + ivalue = strtol(value, &ivalue_endp, 10); + if (ivalue_endp == value || *ivalue_endp != '\0') + { + result->parse_error = + psprintf(_("value for compression option \"%s\" must be an integer"), + keyword); + return -1; + } + return ivalue; +} + +/* + * Returns NULL if the compression specification string was syntactically + * valid and semantically sensible. Otherwise, returns an error message. + * + * Does not test whether this build of PostgreSQL supports the requested + * compression method. + */ +char * +validate_bc_specification(bc_specification *spec) +{ + /* If it didn't even parse OK, it's definitely no good. */ + if (spec->parse_error != NULL) + return spec->parse_error; + + /* + * If a compression level was specified, check that the algorithm expects + * a compression level and that the level is within the legal range for + * the algorithm. + */ + if ((spec->options & BACKUP_COMPRESSION_OPTION_LEVEL) != 0) + { + int min_level = 1; + int max_level; + + if (spec->algorithm == BACKUP_COMPRESSION_GZIP) + max_level = 9; + else if (spec->algorithm == BACKUP_COMPRESSION_LZ4) + max_level = 12; + else if (spec->algorithm == BACKUP_COMPRESSION_ZSTD) + max_level = 22; + else + return psprintf(_("compression algorithm \"%s\" does not accept a compression level"), + get_bc_algorithm_name(spec->algorithm)); + + if (spec->level < min_level || spec->level > max_level) + return psprintf(_("compression algorithm \"%s\" expects a compression level between %d and %d"), + get_bc_algorithm_name(spec->algorithm), + min_level, max_level); + } + + return NULL; +} diff --git a/src/include/common/backup_compression.h b/src/include/common/backup_compression.h new file mode 100644 index 0000000000..0565cbc657 --- /dev/null +++ b/src/include/common/backup_compression.h @@ -0,0 +1,44 @@ +/*------------------------------------------------------------------------- + * + * backup_compression.h + * + * Shared definitions for backup compression methods and specifications. + * + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/common/backup_compression.h + *------------------------------------------------------------------------- + */ + +#ifndef BACKUP_COMPRESSION_H +#define BACKUP_COMPRESSION_H + +typedef enum bc_algorithm +{ + BACKUP_COMPRESSION_NONE, + BACKUP_COMPRESSION_GZIP, + BACKUP_COMPRESSION_LZ4, + BACKUP_COMPRESSION_ZSTD +} bc_algorithm; + +#define BACKUP_COMPRESSION_OPTION_LEVEL (1 << 0) + +typedef struct bc_specification +{ + bc_algorithm algorithm; + unsigned options; /* OR of BACKUP_COMPRESSION_OPTION constants */ + int level; + char *parse_error; /* NULL if parsing was OK, else message */ +} bc_specification; + +extern bool parse_bc_algorithm(char *name, bc_algorithm *algorithm); +extern const char *get_bc_algorithm_name(bc_algorithm algorithm); + +extern void parse_bc_specification(bc_algorithm algorithm, + char *specification, + bc_specification *result); + +extern char *validate_bc_specification(bc_specification *); + +#endif diff --git a/src/include/replication/basebackup_sink.h b/src/include/replication/basebackup_sink.h index a7f16758a4..654df28576 100644 --- a/src/include/replication/basebackup_sink.h +++ b/src/include/replication/basebackup_sink.h @@ -27,6 +27,7 @@ #define BASEBACKUP_SINK_H #include "access/xlog_internal.h" +#include "common/backup_compression.h" #include "nodes/pg_list.h" /* Forward declarations. */ @@ -283,9 +284,9 @@ extern void bbsink_forward_cleanup(bbsink *sink); /* Constructors for various types of sinks. */ extern bbsink *bbsink_copystream_new(bool send_to_client); -extern bbsink *bbsink_gzip_new(bbsink *next, int compresslevel); -extern bbsink *bbsink_lz4_new(bbsink *next, int compresslevel); -extern bbsink *bbsink_zstd_new(bbsink *next, int compresslevel); +extern bbsink *bbsink_gzip_new(bbsink *next, bc_specification *); +extern bbsink *bbsink_lz4_new(bbsink *next, bc_specification *); +extern bbsink *bbsink_zstd_new(bbsink *next, bc_specification *); extern bbsink *bbsink_progress_new(bbsink *next, bool estimate_backup_size); extern bbsink *bbsink_server_new(bbsink *next, char *pathname); extern bbsink *bbsink_throttle_new(bbsink *next, uint32 maxrate); diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 441d6ae6bf..de8676d339 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -124,7 +124,7 @@ sub mkvcbuild } our @pgcommonallfiles = qw( - archive.c base64.c checksum_helper.c + archive.c backup_compression.c base64.c checksum_helper.c config_info.c controldata_utils.c d2s.c encnames.c exec.c f2s.c file_perm.c file_utils.c hashfn.c ip.c jsonapi.c keywords.c kwlookup.c link-canary.c md5_common.c