From 0eaf453ebf6788885fbb5d40426b154ef8805407 Mon Sep 17 00:00:00 2001 From: Raphael Pour Date: Fri, 15 May 2020 08:36:07 +0200 Subject: [PATCH 1/6] qemu-nbd: Close inherited stderr Close inherited stderr of the parent if fork_process is false. Otherwise no one will close it. (introduced by e6df58a5) This only affected 'qemu-nbd -c /dev/nbd0'. Signed-off-by: Raphael Pour Message-Id: Reviewed-by: Eric Blake [eblake: Enhance commit message] Signed-off-by: Eric Blake --- qemu-nbd.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/qemu-nbd.c b/qemu-nbd.c index 4aa005004e..306e44fb0a 100644 --- a/qemu-nbd.c +++ b/qemu-nbd.c @@ -916,7 +916,11 @@ int main(int argc, char **argv) } else if (pid == 0) { close(stderr_fd[0]); - old_stderr = dup(STDERR_FILENO); + /* Remember parent's stderr if we will be restoring it. */ + if (fork_process) { + old_stderr = dup(STDERR_FILENO); + } + ret = qemu_daemon(1, 0); /* Temporarily redirect stderr to the parent's pipe... */ From 43d589b074370ebc9b340340b5f641b385da9df8 Mon Sep 17 00:00:00 2001 From: Eyal Moscovici Date: Wed, 13 May 2020 16:36:26 +0300 Subject: [PATCH 2/6] qemu_img: add cvtnum_full to print error reports All calls to cvtnum check the return value and print the same error message more or less. And so error reporting moved to cvtnum_full to reduce code duplication and provide a single error message. Additionally, cvtnum now wraps cvtnum_full with the existing default range of 0 to MAX_INT64. Acked-by: Mark Kanda Signed-off-by: Eyal Moscovici Message-Id: <20200513133629.18508-2-eyal.moscovici@oracle.com> Reviewed-by: Eric Blake [eblake: fix printf formatting, avoid trailing space, change error wording, reformat commit message] Signed-off-by: Eric Blake --- qemu-img.c | 74 +++++++++++++++++--------------------- tests/qemu-iotests/049.out | 8 ++--- 2 files changed, 37 insertions(+), 45 deletions(-) diff --git a/qemu-img.c b/qemu-img.c index 6a4327aaba..5d824fc15f 100644 --- a/qemu-img.c +++ b/qemu-img.c @@ -470,19 +470,31 @@ static int add_old_style_options(const char *fmt, QemuOpts *opts, return 0; } -static int64_t cvtnum(const char *s) +static int64_t cvtnum_full(const char *name, const char *value, int64_t min, + int64_t max) { int err; - uint64_t value; + uint64_t res; - err = qemu_strtosz(s, NULL, &value); - if (err < 0) { + err = qemu_strtosz(value, NULL, &res); + if (err < 0 && err != -ERANGE) { + error_report("Invalid %s specified. You may use " + "k, M, G, T, P or E suffixes for", name); + error_report("kilobytes, megabytes, gigabytes, terabytes, " + "petabytes and exabytes."); return err; } - if (value > INT64_MAX) { + if (err == -ERANGE || res > max || res < min) { + error_report("Invalid %s specified. Must be between %" PRId64 + " and %" PRId64 ".", name, min, max); return -ERANGE; } - return value; + return res; +} + +static int64_t cvtnum(const char *name, const char *value) +{ + return cvtnum_full(name, value, 0, INT64_MAX); } static int img_create(int argc, char **argv) @@ -572,16 +584,8 @@ static int img_create(int argc, char **argv) if (optind < argc) { int64_t sval; - sval = cvtnum(argv[optind++]); + sval = cvtnum("image size", argv[optind++]); if (sval < 0) { - if (sval == -ERANGE) { - error_report("Image size must be less than 8 EiB!"); - } else { - error_report("Invalid image size specified! You may use k, M, " - "G, T, P or E suffixes for "); - error_report("kilobytes, megabytes, gigabytes, terabytes, " - "petabytes and exabytes."); - } goto fail; } img_size = (uint64_t)sval; @@ -2187,8 +2191,10 @@ static int img_convert(int argc, char **argv) { int64_t sval; - sval = cvtnum(optarg); - if (sval < 0 || !QEMU_IS_ALIGNED(sval, BDRV_SECTOR_SIZE) || + sval = cvtnum("buffer size for sparse output", optarg); + if (sval < 0) { + goto fail_getopt; + } else if (!QEMU_IS_ALIGNED(sval, BDRV_SECTOR_SIZE) || sval / BDRV_SECTOR_SIZE > MAX_BUF_SECTORS) { error_report("Invalid buffer size for sparse output specified. " "Valid sizes are multiples of %llu up to %llu. Select " @@ -4291,9 +4297,8 @@ static int img_bench(int argc, char **argv) break; case 'o': { - offset = cvtnum(optarg); + offset = cvtnum("offset", optarg); if (offset < 0) { - error_report("Invalid offset specified"); return 1; } break; @@ -4306,9 +4311,8 @@ static int img_bench(int argc, char **argv) { int64_t sval; - sval = cvtnum(optarg); - if (sval < 0 || sval > INT_MAX) { - error_report("Invalid buffer size specified"); + sval = cvtnum_full("buffer size", optarg, 0, INT_MAX); + if (sval < 0) { return 1; } @@ -4319,9 +4323,8 @@ static int img_bench(int argc, char **argv) { int64_t sval; - sval = cvtnum(optarg); - if (sval < 0 || sval > INT_MAX) { - error_report("Invalid step size specified"); + sval = cvtnum_full("step_size", optarg, 0, INT_MAX); + if (sval < 0) { return 1; } @@ -4491,10 +4494,9 @@ static int img_dd_bs(const char *arg, { int64_t res; - res = cvtnum(arg); + res = cvtnum_full("bs", arg, 1, INT_MAX); - if (res <= 0 || res > INT_MAX) { - error_report("invalid number: '%s'", arg); + if (res < 0) { return 1; } in->bsz = out->bsz = res; @@ -4506,10 +4508,9 @@ static int img_dd_count(const char *arg, struct DdIo *in, struct DdIo *out, struct DdInfo *dd) { - dd->count = cvtnum(arg); + dd->count = cvtnum("count", arg); if (dd->count < 0) { - error_report("invalid number: '%s'", arg); return 1; } @@ -4538,10 +4539,9 @@ static int img_dd_skip(const char *arg, struct DdIo *in, struct DdIo *out, struct DdInfo *dd) { - in->offset = cvtnum(arg); + in->offset = cvtnum("skip", arg); if (in->offset < 0) { - error_report("invalid number: '%s'", arg); return 1; } @@ -4923,16 +4923,8 @@ static int img_measure(int argc, char **argv) { int64_t sval; - sval = cvtnum(optarg); + sval = cvtnum("image size", optarg); if (sval < 0) { - if (sval == -ERANGE) { - error_report("Image size must be less than 8 EiB!"); - } else { - error_report("Invalid image size specified! You may use " - "k, M, G, T, P or E suffixes for "); - error_report("kilobytes, megabytes, gigabytes, terabytes, " - "petabytes and exabytes."); - } goto out; } img_size = (uint64_t)sval; diff --git a/tests/qemu-iotests/049.out b/tests/qemu-iotests/049.out index a5cfba1756..c54ae21b86 100644 --- a/tests/qemu-iotests/049.out +++ b/tests/qemu-iotests/049.out @@ -92,19 +92,19 @@ Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1649267441664 cluster_size=65536 l == 3. Invalid sizes == qemu-img create -f qcow2 TEST_DIR/t.qcow2 -- -1024 -qemu-img: Image size must be less than 8 EiB! +qemu-img: Invalid image size specified. Must be between 0 and 9223372036854775807. qemu-img create -f qcow2 -o size=-1024 TEST_DIR/t.qcow2 qemu-img: TEST_DIR/t.qcow2: Value '-1024' is out of range for parameter 'size' qemu-img create -f qcow2 TEST_DIR/t.qcow2 -- -1k -qemu-img: Image size must be less than 8 EiB! +qemu-img: Invalid image size specified. Must be between 0 and 9223372036854775807. qemu-img create -f qcow2 -o size=-1k TEST_DIR/t.qcow2 qemu-img: TEST_DIR/t.qcow2: Value '-1k' is out of range for parameter 'size' qemu-img create -f qcow2 TEST_DIR/t.qcow2 -- 1kilobyte -qemu-img: Invalid image size specified! You may use k, M, G, T, P or E suffixes for +qemu-img: Invalid image size specified. You may use k, M, G, T, P or E suffixes for qemu-img: kilobytes, megabytes, gigabytes, terabytes, petabytes and exabytes. qemu-img create -f qcow2 -o size=1kilobyte TEST_DIR/t.qcow2 @@ -113,7 +113,7 @@ Optional suffix k, M, G, T, P or E means kilo-, mega-, giga-, tera-, peta- and exabytes, respectively. qemu-img create -f qcow2 TEST_DIR/t.qcow2 -- foobar -qemu-img: Invalid image size specified! You may use k, M, G, T, P or E suffixes for +qemu-img: Invalid image size specified. You may use k, M, G, T, P or E suffixes for qemu-img: kilobytes, megabytes, gigabytes, terabytes, petabytes and exabytes. qemu-img create -f qcow2 -o size=foobar TEST_DIR/t.qcow2 From 8f282e83edd3d1b4ea6e9258f5a4081b490c33cc Mon Sep 17 00:00:00 2001 From: Eyal Moscovici Date: Wed, 13 May 2020 16:36:27 +0300 Subject: [PATCH 3/6] qemu-img: validate image length in img_map The code handles this case correctly: we merely skip the loop. However it is probably best to return an explicit error. Reviewed-by: Eric Blake Acked-by: Mark Kanda Signed-off-by: Eyal Moscovici Message-Id: <20200513133629.18508-3-eyal.moscovici@oracle.com> [eblake: commit message tweak] Signed-off-by: Eric Blake --- qemu-img.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qemu-img.c b/qemu-img.c index 5d824fc15f..c88f412333 100644 --- a/qemu-img.c +++ b/qemu-img.c @@ -3092,6 +3092,11 @@ static int img_map(int argc, char **argv) } length = blk_getlength(blk); + if (length < 0) { + error_report("Failed to get size for '%s'", filename); + return 1; + } + while (curr.start + curr.length < length) { int64_t offset = curr.start + curr.length; int64_t n; From e46c0b18cfd02195a0d527ca73f3ed9f3ce5eacb Mon Sep 17 00:00:00 2001 From: Eyal Moscovici Date: Wed, 13 May 2020 16:36:28 +0300 Subject: [PATCH 4/6] qemu-img: refactor dump_map_entry JSON format output Previously dump_map_entry identified whether we need to start a new JSON array based on whether start address == 0. In this refactor we remove this assumption as in following patches we will allow map to start from an arbitrary position. Reviewed-by: Eric Blake Acked-by: Mark Kanda Signed-off-by: Eyal Moscovici Message-Id: <20200513133629.18508-4-eyal.moscovici@oracle.com> Signed-off-by: Eric Blake --- qemu-img.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/qemu-img.c b/qemu-img.c index c88f412333..4aa9414aba 100644 --- a/qemu-img.c +++ b/qemu-img.c @@ -2902,9 +2902,8 @@ static int dump_map_entry(OutputFormat output_format, MapEntry *e, } break; case OFORMAT_JSON: - printf("%s{ \"start\": %"PRId64", \"length\": %"PRId64"," + printf("{ \"start\": %"PRId64", \"length\": %"PRId64"," " \"depth\": %"PRId64", \"zero\": %s, \"data\": %s", - (e->start == 0 ? "[" : ",\n"), e->start, e->length, e->depth, e->zero ? "true" : "false", e->data ? "true" : "false"); @@ -2913,8 +2912,8 @@ static int dump_map_entry(OutputFormat output_format, MapEntry *e, } putchar('}'); - if (!next) { - printf("]\n"); + if (next) { + puts(","); } break; } @@ -3089,6 +3088,8 @@ static int img_map(int argc, char **argv) if (output_format == OFORMAT_HUMAN) { printf("%-16s%-16s%-16s%s\n", "Offset", "Length", "Mapped to", "File"); + } else if (output_format == OFORMAT_JSON) { + putchar('['); } length = blk_getlength(blk); @@ -3125,6 +3126,9 @@ static int img_map(int argc, char **argv) } ret = dump_map_entry(output_format, &curr, NULL); + if (output_format == OFORMAT_JSON) { + puts("]"); + } out: blk_unref(blk); From c0469496b32910a6a092d5b614efbf4088b13a29 Mon Sep 17 00:00:00 2001 From: Eyal Moscovici Date: Wed, 13 May 2020 16:36:29 +0300 Subject: [PATCH 5/6] qemu-img: Add --start-offset and --max-length to map The mapping operation of large disks especially ones stored over a long chain of QCOW2 files can take a long time to finish. Additionally when mapping fails there was no way recover by restarting the mapping from the failed location. The new options, --start-offset and --max-length allows the user to divide these type of map operations into shorter independent tasks. Reviewed-by: Eric Blake Acked-by: Mark Kanda Co-developed-by: Yoav Elnekave Signed-off-by: Yoav Elnekave Signed-off-by: Eyal Moscovici Message-Id: <20200513133629.18508-5-eyal.moscovici@oracle.com> Signed-off-by: Eric Blake --- docs/tools/qemu-img.rst | 2 +- qemu-img-cmds.hx | 4 ++-- qemu-img.c | 22 +++++++++++++++++++++- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/docs/tools/qemu-img.rst b/docs/tools/qemu-img.rst index 0080f83a76..f4ffe528ea 100644 --- a/docs/tools/qemu-img.rst +++ b/docs/tools/qemu-img.rst @@ -519,7 +519,7 @@ Command description: ``ImageInfoSpecific*`` QAPI object (e.g. ``ImageInfoSpecificQCow2`` for qcow2 images). -.. option:: map [--object OBJECTDEF] [--image-opts] [-f FMT] [--output=OFMT] [-U] FILENAME +.. option:: map [--object OBJECTDEF] [--image-opts] [-f FMT] [--start-offset=OFFSET] [--max-length=LEN] [--output=OFMT] [-U] FILENAME Dump the metadata of image *FILENAME* and its backing file chain. In particular, this commands dumps the allocation state of every sector diff --git a/qemu-img-cmds.hx b/qemu-img-cmds.hx index c9c54de1df..35f832816f 100644 --- a/qemu-img-cmds.hx +++ b/qemu-img-cmds.hx @@ -63,9 +63,9 @@ SRST ERST DEF("map", img_map, - "map [--object objectdef] [--image-opts] [-f fmt] [--output=ofmt] [-U] filename") + "map [--object objectdef] [--image-opts] [-f fmt] [--start-offset=offset] [--max-length=len] [--output=ofmt] [-U] filename") SRST -.. option:: map [--object OBJECTDEF] [--image-opts] [-f FMT] [--output=OFMT] [-U] FILENAME +.. option:: map [--object OBJECTDEF] [--image-opts] [-f FMT] [--start-offset=OFFSET] [--max-length=LEN] [--output=OFMT] [-U] FILENAME ERST DEF("measure", img_measure, diff --git a/qemu-img.c b/qemu-img.c index 4aa9414aba..947bf8b34b 100644 --- a/qemu-img.c +++ b/qemu-img.c @@ -3009,6 +3009,8 @@ static int img_map(int argc, char **argv) int ret = 0; bool image_opts = false; bool force_share = false; + int64_t start_offset = 0; + int64_t max_length = -1; fmt = NULL; output = NULL; @@ -3021,9 +3023,11 @@ static int img_map(int argc, char **argv) {"object", required_argument, 0, OPTION_OBJECT}, {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS}, {"force-share", no_argument, 0, 'U'}, + {"start-offset", required_argument, 0, 's'}, + {"max-length", required_argument, 0, 'l'}, {0, 0, 0, 0} }; - c = getopt_long(argc, argv, ":f:hU", + c = getopt_long(argc, argv, ":f:s:l:hU", long_options, &option_index); if (c == -1) { break; @@ -3047,6 +3051,18 @@ static int img_map(int argc, char **argv) case OPTION_OUTPUT: output = optarg; break; + case 's': + start_offset = cvtnum("start offset", optarg); + if (start_offset < 0) { + return 1; + } + break; + case 'l': + max_length = cvtnum("max length", optarg); + if (max_length < 0) { + return 1; + } + break; case OPTION_OBJECT: { QemuOpts *opts; opts = qemu_opts_parse_noisily(&qemu_object_opts, @@ -3097,7 +3113,11 @@ static int img_map(int argc, char **argv) error_report("Failed to get size for '%s'", filename); return 1; } + if (max_length != -1) { + length = MIN(start_offset + max_length, length); + } + curr.start = start_offset; while (curr.start + curr.length < length) { int64_t offset = curr.start + curr.length; int64_t n; From d8154b0945f795177511ea0e2212bd5c749fe84c Mon Sep 17 00:00:00 2001 From: Eric Blake Date: Wed, 13 May 2020 12:58:34 -0500 Subject: [PATCH 6/6] iotests: Enhance 223 to cover qemu-img map improvements Since qemu-img map + x-dirty-bitmap remains the easiest way to read persistent bitmaps at the moment, it makes a reasonable place to add coverage to ensure we do not regress on the just-added parameters to qemu-img map. Signed-off-by: Eric Blake Message-Id: <20200513181455.295267-1-eblake@redhat.com> --- tests/qemu-iotests/223 | 6 ++++-- tests/qemu-iotests/223.out | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/qemu-iotests/223 b/tests/qemu-iotests/223 index 56fbc5fb09..d68bc3cb6f 100755 --- a/tests/qemu-iotests/223 +++ b/tests/qemu-iotests/223 @@ -2,7 +2,7 @@ # # Test reading dirty bitmap over NBD # -# Copyright (C) 2018-2019 Red Hat, Inc. +# Copyright (C) 2018-2020 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -206,7 +206,9 @@ $QEMU_IMG map --output=json --image-opts \ nbd_server_start_unix_socket -f $IMGFMT -B b2 "$TEST_IMG" IMG="driver=nbd,server.type=unix,server.path=$nbd_unix_socket" -$QEMU_IMG map --output=json --image-opts \ +$QEMU_IMG map --output=json --image-opts --max-length=12345 \ + "$IMG,x-dirty-bitmap=qemu:dirty-bitmap:b2" | _filter_qemu_img_map +$QEMU_IMG map --output=json --image-opts --start-offset=12345 \ "$IMG,x-dirty-bitmap=qemu:dirty-bitmap:b2" | _filter_qemu_img_map # success, all done diff --git a/tests/qemu-iotests/223.out b/tests/qemu-iotests/223.out index 80c0cf6509..e1eaaedb55 100644 --- a/tests/qemu-iotests/223.out +++ b/tests/qemu-iotests/223.out @@ -201,6 +201,7 @@ read 2097152/2097152 bytes at offset 2097152 { "start": 2097152, "length": 2097152, "depth": 0, "zero": false, "data": false}] [{ "start": 0, "length": 512, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, { "start": 512, "length": 512, "depth": 0, "zero": false, "data": false}, -{ "start": 1024, "length": 2096128, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, +{ "start": 1024, "length": 11321, "depth": 0, "zero": false, "data": true, "offset": OFFSET}] +[{ "start": 12345, "length": 2084807, "depth": 0, "zero": false, "data": true, "offset": OFFSET}, { "start": 2097152, "length": 2097152, "depth": 0, "zero": false, "data": false}] *** done