Block patches:

- drop block/io write notifiers
 - qemu-iotests enhancements to make debugging easier
 - rbd parsing fix
 - HMP qemu-io fix (for iothreads)
 - mirror job cancel relaxation (do not cancel in-flight requests when a
   READY mirror job is canceled with force=false)
 - document qcow2's data_file and data_file_raw features
 - fix iotest 297 for pylint 2.8
 - block/copy-on-read refactoring
 -----BEGIN PGP SIGNATURE-----
 
 iQFGBAABCAAwFiEEkb62CjDbPohX0Rgp9AfbAGHVz0AFAmCeqLwSHG1yZWl0ekBy
 ZWRoYXQuY29tAAoJEPQH2wBh1c9A9pMIAKYIlLQfSSMdy0fZ+6AHiAjaTZAaDr4G
 d6NDz/RONZEoxcl01LkUWJfvqH/IdCLx5q4cl9SU4+JzMdKW9K1xBLdAGousuhk/
 geYqymbORj/VntJDYwp30KUlC0pLUBbuuzYN+QXrLp5qJvS9nPBcxEPjfSc6GX9z
 Bt+GCRW08+C4WKJ3lGu9zNGe47gTFUE/VodUYG4tKg5xZFzsAWd/PZZaVOdW0fCz
 /0tdxN4N82XT+cE/lA0Tm6B6L3ZueMAt8byu4BPz21M7kULNn2roVMiFKJELZlZQ
 0RyDXH2jb/aH/ha6gJ4S+JhMvq45rH9GuQeAYl6IPngbta+NbZW+U4w=
 =+Kha
 -----END PGP SIGNATURE-----

Merge remote-tracking branch 'remotes/maxreitz/tags/pull-block-2021-05-14' into staging

Block patches:
- drop block/io write notifiers
- qemu-iotests enhancements to make debugging easier
- rbd parsing fix
- HMP qemu-io fix (for iothreads)
- mirror job cancel relaxation (do not cancel in-flight requests when a
  READY mirror job is canceled with force=false)
- document qcow2's data_file and data_file_raw features
- fix iotest 297 for pylint 2.8
- block/copy-on-read refactoring

# gpg: Signature made Fri 14 May 2021 17:43:40 BST
# gpg:                using RSA key 91BEB60A30DB3E8857D11829F407DB0061D5CF40
# gpg:                issuer "mreitz@redhat.com"
# gpg: Good signature from "Max Reitz <mreitz@redhat.com>" [full]
# Primary key fingerprint: 91BE B60A 30DB 3E88 57D1  1829 F407 DB00 61D5 CF40

* remotes/maxreitz/tags/pull-block-2021-05-14:
  write-threshold: deal with includes
  test-write-threshold: drop extra TestStruct structure
  test-write-threshold: drop extra tests
  block/write-threshold: drop extra APIs
  test-write-threshold: rewrite test_threshold_(not_)trigger tests
  block: drop write notifiers
  block/write-threshold: don't use write notifiers
  qemu-iotests: fix pylint 2.8 consider-using-with error
  block/copy-on-read: use bdrv_drop_filter() and drop s->active
  Document qemu-img options data_file and data_file_raw
  qemu-iotests: fix case of SOCK_DIR already in the environment
  qemu-iotests: let "check" spawn an arbitrary test command
  qemu-iotests: move command line and environment handling from TestRunner to TestEnv
  qemu-iotests: allow passing unittest.main arguments to the test scripts
  qemu-iotests: do not buffer the test output
  mirror: stop cancelling in-flight requests on non-force cancel in READY
  monitor: hmp_qemu_io: acquire aio contex, fix crash
  block/rbd: Add an escape-aware strchr helper
  iotests/231: Update expected deprecation message

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
This commit is contained in:
Peter Maydell 2021-05-17 11:29:59 +01:00
commit 32de74a1ac
28 changed files with 288 additions and 378 deletions

View File

@ -400,7 +400,6 @@ BlockDriverState *bdrv_new(void)
for (i = 0; i < BLOCK_OP_TYPE_MAX; i++) {
QLIST_INIT(&bs->op_blockers[i]);
}
notifier_with_return_list_init(&bs->before_write_notifiers);
qemu_co_mutex_init(&bs->reqs_lock);
qemu_mutex_init(&bs->dirty_bitmap_mutex);
bs->refcnt = 1;

View File

@ -331,7 +331,7 @@ static void coroutine_fn backup_set_speed(BlockJob *job, int64_t speed)
}
}
static void backup_cancel(Job *job)
static void backup_cancel(Job *job, bool force)
{
BackupBlockJob *s = container_of(job, BackupBlockJob, common.job);

View File

@ -29,7 +29,6 @@
typedef struct BDRVStateCOR {
bool active;
BlockDriverState *bottom_bs;
bool chain_frozen;
} BDRVStateCOR;
@ -89,7 +88,6 @@ static int cor_open(BlockDriverState *bs, QDict *options, int flags,
*/
bdrv_ref(bottom_bs);
}
state->active = true;
state->bottom_bs = bottom_bs;
/*
@ -112,17 +110,6 @@ static void cor_child_perm(BlockDriverState *bs, BdrvChild *c,
uint64_t perm, uint64_t shared,
uint64_t *nperm, uint64_t *nshared)
{
BDRVStateCOR *s = bs->opaque;
if (!s->active) {
/*
* While the filter is being removed
*/
*nperm = 0;
*nshared = BLK_PERM_ALL;
return;
}
*nperm = perm & PERM_PASSTHROUGH;
*nshared = (shared & PERM_PASSTHROUGH) | PERM_UNCHANGED;
@ -280,32 +267,14 @@ static BlockDriver bdrv_copy_on_read = {
void bdrv_cor_filter_drop(BlockDriverState *cor_filter_bs)
{
BdrvChild *child;
BlockDriverState *bs;
BDRVStateCOR *s = cor_filter_bs->opaque;
child = bdrv_filter_child(cor_filter_bs);
if (!child) {
return;
}
bs = child->bs;
/* Retain the BDS until we complete the graph change. */
bdrv_ref(bs);
/* Hold a guest back from writing while permissions are being reset. */
bdrv_drained_begin(bs);
/* Drop permissions before the graph change. */
s->active = false;
/* unfreeze, as otherwise bdrv_replace_node() will fail */
if (s->chain_frozen) {
s->chain_frozen = false;
bdrv_unfreeze_backing_chain(cor_filter_bs, s->bottom_bs);
}
bdrv_child_refresh_perms(cor_filter_bs, child, &error_abort);
bdrv_replace_node(cor_filter_bs, bs, &error_abort);
bdrv_drained_end(bs);
bdrv_unref(bs);
bdrv_drop_filter(cor_filter_bs, &error_abort);
bdrv_unref(cor_filter_bs);
}

View File

@ -30,6 +30,7 @@
#include "block/blockjob_int.h"
#include "block/block_int.h"
#include "block/coroutines.h"
#include "block/write-threshold.h"
#include "qemu/cutils.h"
#include "qapi/error.h"
#include "qemu/error-report.h"
@ -2008,8 +2009,8 @@ bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, int64_t bytes,
} else {
assert(child->perm & BLK_PERM_WRITE);
}
return notifier_with_return_list_notify(&bs->before_write_notifiers,
req);
bdrv_write_threshold_check_write(bs, offset, bytes);
return 0;
case BDRV_TRACKED_TRUNCATE:
assert(child->perm & BLK_PERM_RESIZE);
return 0;
@ -3164,12 +3165,6 @@ bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
return true;
}
void bdrv_add_before_write_notifier(BlockDriverState *bs,
NotifierWithReturn *notifier)
{
notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
}
void bdrv_io_plug(BlockDriverState *bs)
{
BdrvChild *child;

View File

@ -1178,12 +1178,14 @@ static bool mirror_drained_poll(BlockJob *job)
return !!s->in_flight;
}
static void mirror_cancel(Job *job)
static void mirror_cancel(Job *job, bool force)
{
MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
BlockDriverState *target = blk_bs(s->target);
if (force || !job_is_ready(job)) {
bdrv_cancel_in_flight(target);
}
}
static const BlockJobDriver mirror_job_driver = {

View File

@ -557,8 +557,10 @@ void hmp_eject(Monitor *mon, const QDict *qdict)
void hmp_qemu_io(Monitor *mon, const QDict *qdict)
{
BlockBackend *blk;
BlockBackend *blk = NULL;
BlockDriverState *bs = NULL;
BlockBackend *local_blk = NULL;
AioContext *ctx = NULL;
bool qdev = qdict_get_try_bool(qdict, "qdev", false);
const char *device = qdict_get_str(qdict, "device");
const char *command = qdict_get_str(qdict, "command");
@ -573,18 +575,22 @@ void hmp_qemu_io(Monitor *mon, const QDict *qdict)
} else {
blk = blk_by_name(device);
if (!blk) {
BlockDriverState *bs = bdrv_lookup_bs(NULL, device, &err);
bs = bdrv_lookup_bs(NULL, device, &err);
if (!bs) {
goto fail;
}
}
}
ctx = blk ? blk_get_aio_context(blk) : bdrv_get_aio_context(bs);
aio_context_acquire(ctx);
if (bs) {
blk = local_blk = blk_new(bdrv_get_aio_context(bs),
0, BLK_PERM_ALL);
blk = local_blk = blk_new(bdrv_get_aio_context(bs), 0, BLK_PERM_ALL);
ret = blk_insert_bs(blk, bs, &err);
if (ret < 0) {
goto fail;
}
} else {
goto fail;
}
}
}
/*
@ -616,6 +622,11 @@ void hmp_qemu_io(Monitor *mon, const QDict *qdict)
fail:
blk_unref(local_blk);
if (ctx) {
aio_context_release(ctx);
}
hmp_handle_error(mon, err);
}

View File

@ -113,21 +113,31 @@ static int qemu_rbd_connect(rados_t *cluster, rados_ioctx_t *io_ctx,
const char *keypairs, const char *secretid,
Error **errp);
static char *qemu_rbd_strchr(char *src, char delim)
{
char *p;
for (p = src; *p; ++p) {
if (*p == delim) {
return p;
}
if (*p == '\\' && p[1] != '\0') {
++p;
}
}
return NULL;
}
static char *qemu_rbd_next_tok(char *src, char delim, char **p)
{
char *end;
*p = NULL;
for (end = src; *end; ++end) {
if (*end == delim) {
break;
}
if (*end == '\\' && end[1] != '\0') {
end++;
}
}
if (*end == delim) {
end = qemu_rbd_strchr(src, delim);
if (end) {
*p = end + 1;
*end = '\0';
}
@ -171,7 +181,7 @@ static void qemu_rbd_parse_filename(const char *filename, QDict *options,
qemu_rbd_unescape(found_str);
qdict_put_str(options, "pool", found_str);
if (strchr(p, '@')) {
if (qemu_rbd_strchr(p, '@')) {
image_name = qemu_rbd_next_tok(p, '@', &p);
found_str = qemu_rbd_next_tok(p, ':', &p);
@ -181,7 +191,7 @@ static void qemu_rbd_parse_filename(const char *filename, QDict *options,
image_name = qemu_rbd_next_tok(p, ':', &p);
}
/* Check for namespace in the image_name */
if (strchr(image_name, '/')) {
if (qemu_rbd_strchr(image_name, '/')) {
found_str = qemu_rbd_next_tok(image_name, '/', &image_name);
qemu_rbd_unescape(found_str);
qdict_put_str(options, "namespace", found_str);

View File

@ -12,9 +12,7 @@
#include "qemu/osdep.h"
#include "block/block_int.h"
#include "qemu/coroutine.h"
#include "block/write-threshold.h"
#include "qemu/notify.h"
#include "qapi/error.h"
#include "qapi/qapi-commands-block-core.h"
#include "qapi/qapi-events-block-core.h"
@ -24,82 +22,9 @@ uint64_t bdrv_write_threshold_get(const BlockDriverState *bs)
return bs->write_threshold_offset;
}
bool bdrv_write_threshold_is_set(const BlockDriverState *bs)
{
return bs->write_threshold_offset > 0;
}
static void write_threshold_disable(BlockDriverState *bs)
{
if (bdrv_write_threshold_is_set(bs)) {
notifier_with_return_remove(&bs->write_threshold_notifier);
bs->write_threshold_offset = 0;
}
}
uint64_t bdrv_write_threshold_exceeded(const BlockDriverState *bs,
const BdrvTrackedRequest *req)
{
if (bdrv_write_threshold_is_set(bs)) {
if (req->offset > bs->write_threshold_offset) {
return (req->offset - bs->write_threshold_offset) + req->bytes;
}
if ((req->offset + req->bytes) > bs->write_threshold_offset) {
return (req->offset + req->bytes) - bs->write_threshold_offset;
}
}
return 0;
}
static int coroutine_fn before_write_notify(NotifierWithReturn *notifier,
void *opaque)
{
BdrvTrackedRequest *req = opaque;
BlockDriverState *bs = req->bs;
uint64_t amount = 0;
amount = bdrv_write_threshold_exceeded(bs, req);
if (amount > 0) {
qapi_event_send_block_write_threshold(
bs->node_name,
amount,
bs->write_threshold_offset);
/* autodisable to avoid flooding the monitor */
write_threshold_disable(bs);
}
return 0; /* should always let other notifiers run */
}
static void write_threshold_register_notifier(BlockDriverState *bs)
{
bs->write_threshold_notifier.notify = before_write_notify;
bdrv_add_before_write_notifier(bs, &bs->write_threshold_notifier);
}
static void write_threshold_update(BlockDriverState *bs,
int64_t threshold_bytes)
{
bs->write_threshold_offset = threshold_bytes;
}
void bdrv_write_threshold_set(BlockDriverState *bs, uint64_t threshold_bytes)
{
if (bdrv_write_threshold_is_set(bs)) {
if (threshold_bytes > 0) {
write_threshold_update(bs, threshold_bytes);
} else {
write_threshold_disable(bs);
}
} else {
if (threshold_bytes > 0) {
/* avoid multiple registration */
write_threshold_register_notifier(bs);
write_threshold_update(bs, threshold_bytes);
}
/* discard bogus disable request */
}
bs->write_threshold_offset = threshold_bytes;
}
void qmp_block_set_write_threshold(const char *node_name,
@ -122,3 +47,17 @@ void qmp_block_set_write_threshold(const char *node_name,
aio_context_release(aio_context);
}
void bdrv_write_threshold_check_write(BlockDriverState *bs, int64_t offset,
int64_t bytes)
{
int64_t end = offset + bytes;
uint64_t wtr = bs->write_threshold_offset;
if (wtr > 0 && end > wtr) {
qapi_event_send_block_write_threshold(bs->node_name, end - wtr, wtr);
/* autodisable to avoid flooding the monitor */
bdrv_write_threshold_set(bs, 0);
}
}

View File

@ -866,6 +866,37 @@ Supported image file formats:
issue ``lsattr filename`` to check if the NOCOW flag is set or not
(Capital 'C' is NOCOW flag).
``data_file``
Filename where all guest data will be stored. If this option is used,
the qcow2 file will only contain the image's metadata.
Note: Data loss will occur if the given filename already exists when
using this option with ``qemu-img create`` since ``qemu-img`` will create
the data file anew, overwriting the file's original contents. To simply
update the reference to point to the given pre-existing file, use
``qemu-img amend``.
``data_file_raw``
If this option is set to ``on``, QEMU will always keep the external data
file consistent as a standalone read-only raw image.
It does this by forwarding all write accesses to the qcow2 file through to
the raw data file, including their offsets. Therefore, data that is visible
on the qcow2 node (i.e., to the guest) at some offset is visible at the same
offset in the raw data file. This results in a read-only raw image. Writes
that bypass the qcow2 metadata may corrupt the qcow2 metadata because the
out-of-band writes may result in the metadata falling out of sync with the
raw image.
If this option is ``off``, QEMU will use the data file to store data in an
arbitrary manner. The files content will not make sense without the
accompanying qcow2 metadata. Where data is written will have no relation to
its offset as seen by the guest, and some writes (specifically zero writes)
may not be forwarded to the data file at all, but will only be handled by
modifying qcow2 metadata.
This option can only be enabled if ``data_file`` is set.
``Other``
QEMU also supports various other image file formats for

View File

@ -357,7 +357,7 @@ struct BlockDriver {
* of in-flight requests, so don't waste the time if possible.
*
* One example usage is to avoid waiting for an nbd target node reconnect
* timeout during job-cancel.
* timeout during job-cancel with force=true.
*/
void (*bdrv_cancel_in_flight)(BlockDriverState *bs);
@ -954,12 +954,8 @@ struct BlockDriverState {
*/
int64_t total_sectors;
/* Callback before write request is processed */
NotifierWithReturnList before_write_notifiers;
/* threshold limit for writes, in bytes. "High water mark". */
uint64_t write_threshold_offset;
NotifierWithReturn write_threshold_notifier;
/* Writing to the list requires the BQL _and_ the dirty_bitmap_mutex.
* Reading from the list can be done with either the BQL or the
@ -1084,15 +1080,6 @@ void bdrv_parse_filename_strip_prefix(const char *filename, const char *prefix,
bool bdrv_backing_overridden(BlockDriverState *bs);
/**
* bdrv_add_before_write_notifier:
*
* Register a callback that is invoked before write requests are processed but
* after any throttling or waiting for overlapping requests.
*/
void bdrv_add_before_write_notifier(BlockDriverState *bs,
NotifierWithReturn *notifier);
/**
* bdrv_add_aio_context_notifier:
*

View File

@ -13,7 +13,7 @@
#ifndef BLOCK_WRITE_THRESHOLD_H
#define BLOCK_WRITE_THRESHOLD_H
#include "block/block_int.h"
#include "qemu/typedefs.h"
/*
* bdrv_write_threshold_set:
@ -36,27 +36,12 @@ void bdrv_write_threshold_set(BlockDriverState *bs, uint64_t threshold_bytes);
uint64_t bdrv_write_threshold_get(const BlockDriverState *bs);
/*
* bdrv_write_threshold_is_set
* bdrv_write_threshold_check_write
*
* Tell if a write threshold is set for a given BDS.
* Check whether the specified request exceeds the write threshold.
* If so, send a corresponding event and disable write threshold checking.
*/
bool bdrv_write_threshold_is_set(const BlockDriverState *bs);
/*
* bdrv_write_threshold_exceeded
*
* Return the extent of a write request that exceeded the threshold,
* or zero if the request is below the threshold.
* Return zero also if the threshold was not set.
*
* NOTE: here we assume the following holds for each request this code
* deals with:
*
* assert((req->offset + req->bytes) <= UINT64_MAX)
*
* Please not there is *not* an actual C assert().
*/
uint64_t bdrv_write_threshold_exceeded(const BlockDriverState *bs,
const BdrvTrackedRequest *req);
void bdrv_write_threshold_check_write(BlockDriverState *bs, int64_t offset,
int64_t bytes);
#endif

View File

@ -254,7 +254,7 @@ struct JobDriver {
/**
* If the callback is not NULL, it will be invoked in job_cancel_async
*/
void (*cancel)(Job *job);
void (*cancel)(Job *job, bool force);
/** Called when the job is freed */

2
job.c
View File

@ -716,7 +716,7 @@ static int job_finalize_single(Job *job)
static void job_cancel_async(Job *job, bool force)
{
if (job->driver->cancel) {
job->driver->cancel(job);
job->driver->cancel(job, force);
}
if (job->user_paused) {
/* Do not call job_enter here, the caller will handle it. */

View File

@ -2457,9 +2457,12 @@ static const cmdinfo_t help_cmd = {
.oneline = "help for one or all commands",
};
/*
* Called with aio context of blk acquired. Or with qemu_get_aio_context()
* context acquired if blk is NULL.
*/
int qemuio_command(BlockBackend *blk, const char *cmd)
{
AioContext *ctx;
char *input;
const cmdinfo_t *ct;
char **v;
@ -2471,10 +2474,7 @@ int qemuio_command(BlockBackend *blk, const char *cmd)
if (c) {
ct = find_command(v[0]);
if (ct) {
ctx = blk ? blk_get_aio_context(blk) : qemu_get_aio_context();
aio_context_acquire(ctx);
ret = command(blk, ct, c, v);
aio_context_release(ctx);
} else {
fprintf(stderr, "command \"%s\" not found\n", v[0]);
ret = -EINVAL;

View File

@ -411,6 +411,19 @@ static void prep_fetchline(void *opaque)
*fetchable= 1;
}
static int do_qemuio_command(const char *cmd)
{
int ret;
AioContext *ctx =
qemuio_blk ? blk_get_aio_context(qemuio_blk) : qemu_get_aio_context();
aio_context_acquire(ctx);
ret = qemuio_command(qemuio_blk, cmd);
aio_context_release(ctx);
return ret;
}
static int command_loop(void)
{
int i, fetchable = 0, prompted = 0;
@ -418,7 +431,7 @@ static int command_loop(void)
char *input;
for (i = 0; !quit_qemu_io && i < ncmdline; i++) {
ret = qemuio_command(qemuio_blk, cmdline[i]);
ret = do_qemuio_command(cmdline[i]);
if (ret < 0) {
last_error = ret;
}
@ -446,7 +459,7 @@ static int command_loop(void)
if (input == NULL) {
break;
}
ret = qemuio_command(qemuio_blk, input);
ret = do_qemuio_command(input);
g_free(input);
if (ret < 0) {

View File

@ -55,6 +55,10 @@ _filter_conf()
$QEMU_IMG info "json:{'file.driver':'rbd','file.filename':'rbd:rbd/bogus:conf=${BOGUS_CONF}'}" 2>&1 | _filter_conf
$QEMU_IMG info "json:{'file.driver':'rbd','file.pool':'rbd','file.image':'bogus','file.conf':'${BOGUS_CONF}'}" 2>&1 | _filter_conf
# Regression test: the qemu-img invocation is expected to fail, but it should
# not seg fault the parser.
$QEMU_IMG create "rbd:rbd/aa\/bb:conf=${BOGUS_CONF}" 1M 2>&1 | _filter_conf
# success, all done
echo "*** done"
rm -f $seq.full

View File

@ -1,9 +1,10 @@
QA output created by 231
qemu-img: RBD options encoded in the filename as keyvalue pairs is deprecated. Future versions may cease to parse these options in the future.
qemu-img: warning: RBD options encoded in the filename as keyvalue pairs is deprecated
unable to get monitor info from DNS SRV with service name: ceph-mon
no monitors specified to connect to.
qemu-img: Could not open 'json:{'file.driver':'rbd','file.filename':'rbd:rbd/bogus:conf=BOGUS_CONF'}': error connecting: No such file or directory
unable to get monitor info from DNS SRV with service name: ceph-mon
no monitors specified to connect to.
qemu-img: Could not open 'json:{'file.driver':'rbd','file.pool':'rbd','file.image':'bogus','file.conf':'BOGUS_CONF'}': error connecting: No such file or directory
Formatting 'rbd:rbd/aa\/bb:conf=BOGUS_CONF', fmt=raw size=1048576
unable to get monitor info from DNS SRV with service name: ceph-mon
qemu-img: rbd:rbd/aa\/bb:conf=BOGUS_CONF: error connecting: No such file or directory
*** done

View File

@ -15,7 +15,7 @@
{"return": {}}
{"execute": "blockdev-del", "arguments": {"node-name": "hd0"}}
{"return": {}}
==Attach two SCSI disks using the same block device and the same iothread==
.==Attach two SCSI disks using the same block device and the same iothread==
{"execute": "blockdev-add", "arguments": {"driver": "null-co", "node-name": "hd0", "read-only": true, "read-zeroes": true}}
{"return": {}}
{"execute": "object-add", "arguments": {"id": "iothread0", "qom-type": "iothread"}}
@ -32,7 +32,7 @@
{"return": {}}
{"execute": "blockdev-del", "arguments": {"node-name": "hd0"}}
{"return": {}}
==Attach two SCSI disks using the same block device but different iothreads==
.==Attach two SCSI disks using the same block device but different iothreads==
{"execute": "blockdev-add", "arguments": {"driver": "null-co", "node-name": "hd0", "read-only": true, "read-zeroes": true}}
{"return": {}}
{"execute": "object-add", "arguments": {"id": "iothread0", "qom-type": "iothread"}}
@ -55,7 +55,7 @@
{"return": {}}
{"execute": "blockdev-del", "arguments": {"node-name": "hd0"}}
{"return": {}}
==Attach a SCSI disks using the same block device as a NBD server==
.==Attach a SCSI disks using the same block device as a NBD server==
{"execute": "blockdev-add", "arguments": {"driver": "null-co", "node-name": "hd0", "read-only": true, "read-zeroes": true}}
{"return": {}}
{"execute": "nbd-server-start", "arguments": {"addr": {"data": {"path": "SOCK_DIR/PID-nbd.sock"}, "type": "unix"}}}
@ -68,7 +68,7 @@
{"return": {}}
{"execute": "device_add", "arguments": {"drive": "hd0", "driver": "scsi-hd", "id": "scsi-hd0"}}
{"return": {}}
....
.
----------------------------------------------------------------------
Ran 4 tests

View File

@ -1,16 +1,16 @@
{"execute": "job-finalize", "arguments": {"id": "commit0"}}
..{"execute": "job-finalize", "arguments": {"id": "commit0"}}
{"return": {}}
{"data": {"id": "commit0", "type": "commit"}, "event": "BLOCK_JOB_PENDING", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
{"data": {"device": "commit0", "len": 3145728, "offset": 3145728, "speed": 0, "type": "commit"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
{"execute": "job-finalize", "arguments": {"id": "stream0"}}
...{"execute": "job-finalize", "arguments": {"id": "stream0"}}
{"return": {}}
{"data": {"id": "stream0", "type": "stream"}, "event": "BLOCK_JOB_PENDING", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
{"data": {"device": "stream0", "len": 3145728, "offset": 3145728, "speed": 0, "type": "stream"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
{"execute": "job-finalize", "arguments": {"id": "stream0"}}
.{"execute": "job-finalize", "arguments": {"id": "stream0"}}
{"return": {}}
{"data": {"id": "stream0", "type": "stream"}, "event": "BLOCK_JOB_PENDING", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
{"data": {"device": "stream0", "len": 3145728, "offset": 3145728, "speed": 0, "type": "stream"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}}
.....................
...............
----------------------------------------------------------------------
Ran 21 tests

View File

@ -95,7 +95,7 @@ class TestNbdReconnect(iotests.QMPTestCase):
self.assert_qmp(result, 'return', {})
def cancel_job(self):
result = self.vm.qmp('block-job-cancel', device='drive0')
result = self.vm.qmp('block-job-cancel', device='drive0', force=True)
self.assert_qmp(result, 'return', {})
start_t = time.time()

View File

@ -4,7 +4,7 @@
{"return": {}}
{"execute": "job-dismiss", "arguments": {"id": "job_erase_key"}}
{"return": {}}
{"execute": "job-dismiss", "arguments": {"id": "job_add_key"}}
.{"execute": "job-dismiss", "arguments": {"id": "job_add_key"}}
{"return": {}}
{"execute": "job-dismiss", "arguments": {"id": "job_erase_key"}}
{"return": {}}
@ -13,7 +13,7 @@ Job failed: Invalid password, cannot unlock any keyslot
{"return": {}}
{"execute": "job-dismiss", "arguments": {"id": "job_add_key"}}
{"return": {}}
{"execute": "job-dismiss", "arguments": {"id": "job_add_key"}}
.{"execute": "job-dismiss", "arguments": {"id": "job_add_key"}}
{"return": {}}
{"execute": "job-dismiss", "arguments": {"id": "job_add_key"}}
{"return": {}}
@ -33,7 +33,7 @@ Job failed: All the active keyslots match the (old) password that was given and
{"return": {}}
{"execute": "job-dismiss", "arguments": {"id": "job_erase_key"}}
{"return": {}}
...
.
----------------------------------------------------------------------
Ran 3 tests

View File

@ -13,7 +13,7 @@ Job failed: Failed to get shared "consistent read" lock
qemu-img: Failed to get shared "consistent read" lock
Is another process using the image [TEST_DIR/test.img]?
Formatting 'TEST_DIR/test.img', fmt=luks size=1048576 key-secret=keysec0 iter-time=10
.Formatting 'TEST_DIR/test.img', fmt=luks size=1048576 key-secret=keysec0 iter-time=10
Job failed: Block node is read-only
{"execute": "job-dismiss", "arguments": {"id": "job0"}}
@ -26,15 +26,15 @@ Job failed: Failed to get shared "consistent read" lock
{"return": {}}
{"execute": "job-dismiss", "arguments": {"id": "job0"}}
{"return": {}}
Formatting 'TEST_DIR/test.img', fmt=luks size=1048576 key-secret=keysec0 iter-time=10
.Formatting 'TEST_DIR/test.img', fmt=luks size=1048576 key-secret=keysec0 iter-time=10
{"return": {}}
{"error": {"class": "GenericError", "desc": "Failed to get \"write\" lock"}}
Formatting 'TEST_DIR/test.img', fmt=luks size=1048576 key-secret=keysec0 iter-time=10
.Formatting 'TEST_DIR/test.img', fmt=luks size=1048576 key-secret=keysec0 iter-time=10
{"return": {}}
{"return": {}}
....
.
----------------------------------------------------------------------
Ran 4 tests

View File

@ -19,6 +19,9 @@
import os
import sys
import argparse
import shutil
from pathlib import Path
from findtests import TestFinder
from testenv import TestEnv
from testrunner import TestRunner
@ -100,7 +103,7 @@ def make_argparser() -> argparse.ArgumentParser:
'rerun failed ./check command, starting from the '
'middle of the process.')
g_sel.add_argument('tests', metavar='TEST_FILES', nargs='*',
help='tests to run')
help='tests to run, or "--" followed by a command')
return p
@ -113,6 +116,20 @@ if __name__ == '__main__':
imgopts=args.imgopts, misalign=args.misalign,
debug=args.debug, valgrind=args.valgrind)
if len(sys.argv) > 1 and sys.argv[-len(args.tests)-1] == '--':
if not args.tests:
sys.exit("missing command after '--'")
cmd = args.tests
env.print_env()
exec_pathstr = shutil.which(cmd[0])
if exec_pathstr is None:
sys.exit('command not found: ' + cmd[0])
exec_path = Path(exec_pathstr).resolve()
cmd[0] = str(exec_path)
full_env = env.prepare_subprocess(cmd)
os.chdir(exec_path.parent)
os.execve(cmd[0], cmd, full_env)
testfinder = TestFinder(test_dir=env.source_iotests)
groups = args.groups.split(',') if args.groups else None

View File

@ -20,7 +20,6 @@ import atexit
import bz2
from collections import OrderedDict
import faulthandler
import io
import json
import logging
import os
@ -32,7 +31,7 @@ import subprocess
import sys
import time
from typing import (Any, Callable, Dict, Iterable,
List, Optional, Sequence, Tuple, TypeVar)
List, Optional, Sequence, TextIO, Tuple, Type, TypeVar)
import unittest
from contextlib import contextmanager
@ -113,14 +112,13 @@ def qemu_tool_pipe_and_status(tool: str, args: Sequence[str],
Run a tool and return both its output and its exit code
"""
stderr = subprocess.STDOUT if connect_stderr else None
subp = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=stderr,
universal_newlines=True)
with subprocess.Popen(args, stdout=subprocess.PIPE,
stderr=stderr, universal_newlines=True) as subp:
output = subp.communicate()[0]
if subp.returncode < 0:
cmd = ' '.join(args)
sys.stderr.write(f'{tool} received signal {-subp.returncode}: {cmd}\n')
sys.stderr.write(f'{tool} received signal \
{-subp.returncode}: {cmd}\n')
return (output, subp.returncode)
def qemu_img_pipe_and_status(*args: str) -> Tuple[str, int]:
@ -237,6 +235,9 @@ def qemu_io_silent_check(*args):
class QemuIoInteractive:
def __init__(self, *args):
self.args = qemu_io_args_no_fmt + list(args)
# We need to keep the Popen objext around, and not
# close it immediately. Therefore, disable the pylint check:
# pylint: disable=consider-using-with
self._p = subprocess.Popen(self.args, stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
@ -310,7 +311,7 @@ def qemu_nbd_popen(*args):
cmd.extend(args)
log('Start NBD server')
p = subprocess.Popen(cmd)
with subprocess.Popen(cmd) as p:
try:
while not os.path.exists(pid_file):
if p.poll() is not None:
@ -334,13 +335,12 @@ def compare_images(img1, img2, fmt1=imgfmt, fmt2=imgfmt):
def create_image(name, size):
'''Create a fully-allocated raw image with sector markers'''
file = open(name, 'wb')
with open(name, 'wb') as file:
i = 0
while i < size:
sector = struct.pack('>l504xl', i // 512, i // 512)
file.write(sector)
i = i + 512
file.close()
def image_size(img):
'''Return image's virtual size'''
@ -1271,37 +1271,54 @@ def skip_if_user_is_root(func):
return func(*args, **kwargs)
return func_wrapper
def execute_unittest(debug=False):
# We need to filter out the time taken from the output so that
# qemu-iotest can reliably diff the results against master output,
# and hide skipped tests from the reference output.
class ReproducibleTestResult(unittest.TextTestResult):
def addSkip(self, test, reason):
# Same as TextTestResult, but print dot instead of "s"
unittest.TestResult.addSkip(self, test, reason)
if self.showAll:
self.stream.writeln("skipped {0!r}".format(reason))
elif self.dots:
self.stream.write(".")
self.stream.flush()
class ReproducibleStreamWrapper:
def __init__(self, stream: TextIO):
self.stream = stream
def __getattr__(self, attr):
if attr in ('stream', '__getstate__'):
raise AttributeError(attr)
return getattr(self.stream, attr)
def write(self, arg=None):
arg = re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', arg)
arg = re.sub(r' \(skipped=\d+\)', r'', arg)
self.stream.write(arg)
class ReproducibleTestRunner(unittest.TextTestRunner):
def __init__(self, stream: Optional[TextIO] = None,
resultclass: Type[unittest.TestResult] = ReproducibleTestResult,
**kwargs: Any) -> None:
rstream = ReproducibleStreamWrapper(stream or sys.stdout)
super().__init__(stream=rstream, # type: ignore
descriptions=True,
resultclass=resultclass,
**kwargs)
def execute_unittest(argv: List[str], debug: bool = False) -> None:
"""Executes unittests within the calling module."""
verbosity = 2 if debug else 1
if debug:
output = sys.stdout
else:
# We need to filter out the time taken from the output so that
# qemu-iotest can reliably diff the results against master output.
output = io.StringIO()
runner = unittest.TextTestRunner(stream=output, descriptions=True,
verbosity=verbosity)
try:
# unittest.main() will use sys.exit(); so expect a SystemExit
# exception
unittest.main(testRunner=runner)
finally:
# We need to filter out the time taken from the output so that
# qemu-iotest can reliably diff the results against master output.
if not debug:
out = output.getvalue()
out = re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', out)
# Hide skipped tests from the reference output
out = re.sub(r'OK \(skipped=\d+\)', 'OK', out)
out_first_line, out_rest = out.split('\n', 1)
out = out_first_line.replace('s', '.') + '\n' + out_rest
sys.stderr.write(out)
# Some tests have warnings, especially ResourceWarnings for unclosed
# files and sockets. Ignore them for now to ensure reproducibility of
# the test output.
unittest.main(argv=argv,
testRunner=ReproducibleTestRunner,
verbosity=2 if debug else 1,
warnings=None if sys.warnoptions else 'ignore')
def execute_setup_common(supported_fmts: Sequence[str] = (),
supported_platforms: Sequence[str] = (),
@ -1338,7 +1355,7 @@ def execute_test(*args, test_function=None, **kwargs):
debug = execute_setup_common(*args, **kwargs)
if not test_function:
execute_unittest(debug)
execute_unittest(sys.argv, debug)
else:
test_function()

View File

@ -19,6 +19,9 @@ disable=invalid-name,
too-many-public-methods,
# pylint warns about Optional[] etc. as unsubscriptable in 3.9
unsubscriptable-object,
# Sometimes we need to disable a newly introduced pylint warning.
# Doing so should not produce a warning in older versions of pylint.
bad-option-value,
# These are temporary, and should be removed:
missing-docstring,
too-many-return-statements,

View File

@ -25,7 +25,7 @@ import collections
import random
import subprocess
import glob
from typing import Dict, Any, Optional, ContextManager
from typing import List, Dict, Any, Optional, ContextManager
def isxfile(path: str) -> bool:
@ -74,6 +74,21 @@ class TestEnv(ContextManager['TestEnv']):
'CACHEMODE_IS_DEFAULT', 'IMGFMT_GENERIC', 'IMGOPTSSYNTAX',
'IMGKEYSECRET', 'QEMU_DEFAULT_MACHINE', 'MALLOC_PERTURB_']
def prepare_subprocess(self, args: List[str]) -> Dict[str, str]:
if self.debug:
args.append('-d')
with open(args[0], encoding="utf-8") as f:
try:
if f.readline().rstrip() == '#!/usr/bin/env python3':
args.insert(0, self.python)
except UnicodeDecodeError: # binary test? for future.
pass
os_env = os.environ.copy()
os_env.update(self.get_env())
return os_env
def get_env(self) -> Dict[str, str]:
env = {}
for v in self.env_variables:
@ -105,7 +120,7 @@ class TestEnv(ContextManager['TestEnv']):
try:
self.sock_dir = os.environ['SOCK_DIR']
self.tmp_sock_dir = False
Path(self.test_dir).mkdir(parents=True, exist_ok=True)
Path(self.sock_dir).mkdir(parents=True, exist_ok=True)
except KeyError:
self.sock_dir = tempfile.mkdtemp()
self.tmp_sock_dir = True
@ -269,7 +284,8 @@ IMGPROTO -- {IMGPROTO}
PLATFORM -- {platform}
TEST_DIR -- {TEST_DIR}
SOCK_DIR -- {SOCK_DIR}
SOCKET_SCM_HELPER -- {SOCKET_SCM_HELPER}"""
SOCKET_SCM_HELPER -- {SOCKET_SCM_HELPER}
"""
args = collections.defaultdict(str, self.get_env())

View File

@ -129,7 +129,6 @@ class TestRunner(ContextManager['TestRunner']):
def __init__(self, env: TestEnv, makecheck: bool = False,
color: str = 'auto') -> None:
self.env = env
self.test_run_env = self.env.get_env()
self.makecheck = makecheck
self.last_elapsed = LastElapsedTime('.last-elapsed-cache', env)
@ -243,23 +242,12 @@ class TestRunner(ContextManager['TestRunner']):
silent_unlink(p)
args = [str(f_test.resolve())]
if self.env.debug:
args.append('-d')
with f_test.open(encoding="utf-8") as f:
try:
if f.readline().rstrip() == '#!/usr/bin/env python3':
args.insert(0, self.env.python)
except UnicodeDecodeError: # binary test? for future.
pass
env = os.environ.copy()
env.update(self.test_run_env)
env = self.env.prepare_subprocess(args)
t0 = time.time()
with f_bad.open('w', encoding="utf-8") as f:
proc = subprocess.Popen(args, cwd=str(f_test.parent), env=env,
stdout=f, stderr=subprocess.STDOUT)
with subprocess.Popen(args, cwd=str(f_test.parent), env=env,
stdout=f, stderr=subprocess.STDOUT) as proc:
try:
proc.wait()
except KeyboardInterrupt:
@ -328,7 +316,6 @@ class TestRunner(ContextManager['TestRunner']):
if not self.makecheck:
self.env.print_env()
print()
test_field_width = max(len(os.path.basename(t)) for t in tests) + 2

View File

@ -7,117 +7,41 @@
*/
#include "qemu/osdep.h"
#include "qapi/error.h"
#include "block/block_int.h"
#include "block/write-threshold.h"
static void test_threshold_not_set_on_init(void)
{
uint64_t res;
BlockDriverState bs;
memset(&bs, 0, sizeof(bs));
g_assert(!bdrv_write_threshold_is_set(&bs));
res = bdrv_write_threshold_get(&bs);
g_assert_cmpint(res, ==, 0);
}
static void test_threshold_set_get(void)
{
uint64_t threshold = 4 * 1024 * 1024;
uint64_t res;
BlockDriverState bs;
memset(&bs, 0, sizeof(bs));
bdrv_write_threshold_set(&bs, threshold);
g_assert(bdrv_write_threshold_is_set(&bs));
res = bdrv_write_threshold_get(&bs);
g_assert_cmpint(res, ==, threshold);
}
static void test_threshold_multi_set_get(void)
{
uint64_t threshold1 = 4 * 1024 * 1024;
uint64_t threshold2 = 15 * 1024 * 1024;
uint64_t res;
BlockDriverState bs;
memset(&bs, 0, sizeof(bs));
bdrv_write_threshold_set(&bs, threshold1);
bdrv_write_threshold_set(&bs, threshold2);
res = bdrv_write_threshold_get(&bs);
g_assert_cmpint(res, ==, threshold2);
}
static void test_threshold_not_trigger(void)
{
uint64_t amount = 0;
uint64_t threshold = 4 * 1024 * 1024;
BlockDriverState bs;
BdrvTrackedRequest req;
memset(&bs, 0, sizeof(bs));
memset(&req, 0, sizeof(req));
req.offset = 1024;
req.bytes = 1024;
bdrv_check_request(req.offset, req.bytes, &error_abort);
bdrv_write_threshold_set(&bs, threshold);
amount = bdrv_write_threshold_exceeded(&bs, &req);
g_assert_cmpuint(amount, ==, 0);
bdrv_write_threshold_check_write(&bs, 1024, 1024);
g_assert_cmpuint(bdrv_write_threshold_get(&bs), ==, threshold);
}
static void test_threshold_trigger(void)
{
uint64_t amount = 0;
uint64_t threshold = 4 * 1024 * 1024;
BlockDriverState bs;
BdrvTrackedRequest req;
memset(&bs, 0, sizeof(bs));
memset(&req, 0, sizeof(req));
req.offset = (4 * 1024 * 1024) - 1024;
req.bytes = 2 * 1024;
bdrv_check_request(req.offset, req.bytes, &error_abort);
bdrv_write_threshold_set(&bs, threshold);
amount = bdrv_write_threshold_exceeded(&bs, &req);
g_assert_cmpuint(amount, >=, 1024);
bdrv_write_threshold_check_write(&bs, threshold - 1024, 2 * 1024);
g_assert_cmpuint(bdrv_write_threshold_get(&bs), ==, 0);
}
typedef struct TestStruct {
const char *name;
void (*func)(void);
} TestStruct;
int main(int argc, char **argv)
{
size_t i;
TestStruct tests[] = {
{ "/write-threshold/not-set-on-init",
test_threshold_not_set_on_init },
{ "/write-threshold/set-get",
test_threshold_set_get },
{ "/write-threshold/multi-set-get",
test_threshold_multi_set_get },
{ "/write-threshold/not-trigger",
test_threshold_not_trigger },
{ "/write-threshold/trigger",
test_threshold_trigger },
{ NULL, NULL }
};
g_test_init(&argc, &argv, NULL);
for (i = 0; tests[i].name != NULL; i++) {
g_test_add_func(tests[i].name, tests[i].func);
}
g_test_add_func("/write-threshold/not-trigger", test_threshold_not_trigger);
g_test_add_func("/write-threshold/trigger", test_threshold_trigger);
return g_test_run();
}