From a6297e1ade6d9981c4a8d43eb426830b49a7913c Mon Sep 17 00:00:00 2001 From: Emanuele Giuseppe Esposito Date: Fri, 3 Sep 2021 13:38:00 +0200 Subject: [PATCH 01/13] include/block.h: remove outdated comment There are a couple of errors in bdrv_drained_begin header comment: - block_job_pause does not exist anymore, it has been replaced with job_pause in b15de82867 - job_pause is automatically invoked as a .drained_begin callback (child_job_drained_begin) by the child_job BdrvChildClass struct in blockjob.c. So no additional pause should be required. Signed-off-by: Emanuele Giuseppe Esposito Message-Id: <20210903113800.59970-1-eesposit@redhat.com> Reviewed-by: Stefan Hajnoczi Signed-off-by: Kevin Wolf --- include/block/block.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/include/block/block.h b/include/block/block.h index 740038a892..ab987e8a99 100644 --- a/include/block/block.h +++ b/include/block/block.h @@ -751,9 +751,7 @@ bool bdrv_drain_poll(BlockDriverState *bs, bool recursive, * bdrv_drained_begin: * * Begin a quiesced section for exclusive access to the BDS, by disabling - * external request sources including NBD server and device model. Note that - * this doesn't block timers or coroutines from submitting more requests, which - * means block_job_pause is still necessary. + * external request sources including NBD server, block jobs, and device model. * * This function can be recursive. */ From d1bbd965bd96028b0f5a8e3dc01b37f1f8ae4456 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 16 Aug 2021 20:04:42 +0200 Subject: [PATCH 02/13] qemu-storage-daemon: Only display FUSE help when FUSE is built-in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When configuring QEMU with --disable-fuse, the qemu-storage-daemon still reports FUSE command line options in its help: $ qemu-storage-daemon -h Usage: qemu-storage-daemon [options] QEMU storage daemon --export [type=]fuse,id=,node-name=,mountpoint= [,growable=on|off][,writable=on|off] export the specified block node over FUSE Remove this help message when FUSE is disabled, to avoid: $ qemu-storage-daemon --export fuse qemu-storage-daemon: --export fuse: Invalid parameter 'fuse' Reported-by: Qing Wang Signed-off-by: Philippe Mathieu-Daudé Message-Id: <20210816180442.2000642-1-philmd@redhat.com> Reviewed-by: Eric Blake Reviewed-by: Hanna Reitz Signed-off-by: Kevin Wolf --- storage-daemon/qemu-storage-daemon.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage-daemon/qemu-storage-daemon.c b/storage-daemon/qemu-storage-daemon.c index fc8b150629..10a1a33761 100644 --- a/storage-daemon/qemu-storage-daemon.c +++ b/storage-daemon/qemu-storage-daemon.c @@ -98,10 +98,12 @@ static void help(void) " export the specified block node over NBD\n" " (requires --nbd-server)\n" "\n" +#ifdef CONFIG_FUSE " --export [type=]fuse,id=,node-name=,mountpoint=\n" " [,growable=on|off][,writable=on|off]\n" " export the specified block node over FUSE\n" "\n" +#endif /* CONFIG_FUSE */ " --monitor [chardev=]name[,mode=control][,pretty[=on|off]]\n" " configure a QMP monitor\n" "\n" From 621d17378a40509757d5e03eb1c2f5305ff76df3 Mon Sep 17 00:00:00 2001 From: Vladimir Sementsov-Ogievskiy Date: Mon, 20 Sep 2021 14:55:34 +0300 Subject: [PATCH 03/13] block: implement bdrv_new_open_driver_opts() Add version of bdrv_new_open_driver() that supports QDict options. We'll use it in further commit. Simply add one more argument to bdrv_new_open_driver() is worse, as there are too many invocations of bdrv_new_open_driver() to update then. Signed-off-by: Vladimir Sementsov-Ogievskiy Suggested-by: Kevin Wolf Message-Id: <20210920115538.264372-2-vsementsov@virtuozzo.com> Signed-off-by: Kevin Wolf --- block.c | 25 +++++++++++++++++++++---- include/block/block.h | 4 ++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/block.c b/block.c index 5ce08a79fd..917fb7faca 100644 --- a/block.c +++ b/block.c @@ -1604,16 +1604,26 @@ open_failed: return ret; } -BlockDriverState *bdrv_new_open_driver(BlockDriver *drv, const char *node_name, - int flags, Error **errp) +/* + * Create and open a block node. + * + * @options is a QDict of options to pass to the block drivers, or NULL for an + * empty set of options. The reference to the QDict belongs to the block layer + * after the call (even on failure), so if the caller intends to reuse the + * dictionary, it needs to use qobject_ref() before calling bdrv_open. + */ +BlockDriverState *bdrv_new_open_driver_opts(BlockDriver *drv, + const char *node_name, + QDict *options, int flags, + Error **errp) { BlockDriverState *bs; int ret; bs = bdrv_new(); bs->open_flags = flags; - bs->explicit_options = qdict_new(); - bs->options = qdict_new(); + bs->options = options ?: qdict_new(); + bs->explicit_options = qdict_clone_shallow(bs->options); bs->opaque = NULL; update_options_from_flags(bs->options, flags); @@ -1631,6 +1641,13 @@ BlockDriverState *bdrv_new_open_driver(BlockDriver *drv, const char *node_name, return bs; } +/* Create and open a block node. */ +BlockDriverState *bdrv_new_open_driver(BlockDriver *drv, const char *node_name, + int flags, Error **errp) +{ + return bdrv_new_open_driver_opts(drv, node_name, NULL, flags, errp); +} + QemuOptsList bdrv_runtime_opts = { .name = "bdrv_common", .head = QTAILQ_HEAD_INITIALIZER(bdrv_runtime_opts.head), diff --git a/include/block/block.h b/include/block/block.h index ab987e8a99..e5dd22b034 100644 --- a/include/block/block.h +++ b/include/block/block.h @@ -383,6 +383,10 @@ int bdrv_open_backing_file(BlockDriverState *bs, QDict *parent_options, const char *bdref_key, Error **errp); BlockDriverState *bdrv_open(const char *filename, const char *reference, QDict *options, int flags, Error **errp); +BlockDriverState *bdrv_new_open_driver_opts(BlockDriver *drv, + const char *node_name, + QDict *options, int flags, + Error **errp); BlockDriverState *bdrv_new_open_driver(BlockDriver *drv, const char *node_name, int flags, Error **errp); BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue, From f053b7e8005d7f72c2a8e686c4779f75b0ae631f Mon Sep 17 00:00:00 2001 From: Vladimir Sementsov-Ogievskiy Date: Mon, 20 Sep 2021 14:55:35 +0300 Subject: [PATCH 04/13] block: bdrv_insert_node(): fix and improve error handling - use ERRP_GUARD(): function calls error_prepend(), so it must use ERRP_GUARD(), otherwise error_prepend() would not be called when passed errp is error_fatal - drop error propagation, handle return code instead - for symmetry, do error_prepend() for the second failure Signed-off-by: Vladimir Sementsov-Ogievskiy Message-Id: <20210920115538.264372-3-vsementsov@virtuozzo.com> Signed-off-by: Kevin Wolf --- block.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/block.c b/block.c index 917fb7faca..5d49188073 100644 --- a/block.c +++ b/block.c @@ -5122,8 +5122,9 @@ static void bdrv_delete(BlockDriverState *bs) BlockDriverState *bdrv_insert_node(BlockDriverState *bs, QDict *node_options, int flags, Error **errp) { + ERRP_GUARD(); + int ret; BlockDriverState *new_node_bs; - Error *local_err = NULL; new_node_bs = bdrv_open(NULL, NULL, node_options, flags, errp); if (new_node_bs == NULL) { @@ -5132,12 +5133,12 @@ BlockDriverState *bdrv_insert_node(BlockDriverState *bs, QDict *node_options, } bdrv_drained_begin(bs); - bdrv_replace_node(bs, new_node_bs, &local_err); + ret = bdrv_replace_node(bs, new_node_bs, errp); bdrv_drained_end(bs); - if (local_err) { + if (ret < 0) { + error_prepend(errp, "Could not replace node: "); bdrv_unref(new_node_bs); - error_propagate(errp, local_err); return NULL; } From 96796fae6f22931d91223d086e9fa56d0f3e6720 Mon Sep 17 00:00:00 2001 From: Vladimir Sementsov-Ogievskiy Date: Mon, 20 Sep 2021 14:55:36 +0300 Subject: [PATCH 05/13] block: bdrv_insert_node(): doc and style - options & flags is common pair for open-like functions, let's use it - add a comment that specifies use of @options Signed-off-by: Vladimir Sementsov-Ogievskiy Message-Id: <20210920115538.264372-4-vsementsov@virtuozzo.com> Signed-off-by: Kevin Wolf --- block.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/block.c b/block.c index 5d49188073..3a90407b83 100644 --- a/block.c +++ b/block.c @@ -5119,14 +5119,23 @@ static void bdrv_delete(BlockDriverState *bs) g_free(bs); } -BlockDriverState *bdrv_insert_node(BlockDriverState *bs, QDict *node_options, + +/* + * Replace @bs by newly created block node. + * + * @options is a QDict of options to pass to the block drivers, or NULL for an + * empty set of options. The reference to the QDict belongs to the block layer + * after the call (even on failure), so if the caller intends to reuse the + * dictionary, it needs to use qobject_ref() before calling bdrv_open. + */ +BlockDriverState *bdrv_insert_node(BlockDriverState *bs, QDict *options, int flags, Error **errp) { ERRP_GUARD(); int ret; BlockDriverState *new_node_bs; - new_node_bs = bdrv_open(NULL, NULL, node_options, flags, errp); + new_node_bs = bdrv_open(NULL, NULL, options, flags, errp); if (new_node_bs == NULL) { error_prepend(errp, "Could not create node: "); return NULL; From b11c8739ae38166acac0669cee94b7e236ccb639 Mon Sep 17 00:00:00 2001 From: Vladimir Sementsov-Ogievskiy Date: Mon, 20 Sep 2021 14:55:37 +0300 Subject: [PATCH 06/13] block: bdrv_insert_node(): don't use bdrv_open() Use bdrv_new_open_driver_opts() instead of complicated bdrv_open(). Among other extra things bdrv_open() also check for white-listed formats, which we don't want for internal node creation: currently backup doesn't work when copy-before-write filter is not white-listed. As well block-stream doesn't work when copy-on-read is not white-listed. Fixes: 751cec7a261adaf1145dc7adf6de7c9c084e5a0b Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=2004812 Reported-by: Yanan Fu Signed-off-by: Vladimir Sementsov-Ogievskiy Message-Id: <20210920115538.264372-5-vsementsov@virtuozzo.com> Signed-off-by: Kevin Wolf --- block.c | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/block.c b/block.c index 3a90407b83..45f653a88b 100644 --- a/block.c +++ b/block.c @@ -5133,12 +5133,30 @@ BlockDriverState *bdrv_insert_node(BlockDriverState *bs, QDict *options, { ERRP_GUARD(); int ret; - BlockDriverState *new_node_bs; + BlockDriverState *new_node_bs = NULL; + const char *drvname, *node_name; + BlockDriver *drv; - new_node_bs = bdrv_open(NULL, NULL, options, flags, errp); - if (new_node_bs == NULL) { + drvname = qdict_get_try_str(options, "driver"); + if (!drvname) { + error_setg(errp, "driver is not specified"); + goto fail; + } + + drv = bdrv_find_format(drvname); + if (!drv) { + error_setg(errp, "Unknown driver: '%s'", drvname); + goto fail; + } + + node_name = qdict_get_try_str(options, "node-name"); + + new_node_bs = bdrv_new_open_driver_opts(drv, node_name, options, flags, + errp); + options = NULL; /* bdrv_new_open_driver() eats options */ + if (!new_node_bs) { error_prepend(errp, "Could not create node: "); - return NULL; + goto fail; } bdrv_drained_begin(bs); @@ -5147,11 +5165,15 @@ BlockDriverState *bdrv_insert_node(BlockDriverState *bs, QDict *options, if (ret < 0) { error_prepend(errp, "Could not replace node: "); - bdrv_unref(new_node_bs); - return NULL; + goto fail; } return new_node_bs; + +fail: + qobject_unref(options); + bdrv_unref(new_node_bs); + return NULL; } /* From d318fc20b2ecb785bfc74bd8ad9e0da9e47d2104 Mon Sep 17 00:00:00 2001 From: Vladimir Sementsov-Ogievskiy Date: Mon, 20 Sep 2021 14:55:38 +0300 Subject: [PATCH 07/13] iotests/image-fleecing: declare requirement of copy-before-write Now test fails if copy-before-write is not white-listed. Let's skip test instead. Fixes: c0605985696a19ef034fa25d04f53f3b3b383896 Signed-off-by: Vladimir Sementsov-Ogievskiy Message-Id: <20210920115538.264372-6-vsementsov@virtuozzo.com> Signed-off-by: Kevin Wolf --- tests/qemu-iotests/tests/image-fleecing | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/qemu-iotests/tests/image-fleecing b/tests/qemu-iotests/tests/image-fleecing index f6318492c6..35164e9036 100755 --- a/tests/qemu-iotests/tests/image-fleecing +++ b/tests/qemu-iotests/tests/image-fleecing @@ -28,6 +28,7 @@ from iotests import log, qemu_img, qemu_io, qemu_io_silent iotests.script_initialize( supported_fmts=['qcow2', 'qcow', 'qed', 'vmdk', 'vhdx', 'raw'], supported_platforms=['linux'], + required_fmts=['copy-before-write'], ) patterns = [('0x5d', '0', '64k'), From cc071629539dc1f303175a7e2d4ab854c0a8b20f Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 23 Sep 2021 09:04:36 -0400 Subject: [PATCH 08/13] block: introduce max_hw_iov for use in scsi-generic Linux limits the size of iovecs to 1024 (UIO_MAXIOV in the kernel sources, IOV_MAX in POSIX). Because of this, on some host adapters requests with many iovecs are rejected with -EINVAL by the io_submit() or readv()/writev() system calls. In fact, the same limit applies to SG_IO as well. To fix both the EINVAL and the possible performance issues from using fewer iovecs than allowed by Linux (some HBAs have max_segments as low as 128), introduce a separate entry in BlockLimits to hold the max_segments value from sysfs. This new limit is used only for SG_IO and clamped to bs->bl.max_iov anyway, just like max_hw_transfer is clamped to bs->bl.max_transfer. Reported-by: Halil Pasic Cc: Hanna Reitz Cc: Kevin Wolf Cc: qemu-block@nongnu.org Cc: qemu-stable@nongnu.org Fixes: 18473467d5 ("file-posix: try BLKSECTGET on block devices too, do not round to power of 2", 2021-06-25) Signed-off-by: Paolo Bonzini Message-Id: <20210923130436.1187591-1-pbonzini@redhat.com> Signed-off-by: Kevin Wolf --- block/block-backend.c | 6 ++++++ block/file-posix.c | 2 +- block/io.c | 1 + hw/scsi/scsi-generic.c | 2 +- include/block/block_int.h | 7 +++++++ include/sysemu/block-backend.h | 1 + 6 files changed, 17 insertions(+), 2 deletions(-) diff --git a/block/block-backend.c b/block/block-backend.c index 6140d133e2..ba2b5ebb10 100644 --- a/block/block-backend.c +++ b/block/block-backend.c @@ -1986,6 +1986,12 @@ uint32_t blk_get_max_transfer(BlockBackend *blk) return ROUND_DOWN(max, blk_get_request_alignment(blk)); } +int blk_get_max_hw_iov(BlockBackend *blk) +{ + return MIN_NON_ZERO(blk->root->bs->bl.max_hw_iov, + blk->root->bs->bl.max_iov); +} + int blk_get_max_iov(BlockBackend *blk) { return blk->root->bs->bl.max_iov; diff --git a/block/file-posix.c b/block/file-posix.c index c62e42743d..53be0bdc1b 100644 --- a/block/file-posix.c +++ b/block/file-posix.c @@ -1273,7 +1273,7 @@ static void raw_refresh_limits(BlockDriverState *bs, Error **errp) ret = hdev_get_max_segments(s->fd, &st); if (ret > 0) { - bs->bl.max_iov = ret; + bs->bl.max_hw_iov = ret; } } } diff --git a/block/io.c b/block/io.c index 18d345a87a..bb0a254def 100644 --- a/block/io.c +++ b/block/io.c @@ -136,6 +136,7 @@ static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src) dst->min_mem_alignment = MAX(dst->min_mem_alignment, src->min_mem_alignment); dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov); + dst->max_hw_iov = MIN_NON_ZERO(dst->max_hw_iov, src->max_hw_iov); } typedef struct BdrvRefreshLimitsState { diff --git a/hw/scsi/scsi-generic.c b/hw/scsi/scsi-generic.c index 665baf900e..0306ccc7b1 100644 --- a/hw/scsi/scsi-generic.c +++ b/hw/scsi/scsi-generic.c @@ -180,7 +180,7 @@ static int scsi_handle_inquiry_reply(SCSIGenericReq *r, SCSIDevice *s, int len) page = r->req.cmd.buf[2]; if (page == 0xb0) { uint64_t max_transfer = blk_get_max_hw_transfer(s->conf.blk); - uint32_t max_iov = blk_get_max_iov(s->conf.blk); + uint32_t max_iov = blk_get_max_hw_iov(s->conf.blk); assert(max_transfer); max_transfer = MIN_NON_ZERO(max_transfer, max_iov * qemu_real_host_page_size) diff --git a/include/block/block_int.h b/include/block/block_int.h index ffe86068d4..f4c75e8ba9 100644 --- a/include/block/block_int.h +++ b/include/block/block_int.h @@ -718,6 +718,13 @@ typedef struct BlockLimits { */ uint64_t max_hw_transfer; + /* Maximal number of scatter/gather elements allowed by the hardware. + * Applies whenever transfers to the device bypass the kernel I/O + * scheduler, for example with SG_IO. If larger than max_iov + * or if zero, blk_get_max_hw_iov will fall back to max_iov. + */ + int max_hw_iov; + /* memory alignment, in bytes so that no bounce buffer is needed */ size_t min_mem_alignment; diff --git a/include/sysemu/block-backend.h b/include/sysemu/block-backend.h index 29d4fdbf63..82bae55161 100644 --- a/include/sysemu/block-backend.h +++ b/include/sysemu/block-backend.h @@ -211,6 +211,7 @@ uint32_t blk_get_request_alignment(BlockBackend *blk); uint32_t blk_get_max_transfer(BlockBackend *blk); uint64_t blk_get_max_hw_transfer(BlockBackend *blk); int blk_get_max_iov(BlockBackend *blk); +int blk_get_max_hw_iov(BlockBackend *blk); void blk_set_guest_block_size(BlockBackend *blk, int align); void *blk_try_blockalign(BlockBackend *blk, size_t size); void *blk_blockalign(BlockBackend *blk, size_t size); From af6d4c56e15a45fb4d0cdf8d0335275b5ed8fbf7 Mon Sep 17 00:00:00 2001 From: John Snow Date: Thu, 23 Sep 2021 14:07:10 -0400 Subject: [PATCH 09/13] iotests: add 'qemu' package location to PYTHONPATH in testenv MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We can drop the sys.path hacking in various places by doing this. Additionally, by doing it in one place right up top, we can print interesting warnings in case the environment does not look correct. (See next commit.) If we ever decide to change how the environment is crafted, all of the "help me find my python packages" goop is all in one place, right in one function. Signed-off-by: John Snow Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Kevin Wolf Message-Id: <20210923180715.4168522-2-jsnow@redhat.com> Signed-off-by: Kevin Wolf --- tests/qemu-iotests/235 | 2 -- tests/qemu-iotests/297 | 6 ------ tests/qemu-iotests/300 | 5 ++--- tests/qemu-iotests/iotests.py | 2 -- tests/qemu-iotests/testenv.py | 15 +++++++++------ tests/qemu-iotests/tests/mirror-top-perms | 7 +++---- 6 files changed, 14 insertions(+), 23 deletions(-) diff --git a/tests/qemu-iotests/235 b/tests/qemu-iotests/235 index 8aed45f9a7..4de920c380 100755 --- a/tests/qemu-iotests/235 +++ b/tests/qemu-iotests/235 @@ -24,8 +24,6 @@ import os import iotests from iotests import qemu_img_create, qemu_io, file_path, log -sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) - from qemu.machine import QEMUMachine iotests.script_initialize(supported_fmts=['qcow2']) diff --git a/tests/qemu-iotests/297 b/tests/qemu-iotests/297 index b04cba5366..467b712280 100755 --- a/tests/qemu-iotests/297 +++ b/tests/qemu-iotests/297 @@ -68,12 +68,6 @@ def run_linters(): # Todo notes are fine, but fixme's or xxx's should probably just be # fixed (in tests, at least) env = os.environ.copy() - qemu_module_path = os.path.join(os.path.dirname(__file__), - '..', '..', 'python') - try: - env['PYTHONPATH'] += os.pathsep + qemu_module_path - except KeyError: - env['PYTHONPATH'] = qemu_module_path subprocess.run(('pylint-3', '--score=n', '--notes=FIXME,XXX', *files), env=env, check=False) diff --git a/tests/qemu-iotests/300 b/tests/qemu-iotests/300 index fe94de84ed..10f9f2a8da 100755 --- a/tests/qemu-iotests/300 +++ b/tests/qemu-iotests/300 @@ -24,11 +24,10 @@ import random import re from typing import Dict, List, Optional +from qemu.machine import machine + import iotests -# Import qemu after iotests.py has amended sys.path -# pylint: disable=wrong-import-order -from qemu.machine import machine BlockBitmapMapping = List[Dict[str, object]] diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index ce06cf5630..b06ad76e0c 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -36,8 +36,6 @@ import unittest from contextlib import contextmanager -# pylint: disable=import-error, wrong-import-position -sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) from qemu.machine import qtest from qemu.qmp import QMPMessage diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py index 70da0d60c8..99a57a69f3 100644 --- a/tests/qemu-iotests/testenv.py +++ b/tests/qemu-iotests/testenv.py @@ -108,12 +108,15 @@ class TestEnv(ContextManager['TestEnv']): SAMPLE_IMG_DIR OUTPUT_DIR """ - self.pythonpath = os.getenv('PYTHONPATH') - if self.pythonpath: - self.pythonpath = self.source_iotests + os.pathsep + \ - self.pythonpath - else: - self.pythonpath = self.source_iotests + + # Path where qemu goodies live in this source tree. + qemu_srctree_path = Path(__file__, '../../../python').resolve() + + self.pythonpath = os.pathsep.join(filter(None, ( + self.source_iotests, + str(qemu_srctree_path), + os.getenv('PYTHONPATH'), + ))) self.test_dir = os.getenv('TEST_DIR', os.path.join(os.getcwd(), 'scratch')) diff --git a/tests/qemu-iotests/tests/mirror-top-perms b/tests/qemu-iotests/tests/mirror-top-perms index 2fc8dd66e0..73138a0ef9 100755 --- a/tests/qemu-iotests/tests/mirror-top-perms +++ b/tests/qemu-iotests/tests/mirror-top-perms @@ -20,13 +20,12 @@ # import os + +import qemu + import iotests from iotests import qemu_img -# Import qemu after iotests.py has amended sys.path -# pylint: disable=wrong-import-order -import qemu - image_size = 1 * 1024 * 1024 source = os.path.join(iotests.test_dir, 'source.img') From f39decb583669e4335eaf2d1a5b8183254df51d6 Mon Sep 17 00:00:00 2001 From: John Snow Date: Thu, 23 Sep 2021 14:07:12 -0400 Subject: [PATCH 10/13] iotests/linters: check mypy files all at once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We can circumvent the '__main__' redefinition problem by passing --scripts-are-modules. Take mypy out of the loop per-filename and check everything in one go: it's quite a bit faster. Signed-off-by: John Snow Reviewed-by: Hanna Reitz Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Kevin Wolf Message-Id: <20210923180715.4168522-4-jsnow@redhat.com> Signed-off-by: Kevin Wolf --- tests/qemu-iotests/297 | 44 +++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/tests/qemu-iotests/297 b/tests/qemu-iotests/297 index 467b712280..91ec34d952 100755 --- a/tests/qemu-iotests/297 +++ b/tests/qemu-iotests/297 @@ -74,32 +74,28 @@ def run_linters(): print('=== mypy ===') sys.stdout.flush() - # We have to call mypy separately for each file. Otherwise, it - # will interpret all given files as belonging together (i.e., they - # may not both define the same classes, etc.; most notably, they - # must not both define the __main__ module). env['MYPYPATH'] = env['PYTHONPATH'] - for filename in files: - p = subprocess.run(('mypy', - '--warn-unused-configs', - '--disallow-subclassing-any', - '--disallow-any-generics', - '--disallow-incomplete-defs', - '--disallow-untyped-decorators', - '--no-implicit-optional', - '--warn-redundant-casts', - '--warn-unused-ignores', - '--no-implicit-reexport', - '--namespace-packages', - filename), - env=env, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True) + p = subprocess.run(('mypy', + '--warn-unused-configs', + '--disallow-subclassing-any', + '--disallow-any-generics', + '--disallow-incomplete-defs', + '--disallow-untyped-decorators', + '--no-implicit-optional', + '--warn-redundant-casts', + '--warn-unused-ignores', + '--no-implicit-reexport', + '--namespace-packages', + '--scripts-are-modules', + *files), + env=env, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) - if p.returncode != 0: - print(p.stdout) + if p.returncode != 0: + print(p.stdout) for linter in ('pylint-3', 'mypy'): From ac7424631943c0c015934a3c92dca87b70f8f8e9 Mon Sep 17 00:00:00 2001 From: John Snow Date: Thu, 23 Sep 2021 14:07:13 -0400 Subject: [PATCH 11/13] iotests/mirror-top-perms: Adjust imports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We need to import subpackages from the qemu namespace package; importing the namespace package alone doesn't bring the subpackages with it -- unless someone else (like iotests.py) imports them too. Adjust the imports. Signed-off-by: John Snow Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Hanna Reitz Reviewed-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Kevin Wolf Message-Id: <20210923180715.4168522-5-jsnow@redhat.com> Signed-off-by: Kevin Wolf --- tests/qemu-iotests/tests/mirror-top-perms | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/qemu-iotests/tests/mirror-top-perms b/tests/qemu-iotests/tests/mirror-top-perms index 73138a0ef9..3d475aa3a5 100755 --- a/tests/qemu-iotests/tests/mirror-top-perms +++ b/tests/qemu-iotests/tests/mirror-top-perms @@ -21,7 +21,8 @@ import os -import qemu +from qemu import qmp +from qemu.machine import machine import iotests from iotests import qemu_img @@ -46,7 +47,7 @@ class TestMirrorTopPerms(iotests.QMPTestCase): def tearDown(self): try: self.vm.shutdown() - except qemu.machine.machine.AbnormalShutdown: + except machine.AbnormalShutdown: pass if self.vm_b is not None: @@ -101,7 +102,7 @@ class TestMirrorTopPerms(iotests.QMPTestCase): self.vm_b.launch() print('ERROR: VM B launched successfully, this should not have ' 'happened') - except qemu.qmp.QMPConnectError: + except qmp.QMPConnectError: assert 'Is another process using the image' in self.vm_b.get_log() result = self.vm.qmp('block-job-cancel', From 22968996946d1a4eaca7396099ba40867bb58642 Mon Sep 17 00:00:00 2001 From: John Snow Date: Thu, 23 Sep 2021 14:07:14 -0400 Subject: [PATCH 12/13] iotests/migrate-bitmaps-test: delint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mostly uninteresting stuff. Move the test injections under a function named main() so that the variables used during that process aren't in the global scope. Signed-off-by: John Snow Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Hanna Reitz Reviewed-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Kevin Wolf Message-Id: <20210923180715.4168522-6-jsnow@redhat.com> Signed-off-by: Kevin Wolf --- tests/qemu-iotests/tests/migrate-bitmaps-test | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/tests/qemu-iotests/tests/migrate-bitmaps-test b/tests/qemu-iotests/tests/migrate-bitmaps-test index dc431c35b3..c23df3d75c 100755 --- a/tests/qemu-iotests/tests/migrate-bitmaps-test +++ b/tests/qemu-iotests/tests/migrate-bitmaps-test @@ -19,10 +19,11 @@ # along with this program. If not, see . # -import os import itertools import operator +import os import re + import iotests from iotests import qemu_img, qemu_img_create, Timeout @@ -224,25 +225,6 @@ def inject_test_case(klass, suffix, method, *args, **kwargs): setattr(klass, 'test_' + method + suffix, lambda self: mc(self)) -for cmb in list(itertools.product((True, False), repeat=5)): - name = ('_' if cmb[0] else '_not_') + 'persistent_' - name += ('_' if cmb[1] else '_not_') + 'migbitmap_' - name += '_online' if cmb[2] else '_offline' - name += '_shared' if cmb[3] else '_nonshared' - if cmb[4]: - name += '__pre_shutdown' - - inject_test_case(TestDirtyBitmapMigration, name, 'do_test_migration', - *list(cmb)) - -for cmb in list(itertools.product((True, False), repeat=2)): - name = ('_' if cmb[0] else '_not_') + 'persistent_' - name += ('_' if cmb[1] else '_not_') + 'migbitmap' - - inject_test_case(TestDirtyBitmapMigration, name, - 'do_test_migration_resume_source', *list(cmb)) - - class TestDirtyBitmapBackingMigration(iotests.QMPTestCase): def setUp(self): qemu_img_create('-f', iotests.imgfmt, base_a, size) @@ -304,6 +286,30 @@ class TestDirtyBitmapBackingMigration(iotests.QMPTestCase): self.assert_qmp(result, 'return', {}) +def main() -> None: + for cmb in list(itertools.product((True, False), repeat=5)): + name = ('_' if cmb[0] else '_not_') + 'persistent_' + name += ('_' if cmb[1] else '_not_') + 'migbitmap_' + name += '_online' if cmb[2] else '_offline' + name += '_shared' if cmb[3] else '_nonshared' + if cmb[4]: + name += '__pre_shutdown' + + inject_test_case(TestDirtyBitmapMigration, name, 'do_test_migration', + *list(cmb)) + + for cmb in list(itertools.product((True, False), repeat=2)): + name = ('_' if cmb[0] else '_not_') + 'persistent_' + name += ('_' if cmb[1] else '_not_') + 'migbitmap' + + inject_test_case(TestDirtyBitmapMigration, name, + 'do_test_migration_resume_source', *list(cmb)) + + iotests.main( + supported_fmts=['qcow2'], + supported_protocols=['file'] + ) + + if __name__ == '__main__': - iotests.main(supported_fmts=['qcow2'], - supported_protocols=['file']) + main() From 3765315d4c84f9c0799744f43a314169baaccc05 Mon Sep 17 00:00:00 2001 From: John Snow Date: Thu, 23 Sep 2021 14:07:15 -0400 Subject: [PATCH 13/13] iotests: Update for pylint 2.11.1 1. Ignore the new f-strings warning, we're not interested in doing a full conversion at this time. 2. Just mute the unbalanced-tuple-unpacking warning, it's not a real error in this case and muting the dozens of callsites is just not worth it. 3. Add encodings to read_text(). Signed-off-by: John Snow Reviewed-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Kevin Wolf Message-Id: <20210923180715.4168522-7-jsnow@redhat.com> Signed-off-by: Kevin Wolf --- tests/qemu-iotests/pylintrc | 6 +++++- tests/qemu-iotests/testrunner.py | 7 ++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/qemu-iotests/pylintrc b/tests/qemu-iotests/pylintrc index f2c0b522ac..8cb4e1d6a6 100644 --- a/tests/qemu-iotests/pylintrc +++ b/tests/qemu-iotests/pylintrc @@ -19,13 +19,17 @@ disable=invalid-name, too-many-public-methods, # pylint warns about Optional[] etc. as unsubscriptable in 3.9 unsubscriptable-object, + # pylint's static analysis causes false positivies for file_path(); + # If we really care to make it statically knowable, we'll use mypy. + unbalanced-tuple-unpacking, # Sometimes we need to disable a newly introduced pylint warning. # Doing so should not produce a warning in older versions of pylint. bad-option-value, # These are temporary, and should be removed: missing-docstring, too-many-return-statements, - too-many-statements + too-many-statements, + consider-using-f-string, [FORMAT] diff --git a/tests/qemu-iotests/testrunner.py b/tests/qemu-iotests/testrunner.py index 4a6ec421ed..a56b6da396 100644 --- a/tests/qemu-iotests/testrunner.py +++ b/tests/qemu-iotests/testrunner.py @@ -266,12 +266,13 @@ class TestRunner(ContextManager['TestRunner']): diff=file_diff(str(f_reference), str(f_bad))) if f_notrun.exists(): - return TestResult(status='not run', - description=f_notrun.read_text().strip()) + return TestResult( + status='not run', + description=f_notrun.read_text(encoding='utf-8').strip()) casenotrun = '' if f_casenotrun.exists(): - casenotrun = f_casenotrun.read_text() + casenotrun = f_casenotrun.read_text(encoding='utf-8') diff = file_diff(str(f_reference), str(f_bad)) if diff: