diff --git a/block.c b/block.c index 5ce08a79fd..45f653a88b 100644 --- a/block.c +++ b/block.c @@ -1604,16 +1604,26 @@ open_failed: return ret; } -BlockDriverState *bdrv_new_open_driver(BlockDriver *drv, const char *node_name, - int flags, Error **errp) +/* + * Create and open a block node. + * + * @options is a QDict of options to pass to the block drivers, or NULL for an + * empty set of options. The reference to the QDict belongs to the block layer + * after the call (even on failure), so if the caller intends to reuse the + * dictionary, it needs to use qobject_ref() before calling bdrv_open. + */ +BlockDriverState *bdrv_new_open_driver_opts(BlockDriver *drv, + const char *node_name, + QDict *options, int flags, + Error **errp) { BlockDriverState *bs; int ret; bs = bdrv_new(); bs->open_flags = flags; - bs->explicit_options = qdict_new(); - bs->options = qdict_new(); + bs->options = options ?: qdict_new(); + bs->explicit_options = qdict_clone_shallow(bs->options); bs->opaque = NULL; update_options_from_flags(bs->options, flags); @@ -1631,6 +1641,13 @@ BlockDriverState *bdrv_new_open_driver(BlockDriver *drv, const char *node_name, return bs; } +/* Create and open a block node. */ +BlockDriverState *bdrv_new_open_driver(BlockDriver *drv, const char *node_name, + int flags, Error **errp) +{ + return bdrv_new_open_driver_opts(drv, node_name, NULL, flags, errp); +} + QemuOptsList bdrv_runtime_opts = { .name = "bdrv_common", .head = QTAILQ_HEAD_INITIALIZER(bdrv_runtime_opts.head), @@ -5102,29 +5119,61 @@ static void bdrv_delete(BlockDriverState *bs) g_free(bs); } -BlockDriverState *bdrv_insert_node(BlockDriverState *bs, QDict *node_options, + +/* + * Replace @bs by newly created block node. + * + * @options is a QDict of options to pass to the block drivers, or NULL for an + * empty set of options. The reference to the QDict belongs to the block layer + * after the call (even on failure), so if the caller intends to reuse the + * dictionary, it needs to use qobject_ref() before calling bdrv_open. + */ +BlockDriverState *bdrv_insert_node(BlockDriverState *bs, QDict *options, int flags, Error **errp) { - BlockDriverState *new_node_bs; - Error *local_err = NULL; + ERRP_GUARD(); + int ret; + BlockDriverState *new_node_bs = NULL; + const char *drvname, *node_name; + BlockDriver *drv; - new_node_bs = bdrv_open(NULL, NULL, node_options, flags, errp); - if (new_node_bs == NULL) { + drvname = qdict_get_try_str(options, "driver"); + if (!drvname) { + error_setg(errp, "driver is not specified"); + goto fail; + } + + drv = bdrv_find_format(drvname); + if (!drv) { + error_setg(errp, "Unknown driver: '%s'", drvname); + goto fail; + } + + node_name = qdict_get_try_str(options, "node-name"); + + new_node_bs = bdrv_new_open_driver_opts(drv, node_name, options, flags, + errp); + options = NULL; /* bdrv_new_open_driver() eats options */ + if (!new_node_bs) { error_prepend(errp, "Could not create node: "); - return NULL; + goto fail; } bdrv_drained_begin(bs); - bdrv_replace_node(bs, new_node_bs, &local_err); + ret = bdrv_replace_node(bs, new_node_bs, errp); bdrv_drained_end(bs); - if (local_err) { - bdrv_unref(new_node_bs); - error_propagate(errp, local_err); - return NULL; + if (ret < 0) { + error_prepend(errp, "Could not replace node: "); + goto fail; } return new_node_bs; + +fail: + qobject_unref(options); + bdrv_unref(new_node_bs); + return NULL; } /* diff --git a/block/block-backend.c b/block/block-backend.c index 6140d133e2..ba2b5ebb10 100644 --- a/block/block-backend.c +++ b/block/block-backend.c @@ -1986,6 +1986,12 @@ uint32_t blk_get_max_transfer(BlockBackend *blk) return ROUND_DOWN(max, blk_get_request_alignment(blk)); } +int blk_get_max_hw_iov(BlockBackend *blk) +{ + return MIN_NON_ZERO(blk->root->bs->bl.max_hw_iov, + blk->root->bs->bl.max_iov); +} + int blk_get_max_iov(BlockBackend *blk) { return blk->root->bs->bl.max_iov; diff --git a/block/file-posix.c b/block/file-posix.c index c62e42743d..53be0bdc1b 100644 --- a/block/file-posix.c +++ b/block/file-posix.c @@ -1273,7 +1273,7 @@ static void raw_refresh_limits(BlockDriverState *bs, Error **errp) ret = hdev_get_max_segments(s->fd, &st); if (ret > 0) { - bs->bl.max_iov = ret; + bs->bl.max_hw_iov = ret; } } } diff --git a/block/io.c b/block/io.c index 18d345a87a..bb0a254def 100644 --- a/block/io.c +++ b/block/io.c @@ -136,6 +136,7 @@ static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src) dst->min_mem_alignment = MAX(dst->min_mem_alignment, src->min_mem_alignment); dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov); + dst->max_hw_iov = MIN_NON_ZERO(dst->max_hw_iov, src->max_hw_iov); } typedef struct BdrvRefreshLimitsState { diff --git a/hw/scsi/scsi-generic.c b/hw/scsi/scsi-generic.c index 665baf900e..0306ccc7b1 100644 --- a/hw/scsi/scsi-generic.c +++ b/hw/scsi/scsi-generic.c @@ -180,7 +180,7 @@ static int scsi_handle_inquiry_reply(SCSIGenericReq *r, SCSIDevice *s, int len) page = r->req.cmd.buf[2]; if (page == 0xb0) { uint64_t max_transfer = blk_get_max_hw_transfer(s->conf.blk); - uint32_t max_iov = blk_get_max_iov(s->conf.blk); + uint32_t max_iov = blk_get_max_hw_iov(s->conf.blk); assert(max_transfer); max_transfer = MIN_NON_ZERO(max_transfer, max_iov * qemu_real_host_page_size) diff --git a/include/block/block.h b/include/block/block.h index 740038a892..e5dd22b034 100644 --- a/include/block/block.h +++ b/include/block/block.h @@ -383,6 +383,10 @@ int bdrv_open_backing_file(BlockDriverState *bs, QDict *parent_options, const char *bdref_key, Error **errp); BlockDriverState *bdrv_open(const char *filename, const char *reference, QDict *options, int flags, Error **errp); +BlockDriverState *bdrv_new_open_driver_opts(BlockDriver *drv, + const char *node_name, + QDict *options, int flags, + Error **errp); BlockDriverState *bdrv_new_open_driver(BlockDriver *drv, const char *node_name, int flags, Error **errp); BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue, @@ -751,9 +755,7 @@ bool bdrv_drain_poll(BlockDriverState *bs, bool recursive, * bdrv_drained_begin: * * Begin a quiesced section for exclusive access to the BDS, by disabling - * external request sources including NBD server and device model. Note that - * this doesn't block timers or coroutines from submitting more requests, which - * means block_job_pause is still necessary. + * external request sources including NBD server, block jobs, and device model. * * This function can be recursive. */ diff --git a/include/block/block_int.h b/include/block/block_int.h index ffe86068d4..f4c75e8ba9 100644 --- a/include/block/block_int.h +++ b/include/block/block_int.h @@ -718,6 +718,13 @@ typedef struct BlockLimits { */ uint64_t max_hw_transfer; + /* Maximal number of scatter/gather elements allowed by the hardware. + * Applies whenever transfers to the device bypass the kernel I/O + * scheduler, for example with SG_IO. If larger than max_iov + * or if zero, blk_get_max_hw_iov will fall back to max_iov. + */ + int max_hw_iov; + /* memory alignment, in bytes so that no bounce buffer is needed */ size_t min_mem_alignment; diff --git a/include/sysemu/block-backend.h b/include/sysemu/block-backend.h index 29d4fdbf63..82bae55161 100644 --- a/include/sysemu/block-backend.h +++ b/include/sysemu/block-backend.h @@ -211,6 +211,7 @@ uint32_t blk_get_request_alignment(BlockBackend *blk); uint32_t blk_get_max_transfer(BlockBackend *blk); uint64_t blk_get_max_hw_transfer(BlockBackend *blk); int blk_get_max_iov(BlockBackend *blk); +int blk_get_max_hw_iov(BlockBackend *blk); void blk_set_guest_block_size(BlockBackend *blk, int align); void *blk_try_blockalign(BlockBackend *blk, size_t size); void *blk_blockalign(BlockBackend *blk, size_t size); diff --git a/storage-daemon/qemu-storage-daemon.c b/storage-daemon/qemu-storage-daemon.c index fc8b150629..10a1a33761 100644 --- a/storage-daemon/qemu-storage-daemon.c +++ b/storage-daemon/qemu-storage-daemon.c @@ -98,10 +98,12 @@ static void help(void) " export the specified block node over NBD\n" " (requires --nbd-server)\n" "\n" +#ifdef CONFIG_FUSE " --export [type=]fuse,id=,node-name=,mountpoint=\n" " [,growable=on|off][,writable=on|off]\n" " export the specified block node over FUSE\n" "\n" +#endif /* CONFIG_FUSE */ " --monitor [chardev=]name[,mode=control][,pretty[=on|off]]\n" " configure a QMP monitor\n" "\n" diff --git a/tests/qemu-iotests/235 b/tests/qemu-iotests/235 index 8aed45f9a7..4de920c380 100755 --- a/tests/qemu-iotests/235 +++ b/tests/qemu-iotests/235 @@ -24,8 +24,6 @@ import os import iotests from iotests import qemu_img_create, qemu_io, file_path, log -sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) - from qemu.machine import QEMUMachine iotests.script_initialize(supported_fmts=['qcow2']) diff --git a/tests/qemu-iotests/297 b/tests/qemu-iotests/297 index b04cba5366..91ec34d952 100755 --- a/tests/qemu-iotests/297 +++ b/tests/qemu-iotests/297 @@ -68,44 +68,34 @@ def run_linters(): # Todo notes are fine, but fixme's or xxx's should probably just be # fixed (in tests, at least) env = os.environ.copy() - qemu_module_path = os.path.join(os.path.dirname(__file__), - '..', '..', 'python') - try: - env['PYTHONPATH'] += os.pathsep + qemu_module_path - except KeyError: - env['PYTHONPATH'] = qemu_module_path subprocess.run(('pylint-3', '--score=n', '--notes=FIXME,XXX', *files), env=env, check=False) print('=== mypy ===') sys.stdout.flush() - # We have to call mypy separately for each file. Otherwise, it - # will interpret all given files as belonging together (i.e., they - # may not both define the same classes, etc.; most notably, they - # must not both define the __main__ module). env['MYPYPATH'] = env['PYTHONPATH'] - for filename in files: - p = subprocess.run(('mypy', - '--warn-unused-configs', - '--disallow-subclassing-any', - '--disallow-any-generics', - '--disallow-incomplete-defs', - '--disallow-untyped-decorators', - '--no-implicit-optional', - '--warn-redundant-casts', - '--warn-unused-ignores', - '--no-implicit-reexport', - '--namespace-packages', - filename), - env=env, - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True) + p = subprocess.run(('mypy', + '--warn-unused-configs', + '--disallow-subclassing-any', + '--disallow-any-generics', + '--disallow-incomplete-defs', + '--disallow-untyped-decorators', + '--no-implicit-optional', + '--warn-redundant-casts', + '--warn-unused-ignores', + '--no-implicit-reexport', + '--namespace-packages', + '--scripts-are-modules', + *files), + env=env, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) - if p.returncode != 0: - print(p.stdout) + if p.returncode != 0: + print(p.stdout) for linter in ('pylint-3', 'mypy'): diff --git a/tests/qemu-iotests/300 b/tests/qemu-iotests/300 index fe94de84ed..10f9f2a8da 100755 --- a/tests/qemu-iotests/300 +++ b/tests/qemu-iotests/300 @@ -24,11 +24,10 @@ import random import re from typing import Dict, List, Optional +from qemu.machine import machine + import iotests -# Import qemu after iotests.py has amended sys.path -# pylint: disable=wrong-import-order -from qemu.machine import machine BlockBitmapMapping = List[Dict[str, object]] diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index ce06cf5630..b06ad76e0c 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -36,8 +36,6 @@ import unittest from contextlib import contextmanager -# pylint: disable=import-error, wrong-import-position -sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) from qemu.machine import qtest from qemu.qmp import QMPMessage diff --git a/tests/qemu-iotests/pylintrc b/tests/qemu-iotests/pylintrc index f2c0b522ac..8cb4e1d6a6 100644 --- a/tests/qemu-iotests/pylintrc +++ b/tests/qemu-iotests/pylintrc @@ -19,13 +19,17 @@ disable=invalid-name, too-many-public-methods, # pylint warns about Optional[] etc. as unsubscriptable in 3.9 unsubscriptable-object, + # pylint's static analysis causes false positivies for file_path(); + # If we really care to make it statically knowable, we'll use mypy. + unbalanced-tuple-unpacking, # Sometimes we need to disable a newly introduced pylint warning. # Doing so should not produce a warning in older versions of pylint. bad-option-value, # These are temporary, and should be removed: missing-docstring, too-many-return-statements, - too-many-statements + too-many-statements, + consider-using-f-string, [FORMAT] diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py index 70da0d60c8..99a57a69f3 100644 --- a/tests/qemu-iotests/testenv.py +++ b/tests/qemu-iotests/testenv.py @@ -108,12 +108,15 @@ class TestEnv(ContextManager['TestEnv']): SAMPLE_IMG_DIR OUTPUT_DIR """ - self.pythonpath = os.getenv('PYTHONPATH') - if self.pythonpath: - self.pythonpath = self.source_iotests + os.pathsep + \ - self.pythonpath - else: - self.pythonpath = self.source_iotests + + # Path where qemu goodies live in this source tree. + qemu_srctree_path = Path(__file__, '../../../python').resolve() + + self.pythonpath = os.pathsep.join(filter(None, ( + self.source_iotests, + str(qemu_srctree_path), + os.getenv('PYTHONPATH'), + ))) self.test_dir = os.getenv('TEST_DIR', os.path.join(os.getcwd(), 'scratch')) diff --git a/tests/qemu-iotests/testrunner.py b/tests/qemu-iotests/testrunner.py index 4a6ec421ed..a56b6da396 100644 --- a/tests/qemu-iotests/testrunner.py +++ b/tests/qemu-iotests/testrunner.py @@ -266,12 +266,13 @@ class TestRunner(ContextManager['TestRunner']): diff=file_diff(str(f_reference), str(f_bad))) if f_notrun.exists(): - return TestResult(status='not run', - description=f_notrun.read_text().strip()) + return TestResult( + status='not run', + description=f_notrun.read_text(encoding='utf-8').strip()) casenotrun = '' if f_casenotrun.exists(): - casenotrun = f_casenotrun.read_text() + casenotrun = f_casenotrun.read_text(encoding='utf-8') diff = file_diff(str(f_reference), str(f_bad)) if diff: diff --git a/tests/qemu-iotests/tests/image-fleecing b/tests/qemu-iotests/tests/image-fleecing index f6318492c6..35164e9036 100755 --- a/tests/qemu-iotests/tests/image-fleecing +++ b/tests/qemu-iotests/tests/image-fleecing @@ -28,6 +28,7 @@ from iotests import log, qemu_img, qemu_io, qemu_io_silent iotests.script_initialize( supported_fmts=['qcow2', 'qcow', 'qed', 'vmdk', 'vhdx', 'raw'], supported_platforms=['linux'], + required_fmts=['copy-before-write'], ) patterns = [('0x5d', '0', '64k'), diff --git a/tests/qemu-iotests/tests/migrate-bitmaps-test b/tests/qemu-iotests/tests/migrate-bitmaps-test index dc431c35b3..c23df3d75c 100755 --- a/tests/qemu-iotests/tests/migrate-bitmaps-test +++ b/tests/qemu-iotests/tests/migrate-bitmaps-test @@ -19,10 +19,11 @@ # along with this program. If not, see . # -import os import itertools import operator +import os import re + import iotests from iotests import qemu_img, qemu_img_create, Timeout @@ -224,25 +225,6 @@ def inject_test_case(klass, suffix, method, *args, **kwargs): setattr(klass, 'test_' + method + suffix, lambda self: mc(self)) -for cmb in list(itertools.product((True, False), repeat=5)): - name = ('_' if cmb[0] else '_not_') + 'persistent_' - name += ('_' if cmb[1] else '_not_') + 'migbitmap_' - name += '_online' if cmb[2] else '_offline' - name += '_shared' if cmb[3] else '_nonshared' - if cmb[4]: - name += '__pre_shutdown' - - inject_test_case(TestDirtyBitmapMigration, name, 'do_test_migration', - *list(cmb)) - -for cmb in list(itertools.product((True, False), repeat=2)): - name = ('_' if cmb[0] else '_not_') + 'persistent_' - name += ('_' if cmb[1] else '_not_') + 'migbitmap' - - inject_test_case(TestDirtyBitmapMigration, name, - 'do_test_migration_resume_source', *list(cmb)) - - class TestDirtyBitmapBackingMigration(iotests.QMPTestCase): def setUp(self): qemu_img_create('-f', iotests.imgfmt, base_a, size) @@ -304,6 +286,30 @@ class TestDirtyBitmapBackingMigration(iotests.QMPTestCase): self.assert_qmp(result, 'return', {}) +def main() -> None: + for cmb in list(itertools.product((True, False), repeat=5)): + name = ('_' if cmb[0] else '_not_') + 'persistent_' + name += ('_' if cmb[1] else '_not_') + 'migbitmap_' + name += '_online' if cmb[2] else '_offline' + name += '_shared' if cmb[3] else '_nonshared' + if cmb[4]: + name += '__pre_shutdown' + + inject_test_case(TestDirtyBitmapMigration, name, 'do_test_migration', + *list(cmb)) + + for cmb in list(itertools.product((True, False), repeat=2)): + name = ('_' if cmb[0] else '_not_') + 'persistent_' + name += ('_' if cmb[1] else '_not_') + 'migbitmap' + + inject_test_case(TestDirtyBitmapMigration, name, + 'do_test_migration_resume_source', *list(cmb)) + + iotests.main( + supported_fmts=['qcow2'], + supported_protocols=['file'] + ) + + if __name__ == '__main__': - iotests.main(supported_fmts=['qcow2'], - supported_protocols=['file']) + main() diff --git a/tests/qemu-iotests/tests/mirror-top-perms b/tests/qemu-iotests/tests/mirror-top-perms index 2fc8dd66e0..3d475aa3a5 100755 --- a/tests/qemu-iotests/tests/mirror-top-perms +++ b/tests/qemu-iotests/tests/mirror-top-perms @@ -20,13 +20,13 @@ # import os + +from qemu import qmp +from qemu.machine import machine + import iotests from iotests import qemu_img -# Import qemu after iotests.py has amended sys.path -# pylint: disable=wrong-import-order -import qemu - image_size = 1 * 1024 * 1024 source = os.path.join(iotests.test_dir, 'source.img') @@ -47,7 +47,7 @@ class TestMirrorTopPerms(iotests.QMPTestCase): def tearDown(self): try: self.vm.shutdown() - except qemu.machine.machine.AbnormalShutdown: + except machine.AbnormalShutdown: pass if self.vm_b is not None: @@ -102,7 +102,7 @@ class TestMirrorTopPerms(iotests.QMPTestCase): self.vm_b.launch() print('ERROR: VM B launched successfully, this should not have ' 'happened') - except qemu.qmp.QMPConnectError: + except qmp.QMPConnectError: assert 'Is another process using the image' in self.vm_b.get_log() result = self.vm.qmp('block-job-cancel',