From a3f6be81aa95f4f80504face9bc6de6d2dc1743f Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Fri, 6 Nov 2020 21:03:40 +0000 Subject: [PATCH 01/19] meson: always include contrib/libvhost-user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit libvhost-user is needed when CONFIG_LINUX is set. The CONFIG_VHOST_USER check in meson.build is incorrect. In fact, no explicit check is needed since this dependency is not built by default. If something declares a dependency on libvhost-user then it will be built, otherwise it won't be built (i.e. on non-Linux hosts). This fixes ./configure --disable-vhost-user && make. Fixes: bc15e44cb2191bbb2318878acdf5038134e56394 ("configure: introduce --enable-vhost-user-blk-server") Reported-by: Philippe Mathieu-Daudé Reported-by: Michael S. Tsirkin Signed-off-by: Stefan Hajnoczi Message-Id: <20201106210340.698771-1-stefanha@redhat.com> Signed-off-by: Thomas Huth --- meson.build | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/meson.build b/meson.build index f5175010df..b473620321 100644 --- a/meson.build +++ b/meson.build @@ -1450,11 +1450,7 @@ trace_events_subdirs += [ 'util', ] -vhost_user = not_found -if 'CONFIG_VHOST_USER' in config_host - subdir('contrib/libvhost-user') -endif - +subdir('contrib/libvhost-user') subdir('qapi') subdir('qobject') subdir('stubs') From 45716765b1d0bb1741688e5347aea967b9492d28 Mon Sep 17 00:00:00 2001 From: Brad Smith Date: Tue, 27 Oct 2020 01:30:48 -0400 Subject: [PATCH 02/19] tests/vm: update openbsd to release 6.8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A double dash at the end of a package name removes ambiguity when the intent is to install a non-FLAVORed package. Signed-off-by: Brad Smith Reviewed-by: Gerd Hoffmann Tested-by: Gerd Hoffmann Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20201027053048.GB64546@humpty.home.comstyle.com> Signed-off-by: Thomas Huth --- tests/vm/openbsd | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/vm/openbsd b/tests/vm/openbsd index ad882a76a2..386b2c72f7 100755 --- a/tests/vm/openbsd +++ b/tests/vm/openbsd @@ -22,8 +22,8 @@ class OpenBSDVM(basevm.BaseVM): name = "openbsd" arch = "x86_64" - link = "https://cdn.openbsd.org/pub/OpenBSD/6.6/amd64/install66.iso" - csum = "b22e63df56e6266de6bbeed8e9be0fbe9ee2291551c5bc03f3cc2e4ab9436ee3" + link = "https://cdn.openbsd.org/pub/OpenBSD/6.8/amd64/install68.iso" + csum = "47e291fcc2d0c1a8ae0b66329f040b33af755b6adbd21739e20bb5ad56f62b6c" size = "20G" pkgs = [ # tools @@ -37,10 +37,10 @@ class OpenBSDVM(basevm.BaseVM): "bash", "gmake", "gsed", - "gettext", + "gettext-tools", # libs: usb - "libusb1", + "libusb1--", # libs: crypto "gnutls", From 8a47836548851ac00863a4f520ad761253ea4a86 Mon Sep 17 00:00:00 2001 From: Eduardo Habkost Date: Mon, 26 Oct 2020 08:52:38 -0400 Subject: [PATCH 03/19] device-crash-test: Check if path is actually an executable file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the transition to Meson, the build directory now have subdirectories named "qemu-system-*.p", and device-crash-test will try to execute them as if they were binaries. This results in errors like: PermissionError: [Errno 13] Permission denied: './qemu-system-or1k.p' When generating the default list of binaries to test, check if the path is actually a file and if it's executable. Reported-by: Thomas Huth Signed-off-by: Eduardo Habkost Message-Id: <20201026125238.2752882-1-ehabkost@redhat.com> Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Thomas Huth --- scripts/device-crash-test | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/device-crash-test b/scripts/device-crash-test index 866baf7058..04118669ba 100755 --- a/scripts/device-crash-test +++ b/scripts/device-crash-test @@ -383,7 +383,9 @@ def binariesToTest(args, testcase): if args.qemu: r = args.qemu else: - r = glob.glob('./qemu-system-*') + r = [f.path for f in os.scandir('.') + if f.name.startswith('qemu-system-') and + f.is_file() and os.access(f, os.X_OK)] return r From 3dc057923d3f3cd92ddc1685ff44022eec175c9c Mon Sep 17 00:00:00 2001 From: AlexChen Date: Wed, 4 Nov 2020 18:23:19 +0800 Subject: [PATCH 04/19] qtest: Fix bad printf format specifiers We should use printf format specifier "%u" instead of "%d" for argument of type "unsigned int". Reported-by: Euler Robot Signed-off-by: Alex Chen Message-Id: <5FA28117.3020802@huawei.com> Signed-off-by: Thomas Huth --- tests/qtest/arm-cpu-features.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/qtest/arm-cpu-features.c b/tests/qtest/arm-cpu-features.c index d20094d5a7..bc681a95d5 100644 --- a/tests/qtest/arm-cpu-features.c +++ b/tests/qtest/arm-cpu-features.c @@ -536,7 +536,7 @@ static void test_query_cpu_model_expansion_kvm(const void *data) if (kvm_supports_sve) { g_assert(vls != 0); max_vq = 64 - __builtin_clzll(vls); - sprintf(max_name, "sve%d", max_vq * 128); + sprintf(max_name, "sve%u", max_vq * 128); /* Enabling a supported length is of course fine. */ assert_sve_vls(qts, "host", vls, "{ %s: true }", max_name); @@ -556,7 +556,7 @@ static void test_query_cpu_model_expansion_kvm(const void *data) * unless all larger, supported vector lengths are also * disabled. */ - sprintf(name, "sve%d", vq * 128); + sprintf(name, "sve%u", vq * 128); error = g_strdup_printf("cannot disable %s", name); assert_error(qts, "host", error, "{ %s: true, %s: false }", @@ -569,7 +569,7 @@ static void test_query_cpu_model_expansion_kvm(const void *data) * we need at least one vector length enabled. */ vq = __builtin_ffsll(vls); - sprintf(name, "sve%d", vq * 128); + sprintf(name, "sve%u", vq * 128); error = g_strdup_printf("cannot disable %s", name); assert_error(qts, "host", error, "{ %s: false }", name); g_free(error); @@ -581,7 +581,7 @@ static void test_query_cpu_model_expansion_kvm(const void *data) } } if (vq <= SVE_MAX_VQ) { - sprintf(name, "sve%d", vq * 128); + sprintf(name, "sve%u", vq * 128); error = g_strdup_printf("cannot enable %s", name); assert_error(qts, "host", error, "{ %s: true }", name); g_free(error); From dccaea2514007dd526a54038ffcaef5fb9c95c0c Mon Sep 17 00:00:00 2001 From: AlexChen Date: Thu, 5 Nov 2020 23:03:36 +0800 Subject: [PATCH 05/19] tests/qtest/tpm: Remove redundant check in the tpm_test_swtpm_test() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 'addr' would not be NULL after checking 'succ' is valid, and it has been dereferenced in the previous code(args = g_strdup_printf()). So the check on 'addr' in the tpm_test_swtpm_test() is redundant. Remove it. Reported-by: Euler Robot Signed-off-by: Alex Chen Message-Id: <5FA41448.4040404@huawei.com> Reviewed-by: Marc-André Lureau Reviewed-by: Thomas Huth Signed-off-by: Thomas Huth --- tests/qtest/tpm-tests.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/qtest/tpm-tests.c b/tests/qtest/tpm-tests.c index 70c80f8379..0da3a8a4df 100644 --- a/tests/qtest/tpm-tests.c +++ b/tests/qtest/tpm-tests.c @@ -70,10 +70,8 @@ void tpm_test_swtpm_test(const char *src_tpm_path, tx_func *tx, qtest_end(); tpm_util_swtpm_kill(swtpm_pid); - if (addr) { - g_unlink(addr->u.q_unix.path); - qapi_free_SocketAddress(addr); - } + g_unlink(addr->u.q_unix.path); + qapi_free_SocketAddress(addr); } void tpm_test_swtpm_migration_test(const char *src_tpm_path, From 7433a6860bee36d570c69720760238252a92090b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Sun, 8 Nov 2020 23:19:15 +0100 Subject: [PATCH 06/19] gitlab-ci: Drop generic cache rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This cache rule is meant for Avocado artifacts, but affects all jobs. Moreover the 'acceptance_template' template already include a more detailled rule to cache artifacts. Signed-off-by: Philippe Mathieu-Daudé Message-Id: <20201108221925.2344515-2-philmd@redhat.com> Signed-off-by: Thomas Huth --- .gitlab-ci.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3b15ae5c30..5763318d37 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,12 +7,6 @@ stages: - build - test -# We assume GitLab has it's own caching set up for RPM/APT repositories so we -# just take care of avocado assets here. -cache: - paths: - - $HOME/avocado/data/cache - include: - local: '/.gitlab-ci.d/edk2.yml' - local: '/.gitlab-ci.d/opensbi.yml' From 3758e88bb8b38d54134a90f5afd6b21c7495ed5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Sun, 8 Nov 2020 21:45:22 +0100 Subject: [PATCH 07/19] MAINTAINERS: Add gitlab-pipeline-status script to GitLab CI section MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Do not let the gitlab-pipeline-status script unmaintained, add it to the 'GitLab Continuous Integration' section. Fixes: c02b2eac55e ("GitLab Gating CI: introduce pipeline-status contrib script") Signed-off-by: Philippe Mathieu-Daudé Message-Id: <20201108204535.2319870-5-philmd@redhat.com> Reviewed-by: Wainer dos Santos Moschetta Signed-off-by: Thomas Huth --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 0e60109d28..7b34ea0646 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3156,6 +3156,7 @@ S: Maintained F: .gitlab-ci.yml F: .gitlab-ci.d/crossbuilds.yml F: .gitlab-ci.d/*py +F: scripts/ci/gitlab-pipeline-status Guest Test Compilation Support M: Alex Bennée From f3a0208f24775c23c3db867a5e6df889d5132ed6 Mon Sep 17 00:00:00 2001 From: Alexander Bulekov Date: Fri, 6 Nov 2020 13:05:59 -0500 Subject: [PATCH 08/19] docs/fuzz: rST-ify the fuzzing documentation Signed-off-by: Alexander Bulekov Message-Id: <20201106180600.360110-2-alxndr@bu.edu> Signed-off-by: Thomas Huth --- MAINTAINERS | 2 +- docs/devel/fuzzing.rst | 236 +++++++++++++++++++++++++++++++++++++++++ docs/devel/fuzzing.txt | 214 ------------------------------------- docs/devel/index.rst | 1 + 4 files changed, 238 insertions(+), 215 deletions(-) create mode 100644 docs/devel/fuzzing.rst delete mode 100644 docs/devel/fuzzing.txt diff --git a/MAINTAINERS b/MAINTAINERS index 7b34ea0646..6c2df0bef3 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2525,7 +2525,7 @@ R: Thomas Huth S: Maintained F: tests/qtest/fuzz/ F: scripts/oss-fuzz/ -F: docs/devel/fuzzing.txt +F: docs/devel/fuzzing.rst Register API M: Alistair Francis diff --git a/docs/devel/fuzzing.rst b/docs/devel/fuzzing.rst new file mode 100644 index 0000000000..f19d75ceff --- /dev/null +++ b/docs/devel/fuzzing.rst @@ -0,0 +1,236 @@ +======== +Fuzzing +======== + +This document describes the virtual-device fuzzing infrastructure in QEMU and +how to use it to implement additional fuzzers. + +Basics +------ + +Fuzzing operates by passing inputs to an entry point/target function. The +fuzzer tracks the code coverage triggered by the input. Based on these +findings, the fuzzer mutates the input and repeats the fuzzing. + +To fuzz QEMU, we rely on libfuzzer. Unlike other fuzzers such as AFL, libfuzzer +is an *in-process* fuzzer. For the developer, this means that it is their +responsibility to ensure that state is reset between fuzzing-runs. + +Building the fuzzers +-------------------- + +*NOTE*: If possible, build a 32-bit binary. When forking, the 32-bit fuzzer is +much faster, since the page-map has a smaller size. This is due to the fact that +AddressSanitizer maps ~20TB of memory, as part of its detection. This results +in a large page-map, and a much slower ``fork()``. + +To build the fuzzers, install a recent version of clang: +Configure with (substitute the clang binaries with the version you installed). +Here, enable-sanitizers, is optional but it allows us to reliably detect bugs +such as out-of-bounds accesses, use-after-frees, double-frees etc.:: + + CC=clang-8 CXX=clang++-8 /path/to/configure --enable-fuzzing \ + --enable-sanitizers + +Fuzz targets are built similarly to system targets:: + + make i386-softmmu/fuzz + +This builds ``./i386-softmmu/qemu-fuzz-i386`` + +The first option to this command is: ``--fuzz-target=FUZZ_NAME`` +To list all of the available fuzzers run ``qemu-fuzz-i386`` with no arguments. + +For example:: + + ./i386-softmmu/qemu-fuzz-i386 --fuzz-target=virtio-scsi-fuzz + +Internally, libfuzzer parses all arguments that do not begin with ``"--"``. +Information about these is available by passing ``-help=1`` + +Now the only thing left to do is wait for the fuzzer to trigger potential +crashes. + +Useful libFuzzer flags +---------------------- + +As mentioned above, libFuzzer accepts some arguments. Passing ``-help=1`` will +list the available arguments. In particular, these arguments might be helpful: + +* ``CORPUS_DIR/`` : Specify a directory as the last argument to libFuzzer. + libFuzzer stores each "interesting" input in this corpus directory. The next + time you run libFuzzer, it will read all of the inputs from the corpus, and + continue fuzzing from there. You can also specify multiple directories. + libFuzzer loads existing inputs from all specified directories, but will only + write new ones to the first one specified. + +* ``-max_len=4096`` : specify the maximum byte-length of the inputs libFuzzer + will generate. + +* ``-close_fd_mask={1,2,3}`` : close, stderr, or both. Useful for targets that + trigger many debug/error messages, or create output on the serial console. + +* ``-jobs=4 -workers=4`` : These arguments configure libFuzzer to run 4 fuzzers in + parallel (4 fuzzing jobs in 4 worker processes). Alternatively, with only + ``-jobs=N``, libFuzzer automatically spawns a number of workers less than or equal + to half the available CPU cores. Replace 4 with a number appropriate for your + machine. Make sure to specify a ``CORPUS_DIR``, which will allow the parallel + fuzzers to share information about the interesting inputs they find. + +* ``-use_value_profile=1`` : For each comparison operation, libFuzzer computes + ``(caller_pc&4095) | (popcnt(Arg1 ^ Arg2) << 12)`` and places this in the + coverage table. Useful for targets with "magic" constants. If Arg1 came from + the fuzzer's input and Arg2 is a magic constant, then each time the Hamming + distance between Arg1 and Arg2 decreases, libFuzzer adds the input to the + corpus. + +* ``-shrink=1`` : Tries to make elements of the corpus "smaller". Might lead to + better coverage performance, depending on the target. + +Note that libFuzzer's exact behavior will depend on the version of +clang and libFuzzer used to build the device fuzzers. + +Generating Coverage Reports +--------------------------- + +Code coverage is a crucial metric for evaluating a fuzzer's performance. +libFuzzer's output provides a "cov: " column that provides a total number of +unique blocks/edges covered. To examine coverage on a line-by-line basis we +can use Clang coverage: + + 1. Configure libFuzzer to store a corpus of all interesting inputs (see + CORPUS_DIR above) + 2. ``./configure`` the QEMU build with :: + + --enable-fuzzing \ + --extra-cflags="-fprofile-instr-generate -fcoverage-mapping" + + 3. Re-run the fuzzer. Specify $CORPUS_DIR/* as an argument, telling libfuzzer + to execute all of the inputs in $CORPUS_DIR and exit. Once the process + exits, you should find a file, "default.profraw" in the working directory. + 4. Execute these commands to generate a detailed HTML coverage-report:: + + llvm-profdata merge -output=default.profdata default.profraw + llvm-cov show ./path/to/qemu-fuzz-i386 -instr-profile=default.profdata \ + --format html -output-dir=/path/to/output/report + +Adding a new fuzzer +------------------- + +Coverage over virtual devices can be improved by adding additional fuzzers. +Fuzzers are kept in ``tests/qtest/fuzz/`` and should be added to +``tests/qtest/fuzz/Makefile.include`` + +Fuzzers can rely on both qtest and libqos to communicate with virtual devices. + +1. Create a new source file. For example ``tests/qtest/fuzz/foo-device-fuzz.c``. + +2. Write the fuzzing code using the libqtest/libqos API. See existing fuzzers + for reference. + +3. Register the fuzzer in ``tests/fuzz/Makefile.include`` by appending the + corresponding object to fuzz-obj-y + +Fuzzers can be more-or-less thought of as special qtest programs which can +modify the qtest commands and/or qtest command arguments based on inputs +provided by libfuzzer. Libfuzzer passes a byte array and length. Commonly the +fuzzer loops over the byte-array interpreting it as a list of qtest commands, +addresses, or values. + +The Generic Fuzzer +------------------ + +Writing a fuzz target can be a lot of effort (especially if a device driver has +not be built-out within libqos). Many devices can be fuzzed to some degree, +without any device-specific code, using the generic-fuzz target. + +The generic-fuzz target is capable of fuzzing devices over their PIO, MMIO, +and DMA input-spaces. To apply the generic-fuzz to a device, we need to define +two env-variables, at minimum: + +* ``QEMU_FUZZ_ARGS=`` is the set of QEMU arguments used to configure a machine, with + the device attached. For example, if we want to fuzz the virtio-net device + attached to a pc-i440fx machine, we can specify:: + + QEMU_FUZZ_ARGS="-M pc -nodefaults -netdev user,id=user0 \ + -device virtio-net,netdev=user0" + +* ``QEMU_FUZZ_OBJECTS=`` is a set of space-delimited strings used to identify + the MemoryRegions that will be fuzzed. These strings are compared against + MemoryRegion names and MemoryRegion owner names, to decide whether each + MemoryRegion should be fuzzed. These strings support globbing. For the + virtio-net example, we could use one of :: + + QEMU_FUZZ_OBJECTS='virtio-net' + QEMU_FUZZ_OBJECTS='virtio*' + QEMU_FUZZ_OBJECTS='virtio* pcspk' # Fuzz the virtio devices and the speaker + QEMU_FUZZ_OBJECTS='*' # Fuzz the whole machine`` + +The ``"info mtree"`` and ``"info qom-tree"`` monitor commands can be especially +useful for identifying the ``MemoryRegion`` and ``Object`` names used for +matching. + +As a generic rule-of-thumb, the more ``MemoryRegions``/Devices we match, the +greater the input-space, and the smaller the probability of finding crashing +inputs for individual devices. As such, it is usually a good idea to limit the +fuzzer to only a few ``MemoryRegions``. + +To ensure that these env variables have been configured correctly, we can use:: + + ./qemu-fuzz-i386 --fuzz-target=generic-fuzz -runs=0 + +The output should contain a complete list of matched MemoryRegions. + +Implementation Details / Fuzzer Lifecycle +----------------------------------------- + +The fuzzer has two entrypoints that libfuzzer calls. libfuzzer provides it's +own ``main()``, which performs some setup, and calls the entrypoints: + +``LLVMFuzzerInitialize``: called prior to fuzzing. Used to initialize all of the +necessary state + +``LLVMFuzzerTestOneInput``: called for each fuzzing run. Processes the input and +resets the state at the end of each run. + +In more detail: + +``LLVMFuzzerInitialize`` parses the arguments to the fuzzer (must start with two +dashes, so they are ignored by libfuzzer ``main()``). Currently, the arguments +select the fuzz target. Then, the qtest client is initialized. If the target +requires qos, qgraph is set up and the QOM/LIBQOS modules are initialized. +Then the QGraph is walked and the QEMU cmd_line is determined and saved. + +After this, the ``vl.c:qemu_main`` is called to set up the guest. There are +target-specific hooks that can be called before and after qemu_main, for +additional setup(e.g. PCI setup, or VM snapshotting). + +``LLVMFuzzerTestOneInput``: Uses qtest/qos functions to act based on the fuzz +input. It is also responsible for manually calling ``main_loop_wait`` to ensure +that bottom halves are executed and any cleanup required before the next input. + +Since the same process is reused for many fuzzing runs, QEMU state needs to +be reset at the end of each run. There are currently two implemented +options for resetting state: + +- Reboot the guest between runs. + - *Pros*: Straightforward and fast for simple fuzz targets. + + - *Cons*: Depending on the device, does not reset all device state. If the + device requires some initialization prior to being ready for fuzzing (common + for QOS-based targets), this initialization needs to be done after each + reboot. + + - *Example target*: ``i440fx-qtest-reboot-fuzz`` + +- Run each test case in a separate forked process and copy the coverage + information back to the parent. This is fairly similar to AFL's "deferred" + fork-server mode [3] + + - *Pros*: Relatively fast. Devices only need to be initialized once. No need to + do slow reboots or vmloads. + + - *Cons*: Not officially supported by libfuzzer. Does not work well for + devices that rely on dedicated threads. + + - *Example target*: ``virtio-net-fork-fuzz`` diff --git a/docs/devel/fuzzing.txt b/docs/devel/fuzzing.txt deleted file mode 100644 index 03585c1a9b..0000000000 --- a/docs/devel/fuzzing.txt +++ /dev/null @@ -1,214 +0,0 @@ -= Fuzzing = - -== Introduction == - -This document describes the virtual-device fuzzing infrastructure in QEMU and -how to use it to implement additional fuzzers. - -== Basics == - -Fuzzing operates by passing inputs to an entry point/target function. The -fuzzer tracks the code coverage triggered by the input. Based on these -findings, the fuzzer mutates the input and repeats the fuzzing. - -To fuzz QEMU, we rely on libfuzzer. Unlike other fuzzers such as AFL, libfuzzer -is an _in-process_ fuzzer. For the developer, this means that it is their -responsibility to ensure that state is reset between fuzzing-runs. - -== Building the fuzzers == - -NOTE: If possible, build a 32-bit binary. When forking, the 32-bit fuzzer is -much faster, since the page-map has a smaller size. This is due to the fact that -AddressSanitizer mmaps ~20TB of memory, as part of its detection. This results -in a large page-map, and a much slower fork(). - -To build the fuzzers, install a recent version of clang: -Configure with (substitute the clang binaries with the version you installed). -Here, enable-sanitizers, is optional but it allows us to reliably detect bugs -such as out-of-bounds accesses, use-after-frees, double-frees etc. - - CC=clang-8 CXX=clang++-8 /path/to/configure --enable-fuzzing \ - --enable-sanitizers - -Fuzz targets are built similarly to system/softmmu: - - make i386-softmmu/fuzz - -This builds ./i386-softmmu/qemu-fuzz-i386 - -The first option to this command is: --fuzz-target=FUZZ_NAME -To list all of the available fuzzers run qemu-fuzz-i386 with no arguments. - -For example: - ./i386-softmmu/qemu-fuzz-i386 --fuzz-target=virtio-scsi-fuzz - -Internally, libfuzzer parses all arguments that do not begin with "--". -Information about these is available by passing -help=1 - -Now the only thing left to do is wait for the fuzzer to trigger potential -crashes. - -== Useful libFuzzer flags == - -As mentioned above, libFuzzer accepts some arguments. Passing -help=1 will list -the available arguments. In particular, these arguments might be helpful: - -$CORPUS_DIR/ : Specify a directory as the last argument to libFuzzer. libFuzzer -stores each "interesting" input in this corpus directory. The next time you run -libFuzzer, it will read all of the inputs from the corpus, and continue fuzzing -from there. You can also specify multiple directories. libFuzzer loads existing -inputs from all specified directories, but will only write new ones to the -first one specified. - --max_len=4096 : specify the maximum byte-length of the inputs libFuzzer will -generate. - --close_fd_mask={1,2,3} : close, stderr, or both. Useful for targets that -trigger many debug/error messages, or create output on the serial console. - --jobs=4 -workers=4 : These arguments configure libFuzzer to run 4 fuzzers in -parallel (4 fuzzing jobs in 4 worker processes). Alternatively, with only --jobs=N, libFuzzer automatically spawns a number of workers less than or equal -to half the available CPU cores. Replace 4 with a number appropriate for your -machine. Make sure to specify a $CORPUS_DIR, which will allow the parallel -fuzzers to share information about the interesting inputs they find. - --use_value_profile=1 : For each comparison operation, libFuzzer computes -(caller_pc&4095) | (popcnt(Arg1 ^ Arg2) << 12) and places this in the coverage -table. Useful for targets with "magic" constants. If Arg1 came from the fuzzer's -input and Arg2 is a magic constant, then each time the Hamming distance -between Arg1 and Arg2 decreases, libFuzzer adds the input to the corpus. - --shrink=1 : Tries to make elements of the corpus "smaller". Might lead to -better coverage performance, depending on the target. - -Note that libFuzzer's exact behavior will depend on the version of -clang and libFuzzer used to build the device fuzzers. - -== Generating Coverage Reports == -Code coverage is a crucial metric for evaluating a fuzzer's performance. -libFuzzer's output provides a "cov: " column that provides a total number of -unique blocks/edges covered. To examine coverage on a line-by-line basis we -can use Clang coverage: - - 1. Configure libFuzzer to store a corpus of all interesting inputs (see - CORPUS_DIR above) - 2. ./configure the QEMU build with: - --enable-fuzzing \ - --extra-cflags="-fprofile-instr-generate -fcoverage-mapping" - 3. Re-run the fuzzer. Specify $CORPUS_DIR/* as an argument, telling libfuzzer - to execute all of the inputs in $CORPUS_DIR and exit. Once the process - exits, you should find a file, "default.profraw" in the working directory. - 4. Execute these commands to generate a detailed HTML coverage-report: - llvm-profdata merge -output=default.profdata default.profraw - llvm-cov show ./path/to/qemu-fuzz-i386 -instr-profile=default.profdata \ - --format html -output-dir=/path/to/output/report - -== Adding a new fuzzer == -Coverage over virtual devices can be improved by adding additional fuzzers. -Fuzzers are kept in tests/qtest/fuzz/ and should be added to -tests/qtest/fuzz/Makefile.include - -Fuzzers can rely on both qtest and libqos to communicate with virtual devices. - -1. Create a new source file. For example ``tests/qtest/fuzz/foo-device-fuzz.c``. - -2. Write the fuzzing code using the libqtest/libqos API. See existing fuzzers -for reference. - -3. Register the fuzzer in ``tests/fuzz/Makefile.include`` by appending the -corresponding object to fuzz-obj-y - -Fuzzers can be more-or-less thought of as special qtest programs which can -modify the qtest commands and/or qtest command arguments based on inputs -provided by libfuzzer. Libfuzzer passes a byte array and length. Commonly the -fuzzer loops over the byte-array interpreting it as a list of qtest commands, -addresses, or values. - -== The Generic Fuzzer == -Writing a fuzz target can be a lot of effort (especially if a device driver has -not be built-out within libqos). Many devices can be fuzzed to some degree, -without any device-specific code, using the generic-fuzz target. - -The generic-fuzz target is capable of fuzzing devices over their PIO, MMIO, -and DMA input-spaces. To apply the generic-fuzz to a device, we need to define -two env-variables, at minimum: - -QEMU_FUZZ_ARGS= is the set of QEMU arguments used to configure a machine, with -the device attached. For example, if we want to fuzz the virtio-net device -attached to a pc-i440fx machine, we can specify: -QEMU_FUZZ_ARGS="-M pc -nodefaults -netdev user,id=user0 \ - -device virtio-net,netdev=user0" - -QEMU_FUZZ_OBJECTS= is a set of space-delimited strings used to identify the -MemoryRegions that will be fuzzed. These strings are compared against -MemoryRegion names and MemoryRegion owner names, to decide whether each -MemoryRegion should be fuzzed. These strings support globbing. For the -virtio-net example, we could use QEMU_FUZZ_OBJECTS= - * 'virtio-net' - * 'virtio*' - * 'virtio* pcspk' (Fuzz the virtio devices and the PC speaker...) - * '*' (Fuzz the whole machine) - -The "info mtree" and "info qom-tree" monitor commands can be especially useful -for identifying the MemoryRegion and Object names used for matching. - -As a generic rule-of-thumb, the more MemoryRegions/Devices we match, the greater -the input-space, and the smaller the probability of finding crashing inputs for -individual devices. As such, it is usually a good idea to limit the fuzzer to -only a few MemoryRegions. - -To ensure that these env variables have been configured correctly, we can use: - -./qemu-fuzz-i386 --fuzz-target=generic-fuzz -runs=0 - -The output should contain a complete list of matched MemoryRegions. - -= Implementation Details = - -== The Fuzzer's Lifecycle == - -The fuzzer has two entrypoints that libfuzzer calls. libfuzzer provides it's -own main(), which performs some setup, and calls the entrypoints: - -LLVMFuzzerInitialize: called prior to fuzzing. Used to initialize all of the -necessary state - -LLVMFuzzerTestOneInput: called for each fuzzing run. Processes the input and -resets the state at the end of each run. - -In more detail: - -LLVMFuzzerInitialize parses the arguments to the fuzzer (must start with two -dashes, so they are ignored by libfuzzer main()). Currently, the arguments -select the fuzz target. Then, the qtest client is initialized. If the target -requires qos, qgraph is set up and the QOM/LIBQOS modules are initialized. -Then the QGraph is walked and the QEMU cmd_line is determined and saved. - -After this, the vl.c:qemu__main is called to set up the guest. There are -target-specific hooks that can be called before and after qemu_main, for -additional setup(e.g. PCI setup, or VM snapshotting). - -LLVMFuzzerTestOneInput: Uses qtest/qos functions to act based on the fuzz -input. It is also responsible for manually calling the main loop/main_loop_wait -to ensure that bottom halves are executed and any cleanup required before the -next input. - -Since the same process is reused for many fuzzing runs, QEMU state needs to -be reset at the end of each run. There are currently two implemented -options for resetting state: -1. Reboot the guest between runs. - Pros: Straightforward and fast for simple fuzz targets. - Cons: Depending on the device, does not reset all device state. If the - device requires some initialization prior to being ready for fuzzing - (common for QOS-based targets), this initialization needs to be done after - each reboot. - Example target: i440fx-qtest-reboot-fuzz -2. Run each test case in a separate forked process and copy the coverage - information back to the parent. This is fairly similar to AFL's "deferred" - fork-server mode [3] - Pros: Relatively fast. Devices only need to be initialized once. No need - to do slow reboots or vmloads. - Cons: Not officially supported by libfuzzer. Does not work well for devices - that rely on dedicated threads. - Example target: virtio-net-fork-fuzz diff --git a/docs/devel/index.rst b/docs/devel/index.rst index 77baae5c77..f10ed77e4c 100644 --- a/docs/devel/index.rst +++ b/docs/devel/index.rst @@ -22,6 +22,7 @@ Contents: stable-process testing qtest + fuzzing decodetree secure-coding-practices tcg From e6a3e1322ba9e05a7919d9cd10d05c8c23fa8698 Mon Sep 17 00:00:00 2001 From: Alexander Bulekov Date: Fri, 6 Nov 2020 13:06:00 -0500 Subject: [PATCH 09/19] docs/fuzz: update fuzzing documentation post-meson Signed-off-by: Alexander Bulekov Message-Id: <20201106180600.360110-3-alxndr@bu.edu> Signed-off-by: Thomas Huth --- docs/devel/fuzzing.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/devel/fuzzing.rst b/docs/devel/fuzzing.rst index f19d75ceff..6096242d99 100644 --- a/docs/devel/fuzzing.rst +++ b/docs/devel/fuzzing.rst @@ -34,16 +34,16 @@ such as out-of-bounds accesses, use-after-frees, double-frees etc.:: Fuzz targets are built similarly to system targets:: - make i386-softmmu/fuzz + make qemu-fuzz-i386 -This builds ``./i386-softmmu/qemu-fuzz-i386`` +This builds ``./qemu-fuzz-i386`` The first option to this command is: ``--fuzz-target=FUZZ_NAME`` To list all of the available fuzzers run ``qemu-fuzz-i386`` with no arguments. For example:: - ./i386-softmmu/qemu-fuzz-i386 --fuzz-target=virtio-scsi-fuzz + ./qemu-fuzz-i386 --fuzz-target=virtio-scsi-fuzz Internally, libfuzzer parses all arguments that do not begin with ``"--"``. Information about these is available by passing ``-help=1`` From bb451d248719aaa6c32524e418444a1b8159b7dd Mon Sep 17 00:00:00 2001 From: Alexander Bulekov Date: Sun, 8 Nov 2020 12:11:36 -0500 Subject: [PATCH 10/19] scripts/oss-fuzz: give all fuzzers -target names We switched to hardlinks in a942f64cc4 ("scripts/oss-fuzz: use hardlinks instead of copying") The motivation was to conserve space (50 fuzzers built with ASAN, can weigh close to 9 GB). Unfortunately, OSS-Fuzz (partially) treated the underlying copy of the fuzzer as a standalone fuzzer. To attempt to fix, we tried: f8b8f37463 ("scripts/oss-fuzz: rename bin/qemu-fuzz-i386") This was also not a complete fix, because though OSS-Fuzz ignores the renamed fuzzer, the underlying ClusterFuzz, doesn't: https://storage.googleapis.com/clusterfuzz-builds/qemu/targets.list.address https://oss-fuzz-build-logs.storage.googleapis.com/log-9bfb55f9-1c20-4aa6-a49c-ede12864eeb2.txt (clusterfuzz still lists qemu-fuzz-i386.base as a fuzzer) This change keeps the hard-links, but makes them all point to a file with a qemu-fuzz-i386-target-.. name. If we have targets, A, B, C, the result will be: qemu-fuzz-i386-target-A (base file) qemu-fuzz-i386-target-B -> qemu-fuzz-i386-target-A qemu-fuzz-i386-target-C -> qemu-fuzz-i386-target-A The result should be that every file that looks like a fuzzer to OSS-Fuzz/ClusterFuzz, can run as a fuzzer (we don't have a separate base copy). Unfortunately, there is not simple way to test this locally. In the future, it might be worth it to link the majority of QEMU in as a shared-object (see https://github.com/google/oss-fuzz/issues/4575 ) Signed-off-by: Alexander Bulekov Message-Id: <20201108171136.160607-1-alxndr@bu.edu> Signed-off-by: Thomas Huth --- scripts/oss-fuzz/build.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/oss-fuzz/build.sh b/scripts/oss-fuzz/build.sh index 3b1c82b63d..c1af43fded 100755 --- a/scripts/oss-fuzz/build.sh +++ b/scripts/oss-fuzz/build.sh @@ -62,9 +62,6 @@ fi mkdir -p "$DEST_DIR/lib/" # Copy the shared libraries here -mkdir -p "$DEST_DIR/bin/" # Copy executables that shouldn't - # be treated as fuzzers by oss-fuzz here - # Build once to get the list of dynamic lib paths, and copy them over ../configure --disable-werror --cc="$CC" --cxx="$CXX" --enable-fuzzing \ --prefix="$DEST_DIR" --bindir="$DEST_DIR" --datadir="$DEST_DIR/data/" \ @@ -91,20 +88,23 @@ make "-j$(nproc)" qemu-fuzz-i386 V=1 # Copy over the datadir cp -r ../pc-bios/ "$DEST_DIR/pc-bios" -cp "./qemu-fuzz-i386" "$DEST_DIR/bin/qemu-fuzz-i386.base" +targets=$(./qemu-fuzz-i386 | awk '$1 ~ /\*/ {print $2}') +base_copy="$DEST_DIR/qemu-fuzz-i386-target-$(echo "$targets" | head -n 1)" + +cp "./qemu-fuzz-i386" "$base_copy" # Run the fuzzer with no arguments, to print the help-string and get the list # of available fuzz-targets. Copy over the qemu-fuzz-i386, naming it according # to each available fuzz target (See 05509c8e6d fuzz: select fuzz target using # executable name) -for target in $(./qemu-fuzz-i386 | awk '$1 ~ /\*/ {print $2}'); +for target in $(echo "$targets" | tail -n +2); do # Ignore the generic-fuzz target, as it requires some environment variables # to be configured. We have some generic-fuzz-{pc-q35, floppy, ...} targets # that are thin wrappers around this target that set the required # environment variables according to predefined configs. if [ "$target" != "generic-fuzz" ]; then - ln "$DEST_DIR/bin/qemu-fuzz-i386.base" \ + ln $base_copy \ "$DEST_DIR/qemu-fuzz-i386-target-$target" fi done From aba378dee666fe2aa07f3d318fdf904f454389af Mon Sep 17 00:00:00 2001 From: Daniele Buono Date: Thu, 5 Nov 2020 17:18:57 -0500 Subject: [PATCH 11/19] fuzz: Make fork_fuzz.ld compatible with LLVM's LLD LLVM's linker, LLD, supports the keyword "INSERT AFTER", starting with version 11. However, when multiple sections are defined in the same "INSERT AFTER", they are added in a reversed order, compared to BFD's LD. This patch makes fork_fuzz.ld generic enough to work with both linkers. Each section now has its own "INSERT AFTER" keyword, so proper ordering is defined between the sections added. Signed-off-by: Daniele Buono Message-Id: <20201105221905.1350-2-dbuono@linux.vnet.ibm.com> Reviewed-by: Alexander Bulekov Tested-by: Alexander Bulekov Signed-off-by: Thomas Huth --- tests/qtest/fuzz/fork_fuzz.ld | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/qtest/fuzz/fork_fuzz.ld b/tests/qtest/fuzz/fork_fuzz.ld index bfb667ed06..cfb88b7fdb 100644 --- a/tests/qtest/fuzz/fork_fuzz.ld +++ b/tests/qtest/fuzz/fork_fuzz.ld @@ -16,6 +16,11 @@ SECTIONS /* Lowest stack counter */ *(__sancov_lowest_stack); } +} +INSERT AFTER .data; + +SECTIONS +{ .data.fuzz_ordered : { /* @@ -34,6 +39,11 @@ SECTIONS */ *(.bss._ZN6fuzzer3TPCE); } +} +INSERT AFTER .data.fuzz_start; + +SECTIONS +{ .data.fuzz_end : ALIGN(4K) { __FUZZ_COUNTERS_END = .; @@ -43,4 +53,4 @@ SECTIONS * Don't overwrite the SECTIONS in the default linker script. Instead insert the * above into the default script */ -INSERT AFTER .data; +INSERT AFTER .data.fuzz_ordered; From 2deca810d8c2b8d0c56782ef8d9f4bfbfcacd261 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 29 Oct 2020 20:14:49 +0000 Subject: [PATCH 12/19] configure: surface deprecated targets in the help output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Show the targets but keep them separate from the main list. Signed-off-by: Alex Bennée Message-Id: <20201029201449.6926-1-alex.bennee@linaro.org> Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé Reviewed-by: Thomas Huth Signed-off-by: Thomas Huth --- configure | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configure b/configure index 805f779150..4cef321d9d 100755 --- a/configure +++ b/configure @@ -1655,9 +1655,11 @@ Standard options: --prefix=PREFIX install in PREFIX [$prefix] --interp-prefix=PREFIX where to find shared libraries, etc. use %M for cpu name [$interp_prefix] - --target-list=LIST set target list (default: build everything) + --target-list=LIST set target list (default: build all non-deprecated) $(echo Available targets: $default_target_list | \ fold -s -w 53 | sed -e 's/^/ /') +$(echo Deprecated targets: $deprecated_targets_list | \ + fold -s -w 53 | sed -e 's/^/ /') --target-list-exclude=LIST exclude a set of targets from the default target-list Advanced options (experts only): From 4daa9055beffa17ed47a71d52e7af219acc38e29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Mon, 2 Nov 2020 13:09:23 +0000 Subject: [PATCH 13/19] gitlab: publish the docs built during CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Most of the build jobs will create the sphinx documentation. If we expose this as an artifact of a "pages" job in a "public" directory, it will get published using GitLab Pages. This means a user can push a branch with docs changes to GitLab and view the results at https://yourusername.gitlab.io/qemu/ Signed-off-by: Daniel P. Berrangé Message-Id: <20201102130926.161183-2-berrange@redhat.com> Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé Signed-off-by: Thomas Huth --- .gitlab-ci.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5763318d37..5993b64f22 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -417,3 +417,17 @@ check-dco: - $CI_PROJECT_NAMESPACE == 'qemu-project' && $CI_COMMIT_BRANCH == 'master' variables: GIT_DEPTH: 1000 + +pages: + image: $CI_REGISTRY_IMAGE/qemu/ubuntu2004:latest + stage: test + needs: + - job: build-system-ubuntu + artifacts: true + script: + - mkdir public + - mv build/docs/index.html public/ + - for i in devel interop specs system tools user ; do mv build/docs/$i public/ ; done + artifacts: + paths: + - public From d0f26e68a0545db5010e8fac7386739a2c7213b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Mon, 2 Nov 2020 13:09:24 +0000 Subject: [PATCH 14/19] gitlab: force enable docs build in Fedora, Ubuntu, Debian MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Meson runs a test to see if Sphinx works, and automatically disables it on error. This can lead to the CI jobs skipping docs build without maintainers noticing the problem. Use --enable-docs to force a fatal error if Sphinx doesn't work on the jobs where we expect it to be OK. Signed-off-by: Daniel P. Berrangé Message-Id: <20201102130926.161183-3-berrange@redhat.com> Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé Signed-off-by: Thomas Huth --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5993b64f22..9a8b375188 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -74,6 +74,7 @@ build-system-ubuntu: TARGETS: aarch64-softmmu alpha-softmmu cris-softmmu hppa-softmmu moxie-softmmu microblazeel-softmmu mips64el-softmmu MAKE_CHECK_ARGS: check-build + CONFIGURE_ARGS: --enable-docs artifacts: expire_in: 2 days paths: @@ -105,6 +106,7 @@ build-system-debian: TARGETS: arm-softmmu avr-softmmu i386-softmmu mipsel-softmmu riscv64-softmmu sh4eb-softmmu sparc-softmmu xtensaeb-softmmu MAKE_CHECK_ARGS: check-build + CONFIGURE_ARGS: --enable-docs artifacts: expire_in: 2 days paths: @@ -133,7 +135,7 @@ build-system-fedora: <<: *native_build_job_definition variables: IMAGE: fedora - CONFIGURE_ARGS: --disable-gcrypt --enable-nettle + CONFIGURE_ARGS: --disable-gcrypt --enable-nettle --enable-docs TARGETS: tricore-softmmu microblaze-softmmu mips-softmmu xtensa-softmmu m68k-softmmu riscv32-softmmu ppc-softmmu sparc64-softmmu MAKE_CHECK_ARGS: check-build From 704a256da83d6535fce879ba4137299bbe626e5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Mon, 2 Nov 2020 13:09:26 +0000 Subject: [PATCH 15/19] docs: add "page source" link to sphinx documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a link to the top of the sidebar in every docs page that takes the user back to the source code in gitlab. Signed-off-by: Daniel P. Berrangé Message-Id: <20201102130926.161183-5-berrange@redhat.com> Signed-off-by: Thomas Huth --- docs/_templates/editpage.html | 5 +++++ docs/conf.py | 1 + docs/devel/_templates/editpage.html | 5 +++++ docs/interop/_templates/editpage.html | 5 +++++ docs/specs/_templates/editpage.html | 5 +++++ docs/system/_templates/editpage.html | 5 +++++ docs/tools/_templates/editpage.html | 5 +++++ docs/user/_templates/editpage.html | 5 +++++ 8 files changed, 36 insertions(+) create mode 100644 docs/_templates/editpage.html create mode 100644 docs/devel/_templates/editpage.html create mode 100644 docs/interop/_templates/editpage.html create mode 100644 docs/specs/_templates/editpage.html create mode 100644 docs/system/_templates/editpage.html create mode 100644 docs/tools/_templates/editpage.html create mode 100644 docs/user/_templates/editpage.html diff --git a/docs/_templates/editpage.html b/docs/_templates/editpage.html new file mode 100644 index 0000000000..4319b0f5ac --- /dev/null +++ b/docs/_templates/editpage.html @@ -0,0 +1,5 @@ +
+ +
diff --git a/docs/conf.py b/docs/conf.py index e584f68393..d40d8ff37b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -177,6 +177,7 @@ html_theme_options = { html_sidebars = { '**': [ 'about.html', + 'editpage.html', 'navigation.html', 'searchbox.html', ] diff --git a/docs/devel/_templates/editpage.html b/docs/devel/_templates/editpage.html new file mode 100644 index 0000000000..a86d22bca8 --- /dev/null +++ b/docs/devel/_templates/editpage.html @@ -0,0 +1,5 @@ +
+ +
diff --git a/docs/interop/_templates/editpage.html b/docs/interop/_templates/editpage.html new file mode 100644 index 0000000000..215e562681 --- /dev/null +++ b/docs/interop/_templates/editpage.html @@ -0,0 +1,5 @@ +
+ +
diff --git a/docs/specs/_templates/editpage.html b/docs/specs/_templates/editpage.html new file mode 100644 index 0000000000..aaa468aa98 --- /dev/null +++ b/docs/specs/_templates/editpage.html @@ -0,0 +1,5 @@ +
+ +
diff --git a/docs/system/_templates/editpage.html b/docs/system/_templates/editpage.html new file mode 100644 index 0000000000..6586b2e257 --- /dev/null +++ b/docs/system/_templates/editpage.html @@ -0,0 +1,5 @@ +
+ +
diff --git a/docs/tools/_templates/editpage.html b/docs/tools/_templates/editpage.html new file mode 100644 index 0000000000..2a9c8fc92b --- /dev/null +++ b/docs/tools/_templates/editpage.html @@ -0,0 +1,5 @@ +
+ +
diff --git a/docs/user/_templates/editpage.html b/docs/user/_templates/editpage.html new file mode 100644 index 0000000000..1f5ee01e60 --- /dev/null +++ b/docs/user/_templates/editpage.html @@ -0,0 +1,5 @@ +
+ +
From d4e279141bf59e702beae3a1002b482f733a2ac2 Mon Sep 17 00:00:00 2001 From: Dima Stepanov Date: Mon, 9 Nov 2020 14:25:50 +0300 Subject: [PATCH 16/19] fuzz: add virtio-blk fuzz target The virtio-blk fuzz target sets up and fuzzes the available virtio-blk queues. The implementation is based on two files: - tests/qtest/fuzz/virtio_scsi_fuzz.c - tests/qtest/virtio_blk_test.c Signed-off-by: Dima Stepanov Reviewed-by: Alexander Bulekov Message-Id: Signed-off-by: Thomas Huth --- tests/qtest/fuzz/meson.build | 1 + tests/qtest/fuzz/virtio_blk_fuzz.c | 234 +++++++++++++++++++++++++++++ 2 files changed, 235 insertions(+) create mode 100644 tests/qtest/fuzz/virtio_blk_fuzz.c diff --git a/tests/qtest/fuzz/meson.build b/tests/qtest/fuzz/meson.build index 5162321f30..8af6848cd5 100644 --- a/tests/qtest/fuzz/meson.build +++ b/tests/qtest/fuzz/meson.build @@ -5,6 +5,7 @@ specific_fuzz_ss.add(files('fuzz.c', 'fork_fuzz.c', 'qos_fuzz.c', specific_fuzz_ss.add(when: 'CONFIG_I440FX', if_true: files('i440fx_fuzz.c')) specific_fuzz_ss.add(when: 'CONFIG_VIRTIO_NET', if_true: files('virtio_net_fuzz.c')) specific_fuzz_ss.add(when: 'CONFIG_VIRTIO_SCSI', if_true: files('virtio_scsi_fuzz.c')) +specific_fuzz_ss.add(when: 'CONFIG_VIRTIO_BLK', if_true: files('virtio_blk_fuzz.c')) specific_fuzz_ss.add(files('generic_fuzz.c')) fork_fuzz = declare_dependency( diff --git a/tests/qtest/fuzz/virtio_blk_fuzz.c b/tests/qtest/fuzz/virtio_blk_fuzz.c new file mode 100644 index 0000000000..623a756fd4 --- /dev/null +++ b/tests/qtest/fuzz/virtio_blk_fuzz.c @@ -0,0 +1,234 @@ +/* + * virtio-blk Fuzzing Target + * + * Copyright Red Hat Inc., 2020 + * + * Based on virtio-scsi-fuzz target. + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" + +#include "tests/qtest/libqos/libqtest.h" +#include "tests/qtest/libqos/virtio-blk.h" +#include "tests/qtest/libqos/virtio.h" +#include "tests/qtest/libqos/virtio-pci.h" +#include "standard-headers/linux/virtio_ids.h" +#include "standard-headers/linux/virtio_pci.h" +#include "standard-headers/linux/virtio_blk.h" +#include "fuzz.h" +#include "fork_fuzz.h" +#include "qos_fuzz.h" + +#define TEST_IMAGE_SIZE (64 * 1024 * 1024) +#define PCI_SLOT 0x02 +#define PCI_FN 0x00 + +#define MAX_NUM_QUEUES 64 + +/* Based on tests/qtest/virtio-blk-test.c. */ +typedef struct { + int num_queues; + QVirtQueue *vq[MAX_NUM_QUEUES + 2]; +} QVirtioBlkQueues; + +static QVirtioBlkQueues *qvirtio_blk_init(QVirtioDevice *dev, uint64_t mask) +{ + QVirtioBlkQueues *vs; + uint64_t features; + + vs = g_new0(QVirtioBlkQueues, 1); + + features = qvirtio_get_features(dev); + if (!mask) { + mask = ~((1u << VIRTIO_RING_F_INDIRECT_DESC) | + (1u << VIRTIO_RING_F_EVENT_IDX) | + (1u << VIRTIO_BLK_F_SCSI)); + } + mask |= ~QVIRTIO_F_BAD_FEATURE; + features &= mask; + qvirtio_set_features(dev, features); + + vs->num_queues = 1; + vs->vq[0] = qvirtqueue_setup(dev, fuzz_qos_alloc, 0); + + qvirtio_set_driver_ok(dev); + + return vs; +} + +static void virtio_blk_fuzz(QTestState *s, QVirtioBlkQueues* queues, + const unsigned char *Data, size_t Size) +{ + /* + * Data is a sequence of random bytes. We split them up into "actions", + * followed by data: + * [vqa][dddddddd][vqa][dddd][vqa][dddddddddddd] ... + * The length of the data is specified by the preceding vqa.length + */ + typedef struct vq_action { + uint8_t queue; + uint8_t length; + uint8_t write; + uint8_t next; + uint8_t kick; + } vq_action; + + /* Keep track of the free head for each queue we interact with */ + bool vq_touched[MAX_NUM_QUEUES + 2] = {0}; + uint32_t free_head[MAX_NUM_QUEUES + 2]; + + QGuestAllocator *t_alloc = fuzz_qos_alloc; + + QVirtioBlk *blk = fuzz_qos_obj; + QVirtioDevice *dev = blk->vdev; + QVirtQueue *q; + vq_action vqa; + while (Size >= sizeof(vqa)) { + /* Copy the action, so we can normalize length, queue and flags */ + memcpy(&vqa, Data, sizeof(vqa)); + + Data += sizeof(vqa); + Size -= sizeof(vqa); + + vqa.queue = vqa.queue % queues->num_queues; + /* Cap length at the number of remaining bytes in data */ + vqa.length = vqa.length >= Size ? Size : vqa.length; + vqa.write = vqa.write & 1; + vqa.next = vqa.next & 1; + vqa.kick = vqa.kick & 1; + + q = queues->vq[vqa.queue]; + + /* Copy the data into ram, and place it on the virtqueue */ + uint64_t req_addr = guest_alloc(t_alloc, vqa.length); + qtest_memwrite(s, req_addr, Data, vqa.length); + if (vq_touched[vqa.queue] == 0) { + vq_touched[vqa.queue] = 1; + free_head[vqa.queue] = qvirtqueue_add(s, q, req_addr, vqa.length, + vqa.write, vqa.next); + } else { + qvirtqueue_add(s, q, req_addr, vqa.length, vqa.write , vqa.next); + } + + if (vqa.kick) { + qvirtqueue_kick(s, dev, q, free_head[vqa.queue]); + free_head[vqa.queue] = 0; + } + Data += vqa.length; + Size -= vqa.length; + } + /* In the end, kick each queue we interacted with */ + for (int i = 0; i < MAX_NUM_QUEUES + 2; i++) { + if (vq_touched[i]) { + qvirtqueue_kick(s, dev, queues->vq[i], free_head[i]); + } + } +} + +static void virtio_blk_fork_fuzz(QTestState *s, + const unsigned char *Data, size_t Size) +{ + QVirtioBlk *blk = fuzz_qos_obj; + static QVirtioBlkQueues *queues; + if (!queues) { + queues = qvirtio_blk_init(blk->vdev, 0); + } + if (fork() == 0) { + virtio_blk_fuzz(s, queues, Data, Size); + flush_events(s); + _Exit(0); + } else { + flush_events(s); + wait(NULL); + } +} + +static void virtio_blk_with_flag_fuzz(QTestState *s, + const unsigned char *Data, size_t Size) +{ + QVirtioBlk *blk = fuzz_qos_obj; + static QVirtioBlkQueues *queues; + + if (fork() == 0) { + if (Size >= sizeof(uint64_t)) { + queues = qvirtio_blk_init(blk->vdev, *(uint64_t *)Data); + virtio_blk_fuzz(s, queues, + Data + sizeof(uint64_t), Size - sizeof(uint64_t)); + flush_events(s); + } + _Exit(0); + } else { + flush_events(s); + wait(NULL); + } +} + +static void virtio_blk_pre_fuzz(QTestState *s) +{ + qos_init_path(s); + counter_shm_init(); +} + +static void drive_destroy(void *path) +{ + unlink(path); + g_free(path); +} + +static char *drive_create(void) +{ + int fd, ret; + char *t_path = g_strdup("/tmp/qtest.XXXXXX"); + + /* Create a temporary raw image */ + fd = mkstemp(t_path); + g_assert_cmpint(fd, >=, 0); + ret = ftruncate(fd, TEST_IMAGE_SIZE); + g_assert_cmpint(ret, ==, 0); + close(fd); + + g_test_queue_destroy(drive_destroy, t_path); + return t_path; +} + +static void *virtio_blk_test_setup(GString *cmd_line, void *arg) +{ + char *tmp_path = drive_create(); + + g_string_append_printf(cmd_line, + " -drive if=none,id=drive0,file=%s," + "format=raw,auto-read-only=off ", + tmp_path); + + return arg; +} + +static void register_virtio_blk_fuzz_targets(void) +{ + fuzz_add_qos_target(&(FuzzTarget){ + .name = "virtio-blk-fuzz", + .description = "Fuzz the virtio-blk virtual queues, forking " + "for each fuzz run", + .pre_vm_init = &counter_shm_init, + .pre_fuzz = &virtio_blk_pre_fuzz, + .fuzz = virtio_blk_fork_fuzz,}, + "virtio-blk", + &(QOSGraphTestOptions){.before = virtio_blk_test_setup} + ); + + fuzz_add_qos_target(&(FuzzTarget){ + .name = "virtio-blk-flags-fuzz", + .description = "Fuzz the virtio-blk virtual queues, forking " + "for each fuzz run (also fuzzes the virtio flags)", + .pre_vm_init = &counter_shm_init, + .pre_fuzz = &virtio_blk_pre_fuzz, + .fuzz = virtio_blk_with_flag_fuzz,}, + "virtio-blk", + &(QOSGraphTestOptions){.before = virtio_blk_test_setup} + ); +} + +fuzz_target_init(register_virtio_blk_fuzz_targets); From ad57e2b1f53f507392807e6e2b36c34454b270fa Mon Sep 17 00:00:00 2001 From: Peter Maydell Date: Mon, 9 Nov 2020 16:26:21 +0000 Subject: [PATCH 17/19] qtest: Update references to parse_escape() in comments In commit 61030280ca2d67bd in 2018 we renamed the parse_escape() function to parse_interpolation(), but we didn't catch the references to this function in doc comments in libqtest.h. Update them. Signed-off-by: Peter Maydell Message-Id: <20201109162621.18885-1-peter.maydell@linaro.org> Reviewed-by: Thomas Huth Signed-off-by: Thomas Huth --- tests/qtest/libqos/libqtest.h | 18 +++++++++--------- tests/qtest/libqtest-single.h | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/qtest/libqos/libqtest.h b/tests/qtest/libqos/libqtest.h index 5c959f1853..724f65aa94 100644 --- a/tests/qtest/libqos/libqtest.h +++ b/tests/qtest/libqos/libqtest.h @@ -88,7 +88,7 @@ void qtest_quit(QTestState *s); * @fds: array of file descriptors * @fds_num: number of elements in @fds * @fmt: QMP message to send to qemu, formatted like - * qobject_from_jsonf_nofail(). See parse_escape() for what's + * qobject_from_jsonf_nofail(). See parse_interpolation() for what's * supported after '%'. * * Sends a QMP message to QEMU with fds and returns the response. @@ -101,7 +101,7 @@ QDict *qtest_qmp_fds(QTestState *s, int *fds, size_t fds_num, * qtest_qmp: * @s: #QTestState instance to operate on. * @fmt: QMP message to send to qemu, formatted like - * qobject_from_jsonf_nofail(). See parse_escape() for what's + * qobject_from_jsonf_nofail(). See parse_interpolation() for what's * supported after '%'. * * Sends a QMP message to QEMU and returns the response. @@ -113,7 +113,7 @@ QDict *qtest_qmp(QTestState *s, const char *fmt, ...) * qtest_qmp_send: * @s: #QTestState instance to operate on. * @fmt: QMP message to send to qemu, formatted like - * qobject_from_jsonf_nofail(). See parse_escape() for what's + * qobject_from_jsonf_nofail(). See parse_interpolation() for what's * supported after '%'. * * Sends a QMP message to QEMU and leaves the response in the stream. @@ -138,7 +138,7 @@ void qtest_qmp_send_raw(QTestState *s, const char *fmt, ...) * @fds: array of file descriptors * @fds_num: number of elements in @fds * @fmt: QMP message to send to QEMU, formatted like - * qobject_from_jsonf_nofail(). See parse_escape() for what's + * qobject_from_jsonf_nofail(). See parse_interpolation() for what's * supported after '%'. * @ap: QMP message arguments * @@ -152,7 +152,7 @@ QDict *qtest_vqmp_fds(QTestState *s, int *fds, size_t fds_num, * qtest_vqmp: * @s: #QTestState instance to operate on. * @fmt: QMP message to send to QEMU, formatted like - * qobject_from_jsonf_nofail(). See parse_escape() for what's + * qobject_from_jsonf_nofail(). See parse_interpolation() for what's * supported after '%'. * @ap: QMP message arguments * @@ -167,7 +167,7 @@ QDict *qtest_vqmp(QTestState *s, const char *fmt, va_list ap) * @fds: array of file descriptors * @fds_num: number of elements in @fds * @fmt: QMP message to send to QEMU, formatted like - * qobject_from_jsonf_nofail(). See parse_escape() for what's + * qobject_from_jsonf_nofail(). See parse_interpolation() for what's * supported after '%'. * @ap: QMP message arguments * @@ -181,7 +181,7 @@ void qtest_qmp_vsend_fds(QTestState *s, int *fds, size_t fds_num, * qtest_qmp_vsend: * @s: #QTestState instance to operate on. * @fmt: QMP message to send to QEMU, formatted like - * qobject_from_jsonf_nofail(). See parse_escape() for what's + * qobject_from_jsonf_nofail(). See parse_interpolation() for what's * supported after '%'. * @ap: QMP message arguments * @@ -636,7 +636,7 @@ void qtest_add_abrt_handler(GHookFunc fn, const void *data); * qtest_qmp_assert_success: * @qts: QTestState instance to operate on * @fmt: QMP message to send to qemu, formatted like - * qobject_from_jsonf_nofail(). See parse_escape() for what's + * qobject_from_jsonf_nofail(). See parse_interpolation() for what's * supported after '%'. * * Sends a QMP message to QEMU and asserts that a 'return' key is present in @@ -683,7 +683,7 @@ void qtest_qmp_device_add_qdict(QTestState *qts, const char *drv, * @driver: Name of the device that should be added * @id: Identification string * @fmt: QMP message to send to qemu, formatted like - * qobject_from_jsonf_nofail(). See parse_escape() for what's + * qobject_from_jsonf_nofail(). See parse_interpolation() for what's * supported after '%'. * * Generic hot-plugging test via the device_add QMP command. diff --git a/tests/qtest/libqtest-single.h b/tests/qtest/libqtest-single.h index 176979a2ce..0d7f568678 100644 --- a/tests/qtest/libqtest-single.h +++ b/tests/qtest/libqtest-single.h @@ -47,7 +47,7 @@ static inline void qtest_end(void) /** * qmp: * @fmt...: QMP message to send to qemu, formatted like - * qobject_from_jsonf_nofail(). See parse_escape() for what's + * qobject_from_jsonf_nofail(). See parse_interpolation() for what's * supported after '%'. * * Sends a QMP message to QEMU and returns the response. From 074df27f744f0a72f8b33b2fd5a6cdc557f48f7b Mon Sep 17 00:00:00 2001 From: Daniele Buono Date: Thu, 5 Nov 2020 17:18:58 -0500 Subject: [PATCH 18/19] s390x: fix clang 11 warnings in cpu_models.c There are void * pointers that get casted to enums, in cpu_models.c Such casts can result in a small integer type and are caught as warnings with clang, starting with version 11: Clang 11 finds a bunch of spots in the code that trigger this new warnings: ../qemu-base/target/s390x/cpu_models.c:985:21: error: cast to smaller integer type 'S390Feat' from 'void *' [-Werror,-Wvoid-pointer-to-enum-cast] S390Feat feat = (S390Feat) opaque; ^~~~~~~~~~~~~~~~~ ../qemu-base/target/s390x/cpu_models.c:1002:21: error: cast to smaller integer type 'S390Feat' from 'void *' [-Werror,-Wvoid-pointer-to-enum-cast] S390Feat feat = (S390Feat) opaque; ^~~~~~~~~~~~~~~~~ ../qemu-base/target/s390x/cpu_models.c:1036:27: error: cast to smaller integer type 'S390FeatGroup' from 'void *' [-Werror,-Wvoid-pointer-to-enum-cast] S390FeatGroup group = (S390FeatGroup) opaque; ^~~~~~~~~~~~~~~~~~~~~~ ../qemu-base/target/s390x/cpu_models.c:1057:27: error: cast to smaller integer type 'S390FeatGroup' from 'void *' [-Werror,-Wvoid-pointer-to-enum-cast] S390FeatGroup group = (S390FeatGroup) opaque; ^~~~~~~~~~~~~~~~~~~~~~ 4 errors generated. Avoid this warning by casting the pointer to uintptr_t first. Signed-off-by: Daniele Buono Message-Id: <20201105221905.1350-3-dbuono@linux.vnet.ibm.com> Acked-by: Cornelia Huck Signed-off-by: Thomas Huth --- target/s390x/cpu_models.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/target/s390x/cpu_models.c b/target/s390x/cpu_models.c index 461e0b8f4a..b5abff8bef 100644 --- a/target/s390x/cpu_models.c +++ b/target/s390x/cpu_models.c @@ -986,7 +986,7 @@ void s390_realize_cpu_model(CPUState *cs, Error **errp) static void get_feature(Object *obj, Visitor *v, const char *name, void *opaque, Error **errp) { - S390Feat feat = (S390Feat) opaque; + S390Feat feat = (S390Feat) (uintptr_t) opaque; S390CPU *cpu = S390_CPU(obj); bool value; @@ -1003,7 +1003,7 @@ static void get_feature(Object *obj, Visitor *v, const char *name, static void set_feature(Object *obj, Visitor *v, const char *name, void *opaque, Error **errp) { - S390Feat feat = (S390Feat) opaque; + S390Feat feat = (S390Feat) (uintptr_t) opaque; DeviceState *dev = DEVICE(obj); S390CPU *cpu = S390_CPU(obj); bool value; @@ -1037,7 +1037,7 @@ static void set_feature(Object *obj, Visitor *v, const char *name, static void get_feature_group(Object *obj, Visitor *v, const char *name, void *opaque, Error **errp) { - S390FeatGroup group = (S390FeatGroup) opaque; + S390FeatGroup group = (S390FeatGroup) (uintptr_t) opaque; const S390FeatGroupDef *def = s390_feat_group_def(group); S390CPU *cpu = S390_CPU(obj); S390FeatBitmap tmp; @@ -1058,7 +1058,7 @@ static void get_feature_group(Object *obj, Visitor *v, const char *name, static void set_feature_group(Object *obj, Visitor *v, const char *name, void *opaque, Error **errp) { - S390FeatGroup group = (S390FeatGroup) opaque; + S390FeatGroup group = (S390FeatGroup) (uintptr_t) opaque; const S390FeatGroupDef *def = s390_feat_group_def(group); DeviceState *dev = DEVICE(obj); S390CPU *cpu = S390_CPU(obj); From a58cabd0e355fc51f18db359ba260da268df26ef Mon Sep 17 00:00:00 2001 From: Daniele Buono Date: Thu, 5 Nov 2020 17:19:00 -0500 Subject: [PATCH 19/19] s390x: Avoid variable size warning in ipl.h S390IPLState contains two IplParameterBlock, which may in turn have either a IPLBlockPV or a IplBlockFcp, both ending with a variable sized field (an array). This causes a warning with clang 11 or greater, which checks that variable sized type are only allocated at the end of the struct: In file included from ../qemu-cfi-v3/target/s390x/diag.c:21: ../qemu-cfi-v3/hw/s390x/ipl.h:161:23: error: field 'iplb' with variable sized type 'IplParameterBlock' (aka 'union IplParameterBlock') not at the end of a struct or class is a GNU extension [-Werror,-Wgnu-variable-sized-type-not-at-end] IplParameterBlock iplb; ^ ../qemu-cfi-v3/hw/s390x/ipl.h:162:23: error: field 'iplb_pv' with variable sized type 'IplParameterBlock' (aka 'union IplParameterBlock') not at the end of a struct or class is a GNU extension [-Werror,-Wgnu-variable-sized-type-not-at-end] IplParameterBlock iplb_pv; In this case, however, the warning is a false positive, because IPLBlockPV and IplBlockFcp are allocated in a union wrapped at 4K, making the union non-variable sized. Fix the warning by turning the two variable sized arrays into arrays of size 0. This avoids the compiler error and should produce the same code. Signed-off-by: Daniele Buono Message-Id: <20201105221905.1350-5-dbuono@linux.vnet.ibm.com> Acked-by: Cornelia Huck Signed-off-by: Thomas Huth --- hw/s390x/ipl.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hw/s390x/ipl.h b/hw/s390x/ipl.h index 9e90169695..dfc6dfd89c 100644 --- a/hw/s390x/ipl.h +++ b/hw/s390x/ipl.h @@ -32,7 +32,7 @@ struct IPLBlockPV { uint32_t num_comp; /* 0x74 */ uint64_t pv_header_addr; /* 0x78 */ uint64_t pv_header_len; /* 0x80 */ - struct IPLBlockPVComp components[]; + struct IPLBlockPVComp components[0]; } QEMU_PACKED; typedef struct IPLBlockPV IPLBlockPV; @@ -63,7 +63,7 @@ struct IplBlockFcp { uint64_t br_lba; uint32_t scp_data_len; uint8_t reserved6[260]; - uint8_t scp_data[]; + uint8_t scp_data[0]; } QEMU_PACKED; typedef struct IplBlockFcp IplBlockFcp;