From 4f3ccd4f166c12ae2bbdf21af3a16da8a7f16a78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Sun, 11 Oct 2020 21:49:18 +0200 Subject: [PATCH 01/23] tests/qtest: Replace magic value by NANOSECONDS_PER_SECOND definition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use self-explicit NANOSECONDS_PER_SECOND definition instead of a magic value. Signed-off-by: Philippe Mathieu-Daudé Message-Id: <20201011194918.3219195-5-f4bug@amsat.org> Reviewed-by: Thomas Huth Signed-off-by: Thomas Huth --- tests/qtest/rtc-test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/qtest/rtc-test.c b/tests/qtest/rtc-test.c index c7af34f6b1..402ce2c609 100644 --- a/tests/qtest/rtc-test.c +++ b/tests/qtest/rtc-test.c @@ -292,7 +292,7 @@ static void alarm_time(void) break; } - clock_step(1000000000); + clock_step(NANOSECONDS_PER_SECOND); } g_assert(get_irq(RTC_ISA_IRQ)); From 320c6e78930e588c548f7bfe84f43fe5dfe2e5ad Mon Sep 17 00:00:00 2001 From: Alexander Bulekov Date: Fri, 2 Oct 2020 10:35:24 -0400 Subject: [PATCH 02/23] gitlab-ci.yml: Only run one test-case per fuzzer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With 1000 runs, there is a non-negligible chance that the fuzzer can trigger a crash. With this CI job, we care about catching build/runtime issues in the core fuzzing code. Actual device fuzzing takes place on oss-fuzz. For these purposes, only running one input should be sufficient. Signed-off-by: Alexander Bulekov Suggested-by: Philippe Mathieu-Daudé Message-Id: <20201002143524.56930-1-alxndr@bu.edu> Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Darren Kenny Signed-off-by: Thomas Huth --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a51c89554f..075c15d45c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -303,7 +303,7 @@ build-oss-fuzz: | grep -v slirp); do grep "LLVMFuzzerTestOneInput" ${fuzzer} > /dev/null 2>&1 || continue ; echo Testing ${fuzzer} ... ; - "${fuzzer}" -runs=1000 -seed=1 || exit 1 ; + "${fuzzer}" -runs=1 -seed=1 || exit 1 ; done # Unrelated to fuzzer: run some tests with -fsanitize=address - cd build-oss-fuzz && make check-qtest-i386 check-unit From d44d9b6bc81df6080ed83073ab2711e5a4258a07 Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Fri, 9 Oct 2020 16:55:11 -0400 Subject: [PATCH 03/23] Acceptance tests: bump pycdlib version for easier installation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On with certain versions of "pip", package installations will attempt to create wheels. And, on environments without a "complete" Python installation (as described in the acceptance tests requirements docs), that will fail. pycdlib, starting with version 1.11.0, is now being made available as wheels, so its instalation on those constrained environments is now possible. Buglink: https://bugs.launchpad.net/qemu/+bug/1897783 Reported-by: Philippe Mathieu-Daudé Signed-off-by: Cleber Rosa Message-Id: <20201009205513.751968-2-crosa@redhat.com> Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé Signed-off-by: Thomas Huth --- tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 036691c922..a1c631fa59 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -2,4 +2,4 @@ # in the tests/venv Python virtual environment. For more info, # refer to: https://pip.pypa.io/en/stable/user_guide/#id1 avocado-framework==81.0 -pycdlib==1.9.0 +pycdlib==1.11.0 From 52dff285698a3c9c7d97d8a790439a16f5fdb038 Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Fri, 9 Oct 2020 16:55:12 -0400 Subject: [PATCH 04/23] Acceptance tests: do not show canceled test logs on GitLab CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests resulting in "CANCEL" in Avocado are usually canceled on purpose, and are almost identical to "SKIP". The logs for canceled tests are adding a lot of noise to the logs being shown on GitLab CI, and causing distraction from real failures. As a side note, this "after script" is scheduled for removal once the feature is implemented within Avocado itself. Reference: https://github.com/avocado-framework/avocado/issues/4266 Signed-off-by: Cleber Rosa Message-Id: <20201009205513.751968-3-crosa@redhat.com> Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé Signed-off-by: Thomas Huth --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 075c15d45c..14be45bbfe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -63,7 +63,7 @@ include: fi after_script: - cd build - - python3 -c 'import json; r = json.load(open("tests/results/latest/results.json")); [print(t["logfile"]) for t in r["tests"] if t["status"] not in ("PASS", "SKIP")]' | xargs cat + - python3 -c 'import json; r = json.load(open("tests/results/latest/results.json")); [print(t["logfile"]) for t in r["tests"] if t["status"] not in ("PASS", "SKIP", "CANCEL")]' | xargs cat - du -chs ${CI_PROJECT_DIR}/avocado-cache build-system-ubuntu: From ec4d2feb27c5e46a69e2c2d588e982c9327e325e Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Fri, 9 Oct 2020 16:55:13 -0400 Subject: [PATCH 05/23] Acceptance tests: show test report on GitLab CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avocado will, by default, produce JUnit files. Let's ask GitLab to present those in the web UI. Signed-off-by: Cleber Rosa Message-Id: <20201009205513.751968-4-crosa@redhat.com> Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé Signed-off-by: Thomas Huth --- .gitlab-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 14be45bbfe..29e934fd53 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -53,6 +53,11 @@ include: paths: - ${CI_PROJECT_DIR}/avocado-cache policy: pull-push + artifacts: + paths: + - build/tests/results/latest/results.xml + reports: + junit: build/tests/results/latest/results.xml before_script: - mkdir -p ~/.config/avocado - echo "[datadir.paths]" > ~/.config/avocado/avocado.conf From d8dd10950199ee4a505dc4872171dcb1b4ca6710 Mon Sep 17 00:00:00 2001 From: Li Qiang Date: Mon, 21 Sep 2020 09:06:05 -0700 Subject: [PATCH 06/23] qtest: add fuzz test case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently the device fuzzer finds more and more issues. For every fuzz case, we need not only the fixes but also the corresponding test case. We can analysis the reproducer for every case and find what happened in where and write a beautiful test case. However the raw data of reproducer is not friendly to analysis. It will take a very long time, even far more than the fixes itself. So let's create a new file to hold all of the fuzz test cases and just use the raw data to act as the test case. This way nobody will be afraid of writing a test case for the fuzz reproducer. This patch adds the issue LP#1878263 test case. Signed-off-by: Li Qiang Message-Id: <20200921160605.19329-1-liq3ea@163.com> Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Alexander Bulekov [thuth: Slightly adjusted commit message, removed empty lines] Signed-off-by: Thomas Huth --- tests/qtest/fuzz-test.c | 49 +++++++++++++++++++++++++++++++++++++++++ tests/qtest/meson.build | 1 + 2 files changed, 50 insertions(+) create mode 100644 tests/qtest/fuzz-test.c diff --git a/tests/qtest/fuzz-test.c b/tests/qtest/fuzz-test.c new file mode 100644 index 0000000000..2f38bb1ec2 --- /dev/null +++ b/tests/qtest/fuzz-test.c @@ -0,0 +1,49 @@ +/* + * QTest testcase for fuzz case + * + * Copyright (c) 2020 Li Qiang + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" + +#include "libqos/libqtest.h" + +/* + * This used to trigger the assert in scsi_dma_complete + * https://bugs.launchpad.net/qemu/+bug/1878263 + */ +static void test_lp1878263_megasas_zero_iov_cnt(void) +{ + QTestState *s; + + s = qtest_init("-nographic -monitor none -serial none " + "-M q35 -device megasas -device scsi-cd,drive=null0 " + "-blockdev driver=null-co,read-zeroes=on,node-name=null0"); + qtest_outl(s, 0xcf8, 0x80001818); + qtest_outl(s, 0xcfc, 0xc101); + qtest_outl(s, 0xcf8, 0x8000181c); + qtest_outl(s, 0xcf8, 0x80001804); + qtest_outw(s, 0xcfc, 0x7); + qtest_outl(s, 0xcf8, 0x8000186a); + qtest_writeb(s, 0x14, 0xfe); + qtest_writeb(s, 0x0, 0x02); + qtest_outb(s, 0xc1c0, 0x17); + qtest_quit(s); +} + +int main(int argc, char **argv) +{ + const char *arch = qtest_get_arch(); + + g_test_init(&argc, &argv, NULL); + + if (strcmp(arch, "i386") == 0 || strcmp(arch, "x86_64") == 0) { + qtest_add_func("fuzz/test_lp1878263_megasas_zero_iov_cnt", + test_lp1878263_megasas_zero_iov_cnt); + } + + return g_test_run(); +} diff --git a/tests/qtest/meson.build b/tests/qtest/meson.build index 0f32ca0895..6c6bfb5e2e 100644 --- a/tests/qtest/meson.build +++ b/tests/qtest/meson.build @@ -54,6 +54,7 @@ qtests_i386 = \ 'bios-tables-test', 'rtc-test', 'i440fx-test', + 'fuzz-test', 'fw_cfg-test', 'device-plug-test', 'drive_del-test', From a4339de2de4def4beb33e22e6f506bcc8b9d9326 Mon Sep 17 00:00:00 2001 From: "Dr. David Alan Gilbert" Date: Thu, 8 Oct 2020 17:03:30 +0100 Subject: [PATCH 07/23] tests/migration: Allow longer timeouts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In travis, with gcov and gprof we're seeing timeouts; hopefully fix this by increasing the test timeouts a bit, but for xbzrle ensure it really does get a couple of cycles through to test the cache. I think the problem in travis is we have about 2 host CPU threads, in the test we have at least 3: a) The vCPU thread (100% flat out) b) The source migration thread c) The destination migration thread if (b) & (c) are slow for any reason - gcov+gperf or a slow host - then they're sharing one host CPU thread so limit the migration bandwidth. Tested on my laptop with: taskset -c 0,1 ./tests/qtest/migration-test -p /x86_64/migration Reported-by: Alex Bennée Signed-off-by: Dr. David Alan Gilbert Message-Id: <20201008160330.130431-1-dgilbert@redhat.com> [thuth: Move the #define to the right location] Signed-off-by: Thomas Huth --- tests/qtest/migration-test.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/qtest/migration-test.c b/tests/qtest/migration-test.c index 00a233cd8c..f410ec5996 100644 --- a/tests/qtest/migration-test.c +++ b/tests/qtest/migration-test.c @@ -34,6 +34,9 @@ unsigned start_address; unsigned end_address; static bool uffd_feature_thread_id; +/* A downtime where the test really should converge */ +#define CONVERGE_DOWNTIME 1000 + #if defined(__linux__) #include #include @@ -864,8 +867,7 @@ static void test_precopy_unix(void) wait_for_migration_pass(from); - /* 300 ms should converge */ - migrate_set_parameter_int(from, "downtime-limit", 300); + migrate_set_parameter_int(from, "downtime-limit", CONVERGE_DOWNTIME); if (!got_stop) { qtest_qmp_eventwait(from, "STOP"); @@ -946,10 +948,12 @@ static void test_xbzrle(const char *uri) migrate_qmp(from, uri, "{}"); + wait_for_migration_pass(from); + /* Make sure we have 2 passes, so the xbzrle cache gets a workout */ wait_for_migration_pass(from); - /* 300ms should converge */ - migrate_set_parameter_int(from, "downtime-limit", 300); + /* 1000ms should converge */ + migrate_set_parameter_int(from, "downtime-limit", 1000); if (!got_stop) { qtest_qmp_eventwait(from, "STOP"); @@ -999,8 +1003,7 @@ static void test_precopy_tcp(void) wait_for_migration_pass(from); - /* 300ms should converge */ - migrate_set_parameter_int(from, "downtime-limit", 300); + migrate_set_parameter_int(from, "downtime-limit", CONVERGE_DOWNTIME); if (!got_stop) { qtest_qmp_eventwait(from, "STOP"); @@ -1068,8 +1071,7 @@ static void test_migrate_fd_proto(void) wait_for_migration_pass(from); - /* 300ms should converge */ - migrate_set_parameter_int(from, "downtime-limit", 300); + migrate_set_parameter_int(from, "downtime-limit", CONVERGE_DOWNTIME); if (!got_stop) { qtest_qmp_eventwait(from, "STOP"); @@ -1304,8 +1306,7 @@ static void test_multifd_tcp(const char *method) wait_for_migration_pass(from); - /* 300ms it should converge */ - migrate_set_parameter_int(from, "downtime-limit", 300); + migrate_set_parameter_int(from, "downtime-limit", CONVERGE_DOWNTIME); if (!got_stop) { qtest_qmp_eventwait(from, "STOP"); From eaf65114f0585a914642c1ce1379bb8339984858 Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Fri, 10 Jul 2020 10:50:20 +0200 Subject: [PATCH 08/23] softmmu/vl: Be less verbose about missing KVM when running the qtests Some of the qtests use "-accel kvm -accel tcg" to run real guest code. This causes some error messages when kvm is not available. We do not really care about these messages since the fallback to tcg is expected here. So let's silence them to avoid that they spoil the output of the tests. Unfortunately, we can not use the qtest_enabled() wrapper in this case, since the qtest accelerator itself is not initialized. Thus we have to test for the qtest_chrdev variable instead. Message-Id: <20200710085020.28222-1-thuth@redhat.com> Reviewed-by: Alexander Bulekov Signed-off-by: Thomas Huth --- softmmu/vl.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/softmmu/vl.c b/softmmu/vl.c index 5a11a62f78..254ee5e525 100644 --- a/softmmu/vl.c +++ b/softmmu/vl.c @@ -165,8 +165,9 @@ bool boot_strict; uint8_t *boot_splash_filedata; int only_migratable; /* turn it off unless user states otherwise */ bool wakeup_suspend_enabled; - int icount_align_option; +static const char *qtest_chrdev; +static const char *qtest_log; /* The bytes in qemu_uuid are in the order specified by RFC4122, _not_ in the * little-endian "wire format" described in the SMBIOS 2.6 specification. @@ -2713,10 +2714,15 @@ static int do_configure_accelerator(void *opaque, QemuOpts *opts, Error **errp) AccelClass *ac = accel_find(acc); AccelState *accel; int ret; + bool qtest_with_kvm; + + qtest_with_kvm = g_str_equal(acc, "kvm") && qtest_chrdev != NULL; if (!ac) { *p_init_failed = true; - error_report("invalid accelerator %s", acc); + if (!qtest_with_kvm) { + error_report("invalid accelerator %s", acc); + } return 0; } accel = ACCEL(object_new_with_class(OBJECT_CLASS(ac))); @@ -2728,8 +2734,9 @@ static int do_configure_accelerator(void *opaque, QemuOpts *opts, Error **errp) ret = accel_init_machine(accel, current_machine); if (ret < 0) { *p_init_failed = true; - error_report("failed to initialize %s: %s", - acc, strerror(-ret)); + if (!qtest_with_kvm || ret != -ENOENT) { + error_report("failed to initialize %s: %s", acc, strerror(-ret)); + } return 0; } @@ -2800,7 +2807,7 @@ static void configure_accelerators(const char *progname) exit(1); } - if (init_failed) { + if (init_failed && !qtest_chrdev) { AccelClass *ac = ACCEL_GET_CLASS(current_accel()); error_report("falling back to %s", ac->name); } @@ -2870,8 +2877,6 @@ void qemu_init(int argc, char **argv, char **envp) MachineClass *machine_class; const char *cpu_option; const char *vga_model = NULL; - const char *qtest_chrdev = NULL; - const char *qtest_log = NULL; const char *incoming = NULL; bool userconfig = true; bool nographic = false; From 07f5903c852de10fa71059f5e7594d203e077b10 Mon Sep 17 00:00:00 2001 From: Ani Sinha Date: Tue, 29 Sep 2020 19:55:01 +0530 Subject: [PATCH 09/23] Add a comment in bios-tables-test.c to clarify the reason behind approach A comment is added in bios-tables-test.c that explains the reasoning behind the process of updating the ACPI table blobs when new tests are added or old tests are modified or code is committed that affect tests. The explanation would help future contributors follow the correct process when making code changes that affect ACPI tables. Signed-off-by: Ani Sinha Acked-by: Igor Mammedov Message-Id: <20200929142501.1057-1-ani@anisinha.ca> Signed-off-by: Thomas Huth --- tests/qtest/bios-tables-test.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/qtest/bios-tables-test.c b/tests/qtest/bios-tables-test.c index e15f36c8c7..5647624492 100644 --- a/tests/qtest/bios-tables-test.c +++ b/tests/qtest/bios-tables-test.c @@ -11,7 +11,7 @@ */ /* - * How to add or update the tests: + * How to add or update the tests or commit changes that affect ACPI tables: * Contributor: * 1. add empty files for new tables, if any, under tests/data/acpi * 2. list any changed files in tests/qtest/bios-tables-test-allowed-diff.h @@ -38,6 +38,11 @@ * $(SRC_PATH)/tests/data/acpi/rebuild-expected-aml.sh * 6. Now commit any changes to the expected binary, include diff from step 4 * in commit log. + * Expected binary updates needs to be a separate patch from the code that + * introduces changes to ACPI tables. It lets the maintainer drop + * and regenerate binary updates in case of merge conflicts. Further, a code + * change is easily reviewable but a binary blob is not (without doing a + * disassembly). * 7. Before sending patches to the list (Contributor) * or before doing a pull request (Maintainer), make sure * tests/qtest/bios-tables-test-allowed-diff.h is empty - this will ensure From 520ddc18bf55fbd7c7409c737b1cd7e8b1e7c971 Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Thu, 1 Oct 2020 06:27:17 +0200 Subject: [PATCH 10/23] MAINTAINERS: Ignore bios-tables-test in the qtest section I'm very often getting CC: on rather large patch series that modify the ACPI stuff of either ARM or x86, just because the bios-table-test is often slightly involved here. I can't say much about ACPI, and the bios-table-test is already covered by the ACPI section in MAINTAINERS, so I'd rather prefer to not getting automatically CC-ed on such patch series anymore. If people want my opinion about qtest-related changes, they can still put me on CC manually. Message-Id: <20201001042717.136033-1-thuth@redhat.com> Acked-by: Laszlo Ersek Signed-off-by: Thomas Huth --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index e9d85cc873..def5ad8ea9 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2495,7 +2495,7 @@ S: Maintained F: softmmu/qtest.c F: accel/qtest/ F: tests/qtest/ -X: tests/qtest/bios-tables-test-allowed-diff.h +X: tests/qtest/bios-tables-test* Device Fuzzing M: Alexander Bulekov From 2ae00c8b2f7536f1bb38c3eb40d76f4196c61d0c Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Sat, 19 Sep 2020 12:27:20 +0200 Subject: [PATCH 11/23] Remove superfluous .gitignore files Since we are now always doing out-of-tree builds, these gitignore files should not be necessary anymore. Message-Id: <20200919133637.72744-1-thuth@redhat.com> Signed-off-by: Thomas Huth --- pc-bios/keymaps/.gitignore | 1 - plugins/.gitignore | 2 -- tests/.gitignore | 32 -------------------------------- tests/fp/.gitignore | 2 -- tests/migration/.gitignore | 2 -- tests/multiboot/.gitignore | 3 --- tests/qemu-iotests/.gitignore | 9 --------- tests/tcg/.gitignore | 5 ----- tests/uefi-test-tools/.gitignore | 3 --- 9 files changed, 59 deletions(-) delete mode 100644 pc-bios/keymaps/.gitignore delete mode 100644 plugins/.gitignore delete mode 100644 tests/.gitignore delete mode 100644 tests/fp/.gitignore delete mode 100644 tests/migration/.gitignore delete mode 100644 tests/multiboot/.gitignore delete mode 100644 tests/qemu-iotests/.gitignore delete mode 100644 tests/tcg/.gitignore delete mode 100644 tests/uefi-test-tools/.gitignore diff --git a/pc-bios/keymaps/.gitignore b/pc-bios/keymaps/.gitignore deleted file mode 100644 index f90738f4dc..0000000000 --- a/pc-bios/keymaps/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/*.stamp diff --git a/plugins/.gitignore b/plugins/.gitignore deleted file mode 100644 index 7b8aaa1f10..0000000000 --- a/plugins/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -qemu-plugins-ld.symbols -qemu-plugins-ld64.symbols diff --git a/tests/.gitignore b/tests/.gitignore deleted file mode 100644 index d03c037d77..0000000000 --- a/tests/.gitignore +++ /dev/null @@ -1,32 +0,0 @@ -atomic_add-bench -benchmark-crypto-cipher -benchmark-crypto-hash -benchmark-crypto-hmac -check-* -!check-*.c -!check-*.sh -fp/*.out -qht-bench -rcutorture -test-* -!test-*.c -!test-*.py -!docker/test-* -test-qapi-commands.[ch] -test-qapi-init-commands.[ch] -include/test-qapi-commands-sub-module.[ch] -test-qapi-commands-sub-sub-module.[ch] -test-qapi-emit-events.[ch] -test-qapi-events.[ch] -include/test-qapi-events-sub-module.[ch] -test-qapi-events-sub-sub-module.[ch] -test-qapi-types.[ch] -include/test-qapi-types-sub-module.[ch] -test-qapi-types-sub-sub-module.[ch] -test-qapi-visit.[ch] -include/test-qapi-visit-sub-module.[ch] -test-qapi-visit-sub-sub-module.[ch] -test-qapi-introspect.[ch] -*-test -qapi-schema/*.test.* -vm/*.img diff --git a/tests/fp/.gitignore b/tests/fp/.gitignore deleted file mode 100644 index 704fd42992..0000000000 --- a/tests/fp/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -fp-test -fp-bench diff --git a/tests/migration/.gitignore b/tests/migration/.gitignore deleted file mode 100644 index 84f37552e4..0000000000 --- a/tests/migration/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -initrd-stress.img -stress diff --git a/tests/multiboot/.gitignore b/tests/multiboot/.gitignore deleted file mode 100644 index 93ef99800b..0000000000 --- a/tests/multiboot/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.bin -*.elf -test.out diff --git a/tests/qemu-iotests/.gitignore b/tests/qemu-iotests/.gitignore deleted file mode 100644 index da62054000..0000000000 --- a/tests/qemu-iotests/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -check.log -check.time* -common.env -*.out.bad -*.notrun -socket_scm_helper - -# ignore everything in the scratch directory -scratch/ diff --git a/tests/tcg/.gitignore b/tests/tcg/.gitignore deleted file mode 100644 index 84d7541b28..0000000000 --- a/tests/tcg/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# These are build artefacts which only appear when you are doing -# builds directly in the source tree. -config-*.mak -*-softmmu/ -*-linux-user/ diff --git a/tests/uefi-test-tools/.gitignore b/tests/uefi-test-tools/.gitignore deleted file mode 100644 index 9f246701de..0000000000 --- a/tests/uefi-test-tools/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -Build -Conf -log From 4258c8e221682652e9b9095c29406f91124d86f7 Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Wed, 23 Sep 2020 16:05:34 +0200 Subject: [PATCH 12/23] docs/system/deprecated: Mark the 'moxie' CPU as deprecated It is currently unclear whether anybody is still using the 'moxie' CPU, and there are no images for testing available this CPU, so the code has likely bit-rotten in the course of time. When I asked the maintainer for information, I did not get a reply within four weeks yet (see https://lists.gnu.org/archive/html/qemu-devel/2020-08/msg07201.html). The last Signed-off-by line from Anthony in our repo is from 2013, so it seems like this code is rather unmaintained. Time to put it onto the deprecation list to see whether somebody is still interested in this code or whether we could remove it in a couple of releases. Message-Id: <20200923171815.97801-1-thuth@redhat.com> Reviewed-by: Richard Henderson Signed-off-by: Thomas Huth --- docs/system/deprecated.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/system/deprecated.rst b/docs/system/deprecated.rst index 5e8346f7bf..09ec8b1ae8 100644 --- a/docs/system/deprecated.rst +++ b/docs/system/deprecated.rst @@ -283,6 +283,14 @@ for VNC should be performed using the pluggable QAuthZ objects. System emulator CPUS -------------------- +``moxie`` CPU (since 5.2.0) +''''''''''''''''''''''''''' + +The ``moxie`` guest CPU support is deprecated and will be removed in +a future version of QEMU. It's unclear whether anybody is still using +CPU emulation in QEMU, and there are no test images available to make +sure that the code is still working. + ``compat`` property of server class POWER CPUs (since 5.0) '''''''''''''''''''''''''''''''''''''''''''''''''''''''''' From 5f8937d63f5b24bbe044c6e00519fa91f86f04bc Mon Sep 17 00:00:00 2001 From: Yonggang Luo Date: Tue, 13 Oct 2020 07:43:44 +0800 Subject: [PATCH 13/23] configure: fixes indent of $meson setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit convert these line from tab to space Signed-off-by: Yonggang Luo Reviewed-by: Daniel P. Berrangé Message-Id: <20201012234348.1427-2-luoyonggang@gmail.com> Signed-off-by: Thomas Huth --- configure | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/configure b/configure index b553288c5e..1b63488521 100755 --- a/configure +++ b/configure @@ -7211,13 +7211,13 @@ NINJA=${ninja:-$PWD/ninjatool} $meson setup \ -Db_pie=$(if test "$pie" = yes; then echo true; else echo false; fi) \ -Db_staticpic=$(if test "$pie" = yes; then echo true; else echo false; fi) \ -Db_coverage=$(if test "$gcov" = yes; then echo true; else echo false; fi) \ - -Dmalloc=$malloc -Dmalloc_trim=$malloc_trim -Dsparse=$sparse \ - -Dkvm=$kvm -Dhax=$hax -Dwhpx=$whpx -Dhvf=$hvf \ - -Dxen=$xen -Dxen_pci_passthrough=$xen_pci_passthrough -Dtcg=$tcg \ - -Dcocoa=$cocoa -Dmpath=$mpath -Dsdl=$sdl -Dsdl_image=$sdl_image \ - -Dvnc=$vnc -Dvnc_sasl=$vnc_sasl -Dvnc_jpeg=$vnc_jpeg -Dvnc_png=$vnc_png \ - -Dgettext=$gettext -Dxkbcommon=$xkbcommon -Du2f=$u2f \ - -Dcapstone=$capstone -Dslirp=$slirp -Dfdt=$fdt \ + -Dmalloc=$malloc -Dmalloc_trim=$malloc_trim -Dsparse=$sparse \ + -Dkvm=$kvm -Dhax=$hax -Dwhpx=$whpx -Dhvf=$hvf \ + -Dxen=$xen -Dxen_pci_passthrough=$xen_pci_passthrough -Dtcg=$tcg \ + -Dcocoa=$cocoa -Dmpath=$mpath -Dsdl=$sdl -Dsdl_image=$sdl_image \ + -Dvnc=$vnc -Dvnc_sasl=$vnc_sasl -Dvnc_jpeg=$vnc_jpeg -Dvnc_png=$vnc_png \ + -Dgettext=$gettext -Dxkbcommon=$xkbcommon -Du2f=$u2f \ + -Dcapstone=$capstone -Dslirp=$slirp -Dfdt=$fdt \ $cross_arg \ "$PWD" "$source_path" From 1f47547256b72fecd9ebf629a8e48cd20282a758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Fri, 18 Sep 2020 14:29:01 +0100 Subject: [PATCH 14/23] gitlab: add a CI job for running checkpatch.pl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This job is advisory since it is expected that certain patches will fail the style checks and checkpatch.pl provides no way to mark exceptions to the rules. Signed-off-by: Daniel P. Berrangé Message-Id: <20200918132903.1848939-2-berrange@redhat.com> [thuth: Use "stage: build" to let it run earlier] Signed-off-by: Thomas Huth --- .gitlab-ci.d/check-patch.py | 48 +++++++++++++++++++++++++++++++++++++ .gitlab-ci.yml | 12 ++++++++++ 2 files changed, 60 insertions(+) create mode 100755 .gitlab-ci.d/check-patch.py diff --git a/.gitlab-ci.d/check-patch.py b/.gitlab-ci.d/check-patch.py new file mode 100755 index 0000000000..5a14a25b13 --- /dev/null +++ b/.gitlab-ci.d/check-patch.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# +# check-patch.py: run checkpatch.pl across all commits in a branch +# +# Copyright (C) 2020 Red Hat, Inc. +# +# SPDX-License-Identifier: GPL-2.0-or-later + +import os +import os.path +import sys +import subprocess + +namespace = "qemu-project" +if len(sys.argv) >= 2: + namespace = sys.argv[1] + +cwd = os.getcwd() +reponame = os.path.basename(cwd) +repourl = "https://gitlab.com/%s/%s.git" % (namespace, reponame) + +# GitLab CI environment does not give us any direct info about the +# base for the user's branch. We thus need to figure out a common +# ancestor between the user's branch and current git master. +subprocess.check_call(["git", "remote", "add", "check-patch", repourl]) +subprocess.check_call(["git", "fetch", "check-patch", "master"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + +ancestor = subprocess.check_output(["git", "merge-base", + "check-patch/master", "HEAD"], + universal_newlines=True) + +ancestor = ancestor.strip() + +subprocess.check_call(["git", "remote", "rm", "check-patch"]) + +errors = False + +print("\nChecking all commits since %s...\n" % ancestor) + +ret = subprocess.run(["scripts/checkpatch.pl", ancestor + "..."]) + +if ret.returncode != 0: + print(" ❌ FAIL one or more commits failed scripts/checkpatch.pl") + sys.exit(1) + +sys.exit(0) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 29e934fd53..f1e18d3e90 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -399,3 +399,15 @@ check-crypto-only-gnutls: variables: IMAGE: centos7 MAKE_CHECK_ARGS: check + + +check-patch: + stage: build + image: $CI_REGISTRY_IMAGE/qemu/centos8:latest + script: .gitlab-ci.d/check-patch.py + except: + variables: + - $CI_PROJECT_NAMESPACE == 'qemu-project' && $CI_COMMIT_BRANCH == 'master' + variables: + GIT_DEPTH: 1000 + allow_failure: true From 34ed46a2845df4305aa35c22ea517b4241faae06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Fri, 18 Sep 2020 14:29:02 +0100 Subject: [PATCH 15/23] gitlab: add a CI job to validate the DCO sign off MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While checkpatch.pl can validate DCO sign off that job must always be advisory only since it is expected that certain patches will fail some code style rules. We require the DCO sign off to be mandatory for all commits though, so it benefits from being validated in a standalone job. Signed-off-by: Daniel P. Berrangé Message-Id: <20200918132903.1848939-3-berrange@redhat.com> [thuth: Use "stage: build" to let it run earlier] Signed-off-by: Thomas Huth --- .gitlab-ci.d/check-dco.py | 94 +++++++++++++++++++++++++++++++++++++++ .gitlab-ci.yml | 10 +++++ 2 files changed, 104 insertions(+) create mode 100755 .gitlab-ci.d/check-dco.py diff --git a/.gitlab-ci.d/check-dco.py b/.gitlab-ci.d/check-dco.py new file mode 100755 index 0000000000..632c8bcce8 --- /dev/null +++ b/.gitlab-ci.d/check-dco.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# +# check-dco.py: validate all commits are signed off +# +# Copyright (C) 2020 Red Hat, Inc. +# +# SPDX-License-Identifier: GPL-2.0-or-later + +import os +import os.path +import sys +import subprocess + +namespace = "qemu-project" +if len(sys.argv) >= 2: + namespace = sys.argv[1] + +cwd = os.getcwd() +reponame = os.path.basename(cwd) +repourl = "https://gitlab.com/%s/%s.git" % (namespace, reponame) + +subprocess.check_call(["git", "remote", "add", "check-dco", repourl]) +subprocess.check_call(["git", "fetch", "check-dco", "master"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + +ancestor = subprocess.check_output(["git", "merge-base", + "check-dco/master", "HEAD"], + universal_newlines=True) + +ancestor = ancestor.strip() + +subprocess.check_call(["git", "remote", "rm", "check-dco"]) + +errors = False + +print("\nChecking for 'Signed-off-by: NAME ' " + + "on all commits since %s...\n" % ancestor) + +log = subprocess.check_output(["git", "log", "--format=%H %s", + ancestor + "..."], + universal_newlines=True) + +if log == "": + commits = [] +else: + commits = [[c[0:40], c[41:]] for c in log.strip().split("\n")] + +for sha, subject in commits: + + msg = subprocess.check_output(["git", "show", "-s", sha], + universal_newlines=True) + lines = msg.strip().split("\n") + + print("🔍 %s %s" % (sha, subject)) + sob = False + for line in lines: + if "Signed-off-by:" in line: + sob = True + if "localhost" in line: + print(" ❌ FAIL: bad email in %s" % line) + errors = True + + if not sob: + print(" ❌ FAIL missing Signed-off-by tag") + errors = True + +if errors: + print(""" + +❌ ERROR: One or more commits are missing a valid Signed-off-By tag. + + +This project requires all contributors to assert that their contributions +are provided in compliance with the terms of the Developer's Certificate +of Origin 1.1 (DCO): + + https://developercertificate.org/ + +To indicate acceptance of the DCO every commit must have a tag + + Signed-off-by: REAL NAME + +This can be achieved by passing the "-s" flag to the "git commit" command. + +To bulk update all commits on current branch "git rebase" can be used: + + git rebase -i master -x 'git commit --amend --no-edit -s' + +""") + + sys.exit(1) + +sys.exit(0) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f1e18d3e90..40805740a6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -411,3 +411,13 @@ check-patch: variables: GIT_DEPTH: 1000 allow_failure: true + +check-dco: + stage: build + image: $CI_REGISTRY_IMAGE/qemu/centos8:latest + script: .gitlab-ci.d/check-dco.py + except: + variables: + - $CI_PROJECT_NAMESPACE == 'qemu-project' && $CI_COMMIT_BRANCH == 'master' + variables: + GIT_DEPTH: 1000 From 4fb716360a6c19d1860dd312ebd3404ca499a186 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Fri, 18 Sep 2020 14:29:03 +0100 Subject: [PATCH 16/23] gitlab: assign python helper files to GitLab maintainers section MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel P. Berrangé Message-Id: <20200918132903.1848939-4-berrange@redhat.com> Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Thomas Huth --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index def5ad8ea9..f7719326e7 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3126,6 +3126,7 @@ R: Wainer dos Santos Moschetta S: Maintained F: .gitlab-ci.yml F: .gitlab-ci.d/crossbuilds.yml +F: .gitlab-ci.d/*py Guest Test Compilation Support M: Alex Bennée From d914375070dc67dead065425fc3e3633fea4e0d3 Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Fri, 4 Sep 2020 12:42:52 -0400 Subject: [PATCH 17/23] scripts/ci/gitlab-pipeline-status: make branch name configurable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the utility function `get_local_staging_branch_commit()`, the name of the branch is hard coded (including in the function name). For extensibility reasons, let's make that configurable. Signed-off-by: Cleber Rosa Message-Id: <20200904164258.240278-2-crosa@redhat.com> Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Thomas Huth --- scripts/ci/gitlab-pipeline-status | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/ci/gitlab-pipeline-status b/scripts/ci/gitlab-pipeline-status index 348a49b6a4..194dd4d0bb 100755 --- a/scripts/ci/gitlab-pipeline-status +++ b/scripts/ci/gitlab-pipeline-status @@ -23,20 +23,20 @@ import time import sys -def get_local_staging_branch_commit(): +def get_local_branch_commit(branch='staging'): """ Returns the commit sha1 for the *local* branch named "staging" """ - result = subprocess.run(['git', 'rev-parse', 'staging'], + result = subprocess.run(['git', 'rev-parse', branch], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, cwd=os.path.dirname(__file__), universal_newlines=True).stdout.strip() - if result == 'staging': - raise ValueError("There's no local branch named 'staging'") + if result == branch: + raise ValueError("There's no local branch named '%s'" % branch) if len(result) != 40: - raise ValueError("Branch staging HEAD doesn't look like a sha1") + raise ValueError("Branch '%s' HEAD doesn't look like a sha1" % branch) return result @@ -110,7 +110,7 @@ def main(): 'for https://gitlab.com/qemu-project/qemu, that ' 'is, "%(default)s"')) try: - default_commit = get_local_staging_branch_commit() + default_commit = get_local_branch_commit() commit_required = False except ValueError: default_commit = '' From 6dfcbff8bf7ca8c52a5ee2d351ec9c291e22fd5e Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Fri, 4 Sep 2020 12:42:53 -0400 Subject: [PATCH 18/23] scripts/ci/gitlab-pipeline-status: improve message regarding timeout The script has its own timeout, which is about how long the script will wait (when called with --wait) for the pipeline to complete, and not necessarily for the pipeline to complete. Hopefully this new wording will be clearer. Signed-off-by: Cleber Rosa Message-Id: <20200904164258.240278-3-crosa@redhat.com> Signed-off-by: Thomas Huth --- scripts/ci/gitlab-pipeline-status | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/ci/gitlab-pipeline-status b/scripts/ci/gitlab-pipeline-status index 194dd4d0bb..2a36f74696 100755 --- a/scripts/ci/gitlab-pipeline-status +++ b/scripts/ci/gitlab-pipeline-status @@ -69,7 +69,10 @@ def wait_on_pipeline_success(timeout, interval, start = time.time() while True: if time.time() >= (start + timeout): - print("Waiting on the pipeline timed out") + msg = ("Timeout (-t/--timeout) of %i seconds reached, " + "won't wait any longer for the pipeline to complete") + msg %= timeout + print(msg) return False status = get_pipeline_status(project_id, commit_sha) From db5424dfda1329da226ad98904c3a26fc9f7310f Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Fri, 4 Sep 2020 12:42:54 -0400 Subject: [PATCH 19/23] scripts/ci/gitlab-pipeline-status: give early feedback on running pipelines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When waiting for a pipeline to run and finish, it's better to give early feedback, and then sleep and wait, than the other wait around. Specially for the first iteration, it's frustrating to see nothing while the script is sleeping. Signed-off-by: Cleber Rosa Message-Id: <20200904164258.240278-4-crosa@redhat.com> Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Thomas Huth --- scripts/ci/gitlab-pipeline-status | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/gitlab-pipeline-status b/scripts/ci/gitlab-pipeline-status index 2a36f74696..18609553be 100755 --- a/scripts/ci/gitlab-pipeline-status +++ b/scripts/ci/gitlab-pipeline-status @@ -77,8 +77,8 @@ def wait_on_pipeline_success(timeout, interval, status = get_pipeline_status(project_id, commit_sha) if status['status'] == 'running': - time.sleep(interval) print('running...') + time.sleep(interval) continue if status['status'] == 'success': From 91641d555ae06726169f6ec8b57caeebcb0b71d0 Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Fri, 4 Sep 2020 12:42:55 -0400 Subject: [PATCH 20/23] scripts/ci/gitlab-pipeline-status: refactor parser creation Out of the main function. Signed-off-by: Cleber Rosa Message-Id: <20200904164258.240278-5-crosa@redhat.com> Signed-off-by: Thomas Huth --- scripts/ci/gitlab-pipeline-status | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/ci/gitlab-pipeline-status b/scripts/ci/gitlab-pipeline-status index 18609553be..8355b6a427 100755 --- a/scripts/ci/gitlab-pipeline-status +++ b/scripts/ci/gitlab-pipeline-status @@ -89,10 +89,7 @@ def wait_on_pipeline_success(timeout, interval, return False -def main(): - """ - Script entry point - """ +def create_parser(): parser = argparse.ArgumentParser( prog='pipeline-status', description='check or wait on a pipeline status') @@ -127,7 +124,13 @@ def main(): parser.add_argument('--verbose', action='store_true', default=False, help=('A minimal verbosity level that prints the ' 'overall result of the check/wait')) + return parser +def main(): + """ + Script entry point + """ + parser = create_parser() args = parser.parse_args() try: From 79df438eeb5e754a0ae22210238e4e6555e7428f Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Fri, 4 Sep 2020 12:42:56 -0400 Subject: [PATCH 21/23] scripts/ci/gitlab-pipeline-status: handle keyboard interrupts So that exits based on user requests are handled more gracefully. Signed-off-by: Cleber Rosa Message-Id: <20200904164258.240278-6-crosa@redhat.com> Signed-off-by: Thomas Huth --- scripts/ci/gitlab-pipeline-status | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/ci/gitlab-pipeline-status b/scripts/ci/gitlab-pipeline-status index 8355b6a427..ced488f27c 100755 --- a/scripts/ci/gitlab-pipeline-status +++ b/scripts/ci/gitlab-pipeline-status @@ -132,7 +132,7 @@ def main(): """ parser = create_parser() args = parser.parse_args() - + success = False try: if args.wait: success = wait_on_pipeline_success( @@ -145,9 +145,11 @@ def main(): args.commit) success = status['status'] == 'success' except Exception as error: # pylint: disable=W0703 - success = False if args.verbose: print("ERROR: %s" % error.args[0]) + except KeyboardInterrupt: + if args.verbose: + print("Exiting on user's request") if success: if args.verbose: From 176498ab57dc14a7c14a58b490aa16319f0cf638 Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Fri, 4 Sep 2020 12:42:57 -0400 Subject: [PATCH 22/23] scripts/ci/gitlab-pipeline-status: use more descriptive exceptions For two very different error conditions. Signed-off-by: Cleber Rosa Message-Id: <20200904164258.240278-7-crosa@redhat.com> Signed-off-by: Thomas Huth --- scripts/ci/gitlab-pipeline-status | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/ci/gitlab-pipeline-status b/scripts/ci/gitlab-pipeline-status index ced488f27c..628150ce0b 100755 --- a/scripts/ci/gitlab-pipeline-status +++ b/scripts/ci/gitlab-pipeline-status @@ -23,6 +23,14 @@ import time import sys +class CommunicationFailure(Exception): + """Failed to communicate to gitlab.com APIs.""" + + +class NoPipelineFound(Exception): + """Communication is successfull but pipeline is not found.""" + + def get_local_branch_commit(branch='staging'): """ Returns the commit sha1 for the *local* branch named "staging" @@ -50,14 +58,14 @@ def get_pipeline_status(project_id, commit_sha1): connection.request('GET', url=url) response = connection.getresponse() if response.code != http.HTTPStatus.OK: - raise ValueError("Failed to receive a successful response") + raise CommunicationFailure("Failed to receive a successful response") json_response = json.loads(response.read()) # As far as I can tell, there should be only one pipeline for the same # project + commit. If this assumption is false, we can add further # filters to the url, such as username, and order_by. if not json_response: - raise ValueError("No pipeline found") + raise NoPipelineFound("No pipeline found") return json_response[0] From ea8bf1e514d2f442dd1a008794eb1563e2ee1c48 Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Fri, 4 Sep 2020 12:42:58 -0400 Subject: [PATCH 23/23] scripts/ci/gitlab-pipeline-status: wait for pipeline creation When called in wait mode, this script will also wait for the pipeline to be get to a "running" state. Because many more status may be seen until a pipeline gets to "running", and those need to be handle too. Reference: https://docs.gitlab.com/ee/api/pipelines.html#list-project-pipelines Signed-off-by: Cleber Rosa Message-Id: <20200904164258.240278-8-crosa@redhat.com> Signed-off-by: Thomas Huth --- scripts/ci/gitlab-pipeline-status | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/ci/gitlab-pipeline-status b/scripts/ci/gitlab-pipeline-status index 628150ce0b..bac8233079 100755 --- a/scripts/ci/gitlab-pipeline-status +++ b/scripts/ci/gitlab-pipeline-status @@ -83,13 +83,22 @@ def wait_on_pipeline_success(timeout, interval, print(msg) return False - status = get_pipeline_status(project_id, commit_sha) - if status['status'] == 'running': - print('running...') + try: + status = get_pipeline_status(project_id, commit_sha) + except NoPipelineFound: + print('Pipeline has not been found, it may not have been created yet.') + time.sleep(1) + continue + + pipeline_status = status['status'] + status_to_wait = ('created', 'waiting_for_resource', 'preparing', + 'pending', 'running') + if pipeline_status in status_to_wait: + print('%s...' % pipeline_status) time.sleep(interval) continue - if status['status'] == 'success': + if pipeline_status == 'success': return True msg = "Pipeline failed, check: %s" % status['web_url']