From ca9cde57abb23753acafab77856f6816858a2089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 14 Sep 2023 16:54:14 +0100 Subject: [PATCH 1/9] tests: update most Debian images to Bookworm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bookworm has been out a while now. Time to update our containers to the current stable. This requires the latest lcitool repo so update the sub-module too. For some reason the MIPs containers won't build so skip those for now. We also have to skip the armel builds due to a stuck libc update. Reviewed-by: Thomas Huth Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Alex Bennée Message-Id: <20230914155422.426639-2-alex.bennee@linaro.org> --- .../dockerfiles/debian-amd64-cross.docker | 9 +++------ tests/docker/dockerfiles/debian-amd64.docker | 9 +++------ .../dockerfiles/debian-arm64-cross.docker | 9 +++------ .../dockerfiles/debian-armhf-cross.docker | 9 +++------ .../dockerfiles/debian-ppc64el-cross.docker | 9 +++------ .../dockerfiles/debian-s390x-cross.docker | 9 +++------ tests/lcitool/libvirt-ci | 2 +- tests/lcitool/refresh | 17 +++++++++-------- 8 files changed, 28 insertions(+), 45 deletions(-) diff --git a/tests/docker/dockerfiles/debian-amd64-cross.docker b/tests/docker/dockerfiles/debian-amd64-cross.docker index 0cf3ba6d60..0991938595 100644 --- a/tests/docker/dockerfiles/debian-amd64-cross.docker +++ b/tests/docker/dockerfiles/debian-amd64-cross.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all --cross-arch x86_64 debian-11 qemu +# $ lcitool dockerfile --layers all --cross-arch x86_64 debian-12 qemu # # https://gitlab.com/libvirt/libvirt-ci -FROM docker.io/library/debian:11-slim +FROM docker.io/library/debian:12-slim RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ @@ -47,11 +47,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-opencv \ python3-pillow \ python3-pip \ - python3-setuptools \ python3-sphinx \ python3-sphinx-rtd-theme \ python3-venv \ - python3-wheel \ python3-yaml \ rpm2cpio \ sed \ @@ -67,8 +65,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ dpkg-reconfigure locales -RUN /usr/bin/pip3 install tomli - ENV CCACHE_WRAPPERSDIR "/usr/libexec/ccache-wrappers" ENV LANG "en_US.UTF-8" ENV MAKE "/usr/bin/make" @@ -146,6 +142,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ libvdeplug-dev:amd64 \ libvirglrenderer-dev:amd64 \ libvte-2.91-dev:amd64 \ + libxdp-dev:amd64 \ libxen-dev:amd64 \ libzstd-dev:amd64 \ nettle-dev:amd64 \ diff --git a/tests/docker/dockerfiles/debian-amd64.docker b/tests/docker/dockerfiles/debian-amd64.docker index e3e1de25dd..61dbc3ff24 100644 --- a/tests/docker/dockerfiles/debian-amd64.docker +++ b/tests/docker/dockerfiles/debian-amd64.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all debian-11 qemu +# $ lcitool dockerfile --layers all debian-12 qemu # # https://gitlab.com/libvirt/libvirt-ci -FROM docker.io/library/debian:11-slim +FROM docker.io/library/debian:12-slim RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ @@ -97,6 +97,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ libvdeplug-dev \ libvirglrenderer-dev \ libvte-2.91-dev \ + libxdp-dev \ libxen-dev \ libzstd-dev \ llvm \ @@ -115,11 +116,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-opencv \ python3-pillow \ python3-pip \ - python3-setuptools \ python3-sphinx \ python3-sphinx-rtd-theme \ python3-venv \ - python3-wheel \ python3-yaml \ rpm2cpio \ sed \ @@ -145,8 +144,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/g++ && \ ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/gcc -RUN /usr/bin/pip3 install tomli - ENV CCACHE_WRAPPERSDIR "/usr/libexec/ccache-wrappers" ENV LANG "en_US.UTF-8" ENV MAKE "/usr/bin/make" diff --git a/tests/docker/dockerfiles/debian-arm64-cross.docker b/tests/docker/dockerfiles/debian-arm64-cross.docker index d8cd4f87b6..74eabb274e 100644 --- a/tests/docker/dockerfiles/debian-arm64-cross.docker +++ b/tests/docker/dockerfiles/debian-arm64-cross.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all --cross-arch aarch64 debian-11 qemu +# $ lcitool dockerfile --layers all --cross-arch aarch64 debian-12 qemu # # https://gitlab.com/libvirt/libvirt-ci -FROM docker.io/library/debian:11-slim +FROM docker.io/library/debian:12-slim RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ @@ -47,11 +47,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-opencv \ python3-pillow \ python3-pip \ - python3-setuptools \ python3-sphinx \ python3-sphinx-rtd-theme \ python3-venv \ - python3-wheel \ python3-yaml \ rpm2cpio \ sed \ @@ -67,8 +65,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ dpkg-reconfigure locales -RUN /usr/bin/pip3 install tomli - ENV CCACHE_WRAPPERSDIR "/usr/libexec/ccache-wrappers" ENV LANG "en_US.UTF-8" ENV MAKE "/usr/bin/make" @@ -145,6 +141,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ libvdeplug-dev:arm64 \ libvirglrenderer-dev:arm64 \ libvte-2.91-dev:arm64 \ + libxdp-dev:arm64 \ libxen-dev:arm64 \ libzstd-dev:arm64 \ nettle-dev:arm64 \ diff --git a/tests/docker/dockerfiles/debian-armhf-cross.docker b/tests/docker/dockerfiles/debian-armhf-cross.docker index f45cfedd3f..1ebd6ebd00 100644 --- a/tests/docker/dockerfiles/debian-armhf-cross.docker +++ b/tests/docker/dockerfiles/debian-armhf-cross.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all --cross-arch armv7l debian-11 qemu +# $ lcitool dockerfile --layers all --cross-arch armv7l debian-12 qemu # # https://gitlab.com/libvirt/libvirt-ci -FROM docker.io/library/debian:11-slim +FROM docker.io/library/debian:12-slim RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ @@ -47,11 +47,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-opencv \ python3-pillow \ python3-pip \ - python3-setuptools \ python3-sphinx \ python3-sphinx-rtd-theme \ python3-venv \ - python3-wheel \ python3-yaml \ rpm2cpio \ sed \ @@ -67,8 +65,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ dpkg-reconfigure locales -RUN /usr/bin/pip3 install tomli - ENV CCACHE_WRAPPERSDIR "/usr/libexec/ccache-wrappers" ENV LANG "en_US.UTF-8" ENV MAKE "/usr/bin/make" @@ -145,6 +141,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ libvdeplug-dev:armhf \ libvirglrenderer-dev:armhf \ libvte-2.91-dev:armhf \ + libxdp-dev:armhf \ libxen-dev:armhf \ libzstd-dev:armhf \ nettle-dev:armhf \ diff --git a/tests/docker/dockerfiles/debian-ppc64el-cross.docker b/tests/docker/dockerfiles/debian-ppc64el-cross.docker index 52f8c34814..59091fed02 100644 --- a/tests/docker/dockerfiles/debian-ppc64el-cross.docker +++ b/tests/docker/dockerfiles/debian-ppc64el-cross.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all --cross-arch ppc64le debian-11 qemu +# $ lcitool dockerfile --layers all --cross-arch ppc64le debian-12 qemu # # https://gitlab.com/libvirt/libvirt-ci -FROM docker.io/library/debian:11-slim +FROM docker.io/library/debian:12-slim RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ @@ -47,11 +47,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-opencv \ python3-pillow \ python3-pip \ - python3-setuptools \ python3-sphinx \ python3-sphinx-rtd-theme \ python3-venv \ - python3-wheel \ python3-yaml \ rpm2cpio \ sed \ @@ -67,8 +65,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ dpkg-reconfigure locales -RUN /usr/bin/pip3 install tomli - ENV CCACHE_WRAPPERSDIR "/usr/libexec/ccache-wrappers" ENV LANG "en_US.UTF-8" ENV MAKE "/usr/bin/make" @@ -145,6 +141,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ libvdeplug-dev:ppc64el \ libvirglrenderer-dev:ppc64el \ libvte-2.91-dev:ppc64el \ + libxdp-dev:ppc64el \ libzstd-dev:ppc64el \ nettle-dev:ppc64el \ systemtap-sdt-dev:ppc64el \ diff --git a/tests/docker/dockerfiles/debian-s390x-cross.docker b/tests/docker/dockerfiles/debian-s390x-cross.docker index 208e57bcf2..48b2f28310 100644 --- a/tests/docker/dockerfiles/debian-s390x-cross.docker +++ b/tests/docker/dockerfiles/debian-s390x-cross.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all --cross-arch s390x debian-11 qemu +# $ lcitool dockerfile --layers all --cross-arch s390x debian-12 qemu # # https://gitlab.com/libvirt/libvirt-ci -FROM docker.io/library/debian:11-slim +FROM docker.io/library/debian:12-slim RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ @@ -47,11 +47,9 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-opencv \ python3-pillow \ python3-pip \ - python3-setuptools \ python3-sphinx \ python3-sphinx-rtd-theme \ python3-venv \ - python3-wheel \ python3-yaml \ rpm2cpio \ sed \ @@ -67,8 +65,6 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ dpkg-reconfigure locales -RUN /usr/bin/pip3 install tomli - ENV CCACHE_WRAPPERSDIR "/usr/libexec/ccache-wrappers" ENV LANG "en_US.UTF-8" ENV MAKE "/usr/bin/make" @@ -144,6 +140,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ libvdeplug-dev:s390x \ libvirglrenderer-dev:s390x \ libvte-2.91-dev:s390x \ + libxdp-dev:s390x \ libzstd-dev:s390x \ nettle-dev:s390x \ systemtap-sdt-dev:s390x \ diff --git a/tests/lcitool/libvirt-ci b/tests/lcitool/libvirt-ci index 5f84a21881..e3ed1e5da1 160000 --- a/tests/lcitool/libvirt-ci +++ b/tests/lcitool/libvirt-ci @@ -1 +1 @@ -Subproject commit 5f84a21881577a5fb56cc956f6fe4e2abd6fcff0 +Subproject commit e3ed1e5da101943e53d8d89424e17b22120743f5 diff --git a/tests/lcitool/refresh b/tests/lcitool/refresh index 4584870ea1..92e7d30982 100755 --- a/tests/lcitool/refresh +++ b/tests/lcitool/refresh @@ -93,7 +93,7 @@ def generate_pkglist(vm, target): # Netmap still needs to be manually built as it is yet to be packaged # into a distro. We also add cscope and gtags which are used in the CI # test -debian11_extras = [ +debian12_extras = [ "# netmap/cscope/global\n", "RUN DEBIAN_FRONTEND=noninteractive eatmydata \\\n", " apt install -y --no-install-recommends \\\n", @@ -123,8 +123,8 @@ try: # generate_dockerfile("alpine", "alpine-318") generate_dockerfile("centos8", "centos-stream-8") - generate_dockerfile("debian-amd64", "debian-11", - trailer="".join(debian11_extras)) + generate_dockerfile("debian-amd64", "debian-12", + trailer="".join(debian12_extras)) generate_dockerfile("fedora", "fedora-38") generate_dockerfile("opensuse-leap", "opensuse-leap-15") generate_dockerfile("ubuntu2004", "ubuntu-2004") @@ -133,24 +133,25 @@ try: # # Cross compiling builds # - generate_dockerfile("debian-amd64-cross", "debian-11", + generate_dockerfile("debian-amd64-cross", "debian-12", cross="x86_64", trailer=cross_build("x86_64-linux-gnu-", "x86_64-softmmu," "x86_64-linux-user," "i386-softmmu,i386-linux-user")) - generate_dockerfile("debian-arm64-cross", "debian-11", + generate_dockerfile("debian-arm64-cross", "debian-12", cross="aarch64", trailer=cross_build("aarch64-linux-gnu-", "aarch64-softmmu,aarch64-linux-user")) + # migration to bookworm stalled: https://lists.debian.org/debian-arm/2023/09/msg00006.html generate_dockerfile("debian-armel-cross", "debian-11", cross="armv6l", trailer=cross_build("arm-linux-gnueabi-", "arm-softmmu,arm-linux-user,armeb-linux-user")) - generate_dockerfile("debian-armhf-cross", "debian-11", + generate_dockerfile("debian-armhf-cross", "debian-12", cross="armv7l", trailer=cross_build("arm-linux-gnueabihf-", "arm-softmmu,arm-linux-user")) @@ -165,7 +166,7 @@ try: trailer=cross_build("mipsel-linux-gnu-", "mipsel-softmmu,mipsel-linux-user")) - generate_dockerfile("debian-ppc64el-cross", "debian-11", + generate_dockerfile("debian-ppc64el-cross", "debian-12", cross="ppc64le", trailer=cross_build("powerpc64le-linux-gnu-", "ppc64-softmmu,ppc64-linux-user")) @@ -176,7 +177,7 @@ try: trailer=cross_build("riscv64-linux-gnu-", "riscv64-softmmu,riscv64-linux-user")) - generate_dockerfile("debian-s390x-cross", "debian-11", + generate_dockerfile("debian-s390x-cross", "debian-12", cross="s390x", trailer=cross_build("s390x-linux-gnu-", "s390x-softmmu,s390x-linux-user")) From 3aa2cfa0e6185ce7c6197786ff2407d564996c85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 14 Sep 2023 16:54:15 +0100 Subject: [PATCH 2/9] gitlab: fix typo/spelling in comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Thomas Huth Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Alex Bennée Message-Id: <20230914155422.426639-3-alex.bennee@linaro.org> --- .gitlab-ci.d/base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.d/base.yml b/.gitlab-ci.d/base.yml index 188a770799..ef173a34e6 100644 --- a/.gitlab-ci.d/base.yml +++ b/.gitlab-ci.d/base.yml @@ -68,7 +68,7 @@ variables: ############################################################# # Stage 2: fine tune execution of jobs in specific scenarios - # where the catch all logic is inapprorpaite + # where the catch all logic is inappropriate ############################################################# # Optional jobs should not be run unless manually triggered From 232ce1eb902846637ac7c2ba33c16c984b145a3f Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 14 Sep 2023 16:54:16 +0100 Subject: [PATCH 3/9] tests/docker: Update docker-loongarch-cross toolchain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update from clfs 5.0 to clfs 8.1, which includes updates to binutils 2.41, gcc 13.2, and glibc 2.38. See https://github.com/loongson/build-tools Signed-off-by: Richard Henderson Reviewed-by: Song Gao Reviewed-by: Alex Bennée Signed-off-by: Alex Bennée Message-Id: <20230829220228.928506-2-richard.henderson@linaro.org> Message-Id: <20230914155422.426639-4-alex.bennee@linaro.org> --- tests/docker/dockerfiles/debian-loongarch-cross.docker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docker/dockerfiles/debian-loongarch-cross.docker b/tests/docker/dockerfiles/debian-loongarch-cross.docker index 9d957547b5..b4bf265717 100644 --- a/tests/docker/dockerfiles/debian-loongarch-cross.docker +++ b/tests/docker/dockerfiles/debian-loongarch-cross.docker @@ -20,7 +20,7 @@ RUN apt-get update && \ git \ python3-minimal -RUN curl -#SL https://github.com/loongson/build-tools/releases/download/2022.05.29/loongarch64-clfs-5.0-cross-tools-gcc-glibc.tar.xz \ +RUN curl -#SL https://github.com/loongson/build-tools/releases/download/2023.08.08/CLFS-loongarch64-8.1-x86_64-cross-tools-gcc-glibc.tar.xz \ | tar -xJC /opt ENV PATH $PATH:/opt/cross-tools/bin From 81395b6ec8364b37ee85b37faf18930800ad5db5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Thu, 14 Sep 2023 16:54:17 +0100 Subject: [PATCH 4/9] microbit: add missing qtest_quit() call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without this call, the QEMU process is being left running which on FreeBSD 13.2 at least, makes meson think the test is still running, and thus execution of "make check" continues forever. This fixes the regression introduced in: commit a9c9bbee855877293683012942d3485d50f286af Author: Chris Laplante Date: Tue Aug 22 17:31:02 2023 +0100 qtest: microbit-test: add tests for nRF51 DETECT Fixes: https://gitlab.com/qemu-project/qemu/-/issues/1882 Signed-off-by: "Daniel P. Berrangé" Reviewed-by: Richard Henderson Reviewed-by: Stefan Hajnoczi Reviewed-by: Thomas Huth Message-Id: <20230912184130.3056054-2-berrange@redhat.com> Signed-off-by: Alex Bennée Message-Id: <20230914155422.426639-5-alex.bennee@linaro.org> --- tests/qtest/microbit-test.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/qtest/microbit-test.c b/tests/qtest/microbit-test.c index 2abcad8e31..72190d38f7 100644 --- a/tests/qtest/microbit-test.c +++ b/tests/qtest/microbit-test.c @@ -434,6 +434,8 @@ static void test_nrf51_gpio_detect(void) g_assert_true(qtest_get_irq(qts, 0)); qtest_set_irq_in(qts, "/machine/nrf51", "unnamed-gpio-in", 3, 0); g_assert_true(qtest_get_irq(qts, 0)); + + qtest_quit(qts); } static void timer_task(QTestState *qts, hwaddr task) From 49e9f8644b036b42b0fc65bc8bef992fcee2f610 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Thu, 14 Sep 2023 16:54:18 +0100 Subject: [PATCH 5/9] qtest: kill orphaned qtest QEMU processes on FreeBSD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Linux we use PR_SET_PDEATHSIG to kill orphaned QEMU processes if we fail to call qtest_quit(), or the test program aborts/segvs. This prevents meson from hanging forever due to the orphaned process keeping stdout open. On FreeBSD we can achieve the same using PROC_PDEATHSIG_CTL, which gives us the equivalent protection against hangs. Signed-off-by: "Daniel P. Berrangé" Reviewed-by: Richard Henderson Reviewed-by: Stefan Hajnoczi Reviewed-by: Thomas Huth Message-Id: <20230912184130.3056054-3-berrange@redhat.com> Signed-off-by: Alex Bennée Message-Id: <20230914155422.426639-6-alex.bennee@linaro.org> --- tests/qtest/libqtest.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/qtest/libqtest.c b/tests/qtest/libqtest.c index 34b9c14b75..b1eba71ffe 100644 --- a/tests/qtest/libqtest.c +++ b/tests/qtest/libqtest.c @@ -24,6 +24,9 @@ #ifdef __linux__ #include #endif /* __linux__ */ +#ifdef __FreeBSD__ +#include +#endif /* __FreeBSD__ */ #include "libqtest.h" #include "libqmp.h" @@ -414,6 +417,10 @@ static QTestState *G_GNUC_PRINTF(1, 2) qtest_spawn_qemu(const char *fmt, ...) */ prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0); #endif /* __linux__ */ +#ifdef __FreeBSD__ + int sig = SIGKILL; + procctl(P_PID, getpid(), PROC_PDEATHSIG_CTL, &sig); +#endif /* __FreeBSD__ */ if (!g_setenv("QEMU_AUDIO_DRV", "none", true)) { exit(1); } From c576d8bfbb6eca37d4258be4275efd617b1f1d94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Thu, 14 Sep 2023 16:54:19 +0100 Subject: [PATCH 6/9] gitlab: make Cirrus CI timeout explicit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the GitLab side we're invoking the Cirrus CI job using the cirrus-run tool which speaks to the Cirrus REST API. Cirrus sometimes tasks 5-10 minutes to actually schedule the task, and thus the execution time of 'cirrus-run' inside GitLab will be slightly longer than the execution time of the Cirrus CI task. Setting the timeout in the GitLab CI job should thus be done in relation to the timeout set for the Cirrus CI job. While Cirrus CI defaults to 60 minutes, it is better to set this explicitly, and make the relationship between the jobs explicit Signed-off-by: "Daniel P. Berrangé" Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Stefan Hajnoczi Reviewed-by: Thomas Huth Message-Id: <20230912184130.3056054-4-berrange@redhat.com> Signed-off-by: Alex Bennée Message-Id: <20230914155422.426639-7-alex.bennee@linaro.org> --- .gitlab-ci.d/cirrus.yml | 3 +++ .gitlab-ci.d/cirrus/build.yml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.gitlab-ci.d/cirrus.yml b/.gitlab-ci.d/cirrus.yml index 41d64d6680..816d89cc2a 100644 --- a/.gitlab-ci.d/cirrus.yml +++ b/.gitlab-ci.d/cirrus.yml @@ -15,6 +15,9 @@ stage: build image: registry.gitlab.com/libvirt/libvirt-ci/cirrus-run:master needs: [] + # 20 mins larger than "timeout_in" in cirrus/build.yml + # as there's often a 5-10 minute delay before Cirrus CI + # actually starts the task timeout: 80m allow_failure: true script: diff --git a/.gitlab-ci.d/cirrus/build.yml b/.gitlab-ci.d/cirrus/build.yml index a9444902ec..29d55c4aa3 100644 --- a/.gitlab-ci.d/cirrus/build.yml +++ b/.gitlab-ci.d/cirrus/build.yml @@ -16,6 +16,8 @@ env: TEST_TARGETS: "@TEST_TARGETS@" build_task: + # A little shorter than GitLab timeout in ../cirrus.yml + timeout_in: 60m install_script: - @UPDATE_COMMAND@ - @INSTALL_COMMAND@ @PKGS@ From eca74afd7dfbe7363fb2b7e8a7b1dae4bc05cd25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Thu, 14 Sep 2023 16:54:20 +0100 Subject: [PATCH 7/9] gitlab: make Cirrus CI jobs gating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Cirrus CI jobs have been non-gating for a while to let us build confidence in their reliability. Aside from periodic dependancy problems when FreeBSD Ports switches to be based on a new FreeBSD image version, the jobs have been reliable. It is thus worth making them gating to prevent build failures being missed during merges. Signed-off-by: "Daniel P. Berrangé" Reviewed-by: Stefan Hajnoczi Reviewed-by: Thomas Huth Message-Id: <20230912184130.3056054-5-berrange@redhat.com> Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Alex Bennée Message-Id: <20230914155422.426639-8-alex.bennee@linaro.org> --- .gitlab-ci.d/cirrus.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.d/cirrus.yml b/.gitlab-ci.d/cirrus.yml index 816d89cc2a..e7f1f83c2c 100644 --- a/.gitlab-ci.d/cirrus.yml +++ b/.gitlab-ci.d/cirrus.yml @@ -19,7 +19,6 @@ # as there's often a 5-10 minute delay before Cirrus CI # actually starts the task timeout: 80m - allow_failure: true script: - source .gitlab-ci.d/cirrus/$NAME.vars - sed -e "s|[@]CI_REPOSITORY_URL@|$CI_REPOSITORY_URL|g" From f0ec14c78c4583fc34c16625ca6bf3900bcccba5 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Thu, 14 Sep 2023 16:54:21 +0100 Subject: [PATCH 8/9] tests/avocado: Fix console data loss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Occasionally some avocado tests will fail waiting for console line despite the machine running correctly. Console data goes missing, as can be seen in the console log. This is due to _console_interaction calling makefile() on the console socket each time it is invoked, which must be losing old buffer contents when going out of scope. It is not enough to makefile() with buffered=0. That helps significantly but data loss is still possible. My guess is that readline() has a line buffer even when the file is in unbuffered mode, that can eat data. Fix this by providing a console file that persists for the life of the console. Signed-off-by: Nicholas Piggin Reviewed-by: "Daniel P. Berrangé" Message-Id: <20230912131340.405619-1-npiggin@gmail.com> Reviewed-by: Philippe Mathieu-Daudé Acked-by: John Snow Signed-off-by: Alex Bennée Message-Id: <20230914155422.426639-9-alex.bennee@linaro.org> --- python/qemu/machine/machine.py | 19 +++++++++++++++++++ tests/avocado/avocado_qemu/__init__.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/python/qemu/machine/machine.py b/python/qemu/machine/machine.py index c16a0b6fed..35d5a672db 100644 --- a/python/qemu/machine/machine.py +++ b/python/qemu/machine/machine.py @@ -191,6 +191,7 @@ class QEMUMachine: self.sock_dir, f"{self._name}.con" ) self._console_socket: Optional[socket.socket] = None + self._console_file: Optional[socket.SocketIO] = None self._remove_files: List[str] = [] self._user_killed = False self._quit_issued = False @@ -509,6 +510,11 @@ class QEMUMachine: # If we keep the console socket open, we may deadlock waiting # for QEMU to exit, while QEMU is waiting for the socket to # become writable. + if self._console_file is not None: + LOG.debug("Closing console file") + self._console_file.close() + self._console_file = None + if self._console_socket is not None: LOG.debug("Closing console socket") self._console_socket.close() @@ -874,12 +880,25 @@ class QEMUMachine: Returns a socket connected to the console """ if self._console_socket is None: + LOG.debug("Opening console socket") self._console_socket = console_socket.ConsoleSocket( self._console_address, file=self._console_log_path, drain=self._drain_console) return self._console_socket + @property + def console_file(self) -> socket.SocketIO: + """ + Returns a file associated with the console socket + """ + if self._console_file is None: + LOG.debug("Opening console file") + self._console_file = self.console_socket.makefile(mode='rb', + buffering=0, + encoding='utf-8') + return self._console_file + @property def temp_dir(self) -> str: """ diff --git a/tests/avocado/avocado_qemu/__init__.py b/tests/avocado/avocado_qemu/__init__.py index 33090903f1..0172a359b7 100644 --- a/tests/avocado/avocado_qemu/__init__.py +++ b/tests/avocado/avocado_qemu/__init__.py @@ -137,7 +137,7 @@ def _console_interaction(test, success_message, failure_message, assert not keep_sending or send_string if vm is None: vm = test.vm - console = vm.console_socket.makefile(mode='rb', encoding='utf-8') + console = vm.console_file console_logger = logging.getLogger('console') while True: if send_string: From f959c3d87ccfa585b105de6964a6261e368cc1da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Thu, 14 Sep 2023 16:54:22 +0100 Subject: [PATCH 9/9] tests/avocado: Disable MIPS Malta tests due to GitLab issue #1884 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 0d58c66068 ("softmmu: Use async_run_on_cpu in tcg_commit") introduced a regression which is only triggered by the MIPS Malta machine. Since those tests are gatting and disturb the CI workflow, disable them until https://gitlab.com/qemu-project/qemu/-/issues/1866 is fixed. $ make check-avocado \ AVOCADO_TAGS='arch:mipsel arch:mips64el' \ AVOCADO_ALLOW_UNTRUSTED_CODE=1 \ AVOCADO_TIMEOUT_EXPECTED=1 AVOCADO tests/avocado (04/24) tests/avocado/boot_linux_console.py:BootLinuxConsole.test_mips_malta32el_nanomips_4k: INTERRUPTED: Test interrupted by SIGTERM\nRunner error occurred: Timeout reached\nOriginal status: ERROR\n... (90.39 s) (05/24) tests/avocado/boot_linux_console.py:BootLinuxConsole.test_mips_malta32el_nanomips_16k_up: INTERRUPTED: Test interrupted by SIGTERM\nRunner error occurred: Timeout reached\nOriginal status: ERROR\n... (90.29 s) (06/24) tests/avocado/boot_linux_console.py:BootLinuxConsole.test_mips_malta32el_nanomips_64k_dbg: INTERRUPTED: Test interrupted by SIGTERM\nRunner error occurred: Timeout reached\nOriginal status: ERROR\n... (92.53 s) (11/24) tests/avocado/machine_mips_malta.py:MaltaMachineFramebuffer.test_mips_malta_i6400_framebuffer_logo_1core: INTERRUPTED: Test interrupted by SIGTERM\nRunner error occurred: Timeout reached\nOriginal status: ERROR\n... (25.78 s) RESULTS : PASS 8 | ERROR 0 | FAIL 0 | SKIP 7 | WARN 2 | INTERRUPT 5 | CANCEL 2 JOB TIME : 525.60 s ^^^^^^^^^^^ Reported-by: Thomas Huth Suggested-by: Thomas Huth Signed-off-by: Philippe Mathieu-Daudé Message-Id: <20230913135339.9128-1-philmd@linaro.org> Signed-off-by: Alex Bennée Message-Id: <20230914155422.426639-10-alex.bennee@linaro.org> --- tests/avocado/boot_linux_console.py | 7 +++++++ tests/avocado/machine_mips_malta.py | 6 ++++++ tests/avocado/replay_kernel.py | 7 +++++++ tests/avocado/tuxrun_baselines.py | 4 ++++ 4 files changed, 24 insertions(+) diff --git a/tests/avocado/boot_linux_console.py b/tests/avocado/boot_linux_console.py index 6eab515718..01ee149812 100644 --- a/tests/avocado/boot_linux_console.py +++ b/tests/avocado/boot_linux_console.py @@ -116,6 +116,7 @@ class BootLinuxConsole(LinuxKernelTest): console_pattern = 'Kernel command line: %s' % kernel_command_line self.wait_for_console_pattern(console_pattern) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta(self): """ :avocado: tags=arch:mips @@ -138,6 +139,7 @@ class BootLinuxConsole(LinuxKernelTest): console_pattern = 'Kernel command line: %s' % kernel_command_line self.wait_for_console_pattern(console_pattern) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips64el_malta(self): """ This test requires the ar tool to extract "data.tar.gz" from @@ -191,6 +193,7 @@ class BootLinuxConsole(LinuxKernelTest): console_pattern = 'Kernel command line: %s' % kernel_command_line self.wait_for_console_pattern(console_pattern) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta_cpio(self): """ :avocado: tags=arch:mips @@ -232,6 +235,7 @@ class BootLinuxConsole(LinuxKernelTest): # Wait for VM to shut down gracefully self.vm.wait() + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') @skipUnless(os.getenv('AVOCADO_ALLOW_UNTRUSTED_CODE'), 'untrusted code') def test_mips64el_malta_5KEc_cpio(self): """ @@ -292,6 +296,7 @@ class BootLinuxConsole(LinuxKernelTest): console_pattern = 'Kernel command line: %s' % kernel_command_line self.wait_for_console_pattern(console_pattern) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta32el_nanomips_4k(self): """ :avocado: tags=arch:mipsel @@ -305,6 +310,7 @@ class BootLinuxConsole(LinuxKernelTest): kernel_hash = '477456aafd2a0f1ddc9482727f20fe9575565dd6' self.do_test_mips_malta32el_nanomips(kernel_url, kernel_hash) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta32el_nanomips_16k_up(self): """ :avocado: tags=arch:mipsel @@ -318,6 +324,7 @@ class BootLinuxConsole(LinuxKernelTest): kernel_hash = 'e882868f944c71c816e832e2303b7874d044a7bc' self.do_test_mips_malta32el_nanomips(kernel_url, kernel_hash) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta32el_nanomips_64k_dbg(self): """ :avocado: tags=arch:mipsel diff --git a/tests/avocado/machine_mips_malta.py b/tests/avocado/machine_mips_malta.py index 92233451c5..3620266589 100644 --- a/tests/avocado/machine_mips_malta.py +++ b/tests/avocado/machine_mips_malta.py @@ -11,6 +11,7 @@ import os import gzip import logging +from avocado import skip from avocado import skipIf from avocado import skipUnless from avocado.utils import archive @@ -93,6 +94,7 @@ class MaltaMachineFramebuffer(QemuSystemTest): cv2.imwrite(debug_png, screendump_bgr) self.assertGreaterEqual(tuxlogo_count, cpu_cores_count) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta_i6400_framebuffer_logo_1core(self): """ :avocado: tags=arch:mips64el @@ -101,6 +103,7 @@ class MaltaMachineFramebuffer(QemuSystemTest): """ self.do_test_i6400_framebuffer_logo(1) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_mips_malta_i6400_framebuffer_logo_7cores(self): """ @@ -111,6 +114,7 @@ class MaltaMachineFramebuffer(QemuSystemTest): """ self.do_test_i6400_framebuffer_logo(7) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_mips_malta_i6400_framebuffer_logo_8cores(self): """ @@ -142,6 +146,7 @@ class MaltaMachine(QemuSystemTest): wait_for_console_pattern(self, prompt) self.vm.shutdown() + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mipsel_malta_yamon(self): """ :avocado: tags=arch:mipsel @@ -150,6 +155,7 @@ class MaltaMachine(QemuSystemTest): """ self.do_test_yamon() + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips64el_malta_yamon(self): """ :avocado: tags=arch:mips64el diff --git a/tests/avocado/replay_kernel.py b/tests/avocado/replay_kernel.py index a18610542e..f7ccfd2462 100644 --- a/tests/avocado/replay_kernel.py +++ b/tests/avocado/replay_kernel.py @@ -98,6 +98,7 @@ class ReplayKernelNormal(ReplayKernelBase): self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta(self): """ :avocado: tags=arch:mips @@ -116,6 +117,7 @@ class ReplayKernelNormal(ReplayKernelBase): self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips64el_malta(self): """ This test requires the ar tool to extract "data.tar.gz" from @@ -431,6 +433,7 @@ class ReplayKernelSlow(ReplayKernelBase): # making it very slow. timeout = 180 + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta_cpio(self): """ :avocado: tags=arch:mips @@ -460,6 +463,7 @@ class ReplayKernelSlow(ReplayKernelBase): self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5, args=('-initrd', initrd_path)) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') @skipUnless(os.getenv('AVOCADO_ALLOW_UNTRUSTED_CODE'), 'untrusted code') def test_mips64el_malta_5KEc_cpio(self): """ @@ -502,6 +506,7 @@ class ReplayKernelSlow(ReplayKernelBase): console_pattern = 'Kernel command line: %s' % kernel_command_line self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta32el_nanomips_4k(self): """ :avocado: tags=arch:mipsel @@ -516,6 +521,7 @@ class ReplayKernelSlow(ReplayKernelBase): kernel_path_xz = self.fetch_asset(kernel_url, asset_hash=kernel_hash) self.do_test_mips_malta32el_nanomips(kernel_path_xz) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta32el_nanomips_16k_up(self): """ :avocado: tags=arch:mipsel @@ -530,6 +536,7 @@ class ReplayKernelSlow(ReplayKernelBase): kernel_path_xz = self.fetch_asset(kernel_url, asset_hash=kernel_hash) self.do_test_mips_malta32el_nanomips(kernel_path_xz) + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips_malta32el_nanomips_64k_dbg(self): """ :avocado: tags=arch:mipsel diff --git a/tests/avocado/tuxrun_baselines.py b/tests/avocado/tuxrun_baselines.py index e12250eabb..610b7e2bfa 100644 --- a/tests/avocado/tuxrun_baselines.py +++ b/tests/avocado/tuxrun_baselines.py @@ -352,6 +352,7 @@ class TuxRunBaselineTest(QemuSystemTest): self.common_tuxrun(csums=sums, drive="virtio-blk-pci") + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips32(self): """ :avocado: tags=arch:mips @@ -370,6 +371,7 @@ class TuxRunBaselineTest(QemuSystemTest): self.common_tuxrun(csums=sums, drive="driver=ide-hd,bus=ide.0,unit=0") + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips32el(self): """ :avocado: tags=arch:mipsel @@ -387,6 +389,7 @@ class TuxRunBaselineTest(QemuSystemTest): self.common_tuxrun(csums=sums, drive="driver=ide-hd,bus=ide.0,unit=0") + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips64(self): """ :avocado: tags=arch:mips64 @@ -404,6 +407,7 @@ class TuxRunBaselineTest(QemuSystemTest): self.common_tuxrun(csums=sums, drive="driver=ide-hd,bus=ide.0,unit=0") + @skip('https://gitlab.com/qemu-project/qemu/-/issues/1884') def test_mips64el(self): """ :avocado: tags=arch:mips64el