From 421a22ef8ec19bc6c1859a905142faeeaa7c35b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Mon, 3 Jun 2024 18:53:17 +0100 Subject: [PATCH 01/12] ci: remove centos-steam-8 customer runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This broke since eef0bae3a7 (migration: Remove block migration) but even after that was addressed it still fails to complete. As it will shortly be EOL lets to remove the runner definition and the related ansible setup bits. We still have centos9 docker images build and test. Reviewed-by: Richard Henderson Signed-off-by: Alex Bennée Message-Id: <20240603175328.3823123-2-alex.bennee@linaro.org> --- .gitlab-ci.d/custom-runners.yml | 1 - .../custom-runners/centos-stream-8-x86_64.yml | 24 --- docs/devel/ci-jobs.rst.inc | 7 - .../org.centos/stream/8/build-environment.yml | 82 -------- .../ci/org.centos/stream/8/x86_64/configure | 198 ------------------ .../org.centos/stream/8/x86_64/test-avocado | 65 ------ scripts/ci/org.centos/stream/README | 17 -- 7 files changed, 394 deletions(-) delete mode 100644 .gitlab-ci.d/custom-runners/centos-stream-8-x86_64.yml delete mode 100644 scripts/ci/org.centos/stream/8/build-environment.yml delete mode 100755 scripts/ci/org.centos/stream/8/x86_64/configure delete mode 100755 scripts/ci/org.centos/stream/8/x86_64/test-avocado delete mode 100644 scripts/ci/org.centos/stream/README diff --git a/.gitlab-ci.d/custom-runners.yml b/.gitlab-ci.d/custom-runners.yml index 29e52df283..1aa3c60efe 100644 --- a/.gitlab-ci.d/custom-runners.yml +++ b/.gitlab-ci.d/custom-runners.yml @@ -32,4 +32,3 @@ include: - local: '/.gitlab-ci.d/custom-runners/ubuntu-22.04-s390x.yml' - local: '/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml' - local: '/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch32.yml' - - local: '/.gitlab-ci.d/custom-runners/centos-stream-8-x86_64.yml' diff --git a/.gitlab-ci.d/custom-runners/centos-stream-8-x86_64.yml b/.gitlab-ci.d/custom-runners/centos-stream-8-x86_64.yml deleted file mode 100644 index 367424db78..0000000000 --- a/.gitlab-ci.d/custom-runners/centos-stream-8-x86_64.yml +++ /dev/null @@ -1,24 +0,0 @@ -# All centos-stream-8 jobs should run successfully in an environment -# setup by the scripts/ci/setup/stream/8/build-environment.yml task -# "Installation of extra packages to build QEMU" - -centos-stream-8-x86_64: - extends: .custom_runner_template - allow_failure: true - needs: [] - stage: build - tags: - - centos_stream_8 - - x86_64 - rules: - - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/' - - if: "$CENTOS_STREAM_8_x86_64_RUNNER_AVAILABLE" - before_script: - - JOBS=$(expr $(nproc) + 1) - script: - - mkdir build - - cd build - - ../scripts/ci/org.centos/stream/8/x86_64/configure - || { cat config.log meson-logs/meson-log.txt; exit 1; } - - make -j"$JOBS" - - make NINJA=":" check check-avocado diff --git a/docs/devel/ci-jobs.rst.inc b/docs/devel/ci-jobs.rst.inc index be06322279..3756bbe355 100644 --- a/docs/devel/ci-jobs.rst.inc +++ b/docs/devel/ci-jobs.rst.inc @@ -182,13 +182,6 @@ If you've got access to an IBM Z host that can be used as a gitlab-CI runner, you can set this variable to enable the tests that require this kind of host. The runner should be tagged with "s390x". -CENTOS_STREAM_8_x86_64_RUNNER_AVAILABLE -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you've got access to a CentOS Stream 8 x86_64 host that can be -used as a gitlab-CI runner, you can set this variable to enable the -tests that require this kind of host. The runner should be tagged with -both "centos_stream_8" and "x86_64". - CCACHE_DISABLE ~~~~~~~~~~~~~~ The jobs are configured to use "ccache" by default since this typically diff --git a/scripts/ci/org.centos/stream/8/build-environment.yml b/scripts/ci/org.centos/stream/8/build-environment.yml deleted file mode 100644 index 1ead77e2cb..0000000000 --- a/scripts/ci/org.centos/stream/8/build-environment.yml +++ /dev/null @@ -1,82 +0,0 @@ ---- -- name: Installation of extra packages to build QEMU - hosts: all - tasks: - - name: Extra check for CentOS Stream 8 - lineinfile: - path: /etc/redhat-release - line: CentOS Stream release 8 - state: present - check_mode: yes - register: centos_stream_8 - - - name: Enable EPEL repo on CentOS Stream 8 - dnf: - name: - - epel-release - state: present - when: - - centos_stream_8 - - - name: Enable PowerTools repo on CentOS Stream 8 - ini_file: - path: /etc/yum.repos.d/CentOS-Stream-PowerTools.repo - section: powertools - option: enabled - value: "1" - when: - - centos_stream_8 - - - name: Install basic packages to build QEMU on CentOS Stream 8 - dnf: - name: - - bzip2 - - bzip2-devel - - capstone-devel - - dbus-daemon - - device-mapper-multipath-devel - - diffutils - - gcc - - gcc-c++ - - genisoimage - - gettext - - git - - glib2-devel - - glusterfs-api-devel - - gnutls-devel - - libaio-devel - - libcap-ng-devel - - libcurl-devel - - libepoxy-devel - - libfdt-devel - - libgcrypt-devel - - libiscsi-devel - - libpmem-devel - - librados-devel - - librbd-devel - - libseccomp-devel - - libslirp-devel - - libssh-devel - - libxkbcommon-devel - - lzo-devel - - make - - mesa-libEGL-devel - - nettle-devel - - ninja-build - - nmap-ncat - - numactl-devel - - pixman-devel - - python38 - - python3-sphinx - - rdma-core-devel - - redhat-rpm-config - - snappy-devel - - spice-glib-devel - - spice-server-devel - - systemd-devel - - systemtap-sdt-devel - - tar - - zlib-devel - state: present - when: - - centos_stream_8 diff --git a/scripts/ci/org.centos/stream/8/x86_64/configure b/scripts/ci/org.centos/stream/8/x86_64/configure deleted file mode 100755 index 868db665f6..0000000000 --- a/scripts/ci/org.centos/stream/8/x86_64/configure +++ /dev/null @@ -1,198 +0,0 @@ -#!/bin/sh -e -# -# Configuration for QEMU based on CentOS Stream 8 x86_64 builds -# -# The "configure" command line is based on: -# -# https://git.centos.org/rpms/qemu-kvm/blob/c8s-stream-rhel/f/SPECS/qemu-kvm.spec -# -# But, because the SPEC file contains a number of conditionals and -# variable and expansions only available at RPM build time, this version -# was initially generated from an actual RPM build on an x86_64 platform. -# -# From that initial version, options that are required or are a -# consequence of non-upstream patches have been adapted. One example -# is "--without-default-devices" which is *not* present here, given -# that patches adding downstream specific devices are not available. -# -../configure \ ---python=/usr/bin/python3.8 \ ---prefix="/usr" \ ---libdir="/usr/lib64" \ ---datadir="/usr/share" \ ---sysconfdir="/etc" \ ---interp-prefix=/usr/qemu-%M \ ---localstatedir="/var" \ ---docdir="/usr/share/doc" \ ---libexecdir="/usr/libexec" \ ---extra-ldflags="-Wl,--build-id -Wl,-z,relro -Wl,-z,now" \ ---extra-cflags="-O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions -fstack-protector-strong -grecord-gcc-switches -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 -mtune=generic -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection" \ ---with-suffix="qemu-kvm" \ ---firmwarepath=/usr/share/qemu-firmware \ ---target-list="x86_64-softmmu" \ ---block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \ ---audio-drv-list="" \ ---block-drv-ro-whitelist="vmdk,vhdx,vpc,https,ssh" \ ---with-coroutine=ucontext \ ---tls-priority=@QEMU,SYSTEM \ ---disable-af-xdp \ ---disable-attr \ ---disable-auth-pam \ ---disable-avx2 \ ---disable-avx512f \ ---disable-bochs \ ---disable-bpf \ ---disable-brlapi \ ---disable-bsd-user \ ---disable-bzip2 \ ---disable-cap-ng \ ---disable-capstone \ ---disable-cfi \ ---disable-cfi-debug \ ---disable-cloop \ ---disable-cocoa \ ---disable-coroutine-pool \ ---disable-crypto-afalg \ ---disable-curl \ ---disable-curses \ ---disable-debug-info \ ---disable-debug-mutex \ ---disable-debug-tcg \ ---disable-dmg \ ---disable-docs \ ---disable-fuse \ ---disable-fuse-lseek \ ---disable-gcrypt \ ---disable-gio \ ---disable-glusterfs \ ---disable-gnutls \ ---disable-gtk \ ---disable-guest-agent \ ---disable-guest-agent-msi \ ---disable-hvf \ ---disable-iconv \ ---disable-kvm \ ---disable-libdaxctl \ ---disable-libiscsi \ ---disable-libnfs \ ---disable-libpmem \ ---disable-libssh \ ---disable-libudev \ ---disable-libusb \ ---disable-linux-aio \ ---disable-linux-io-uring \ ---disable-linux-user \ ---disable-live-block-migration \ ---disable-lto \ ---disable-lzfse \ ---disable-lzo \ ---disable-malloc-trim \ ---disable-membarrier \ ---disable-modules \ ---disable-module-upgrades \ ---disable-mpath \ ---disable-multiprocess \ ---disable-netmap \ ---disable-nettle \ ---disable-numa \ ---disable-nvmm \ ---disable-opengl \ ---disable-parallels \ ---disable-pie \ ---disable-qcow1 \ ---disable-qed \ ---disable-qom-cast-debug \ ---disable-rbd \ ---disable-rdma \ ---disable-replication \ ---disable-rng-none \ ---disable-safe-stack \ ---disable-sanitizers \ ---disable-sdl \ ---disable-sdl-image \ ---disable-seccomp \ ---disable-slirp-smbd \ ---disable-smartcard \ ---disable-snappy \ ---disable-sparse \ ---disable-spice \ ---disable-strip \ ---disable-system \ ---disable-tcg \ ---disable-tools \ ---disable-tpm \ ---disable-u2f \ ---disable-usb-redir \ ---disable-user \ ---disable-vde \ ---disable-vdi \ ---disable-vhost-crypto \ ---disable-vhost-kernel \ ---disable-vhost-net \ ---disable-vhost-user \ ---disable-vhost-user-blk-server \ ---disable-vhost-vdpa \ ---disable-virglrenderer \ ---disable-virtfs \ ---disable-vnc \ ---disable-vnc-jpeg \ ---disable-png \ ---disable-vnc-sasl \ ---disable-vte \ ---disable-vvfat \ ---disable-werror \ ---disable-whpx \ ---disable-xen \ ---disable-xen-pci-passthrough \ ---disable-xkbcommon \ ---disable-zstd \ ---enable-attr \ ---enable-avx2 \ ---enable-cap-ng \ ---enable-capstone \ ---enable-coroutine-pool \ ---enable-curl \ ---enable-debug-info \ ---enable-docs \ ---enable-fdt \ ---enable-gcrypt \ ---enable-glusterfs \ ---enable-gnutls \ ---enable-guest-agent \ ---enable-iconv \ ---enable-kvm \ ---enable-libiscsi \ ---enable-libpmem \ ---enable-libssh \ ---enable-libusb \ ---enable-libudev \ ---enable-linux-aio \ ---enable-lzo \ ---enable-malloc-trim \ ---enable-modules \ ---enable-mpath \ ---enable-numa \ ---enable-opengl \ ---enable-pie \ ---enable-rbd \ ---enable-rdma \ ---enable-seccomp \ ---enable-snappy \ ---enable-smartcard \ ---enable-spice \ ---enable-system \ ---enable-tcg \ ---enable-tools \ ---enable-tpm \ ---enable-trace-backends=dtrace \ ---enable-usb-redir \ ---enable-vhost-kernel \ ---enable-vhost-net \ ---enable-vhost-user \ ---enable-vhost-user-blk-server \ ---enable-vhost-vdpa \ ---enable-vnc \ ---enable-png \ ---enable-vnc-sasl \ ---enable-werror \ ---enable-xkbcommon diff --git a/scripts/ci/org.centos/stream/8/x86_64/test-avocado b/scripts/ci/org.centos/stream/8/x86_64/test-avocado deleted file mode 100755 index 73e7a1a312..0000000000 --- a/scripts/ci/org.centos/stream/8/x86_64/test-avocado +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/sh -e -# -# Runs a previously vetted list of tests, either marked explicitly for -# KVM and x86_64, or tests that are generic enough to be valid for all -# targets. Such a test list can be generated with: -# -# ./pyvenv/bin/avocado list --filter-by-tags-include-empty \ -# --filter-by-tags-include-empty-key -t accel:kvm,arch:x86_64 \ -# tests/avocado/ -# -# This is almost the complete list of avocado based tests available at -# the time this was compile, with the following exceptions: -# -# * Require machine type "x-remote": -# - tests/avocado/multiprocess.py:Multiprocess.test_multiprocess_x86_64 -# -# * Requires display type "egl-headless": -# - tests/avocado/virtio-gpu.py:VirtioGPUx86.test_virtio_vga_virgl -# - tests/avocado/virtio-gpu.py:VirtioGPUx86.test_vhost_user_vga_virgl -# -# * Test is marked (unconditionally) to be skipped: -# - tests/avocado/virtio_check_params.py:VirtioMaxSegSettingsCheck.test_machine_types -# -make get-vm-images -./pyvenv/bin/avocado run \ - --job-results-dir=tests/results/ \ - tests/avocado/boot_linux.py:BootLinuxX8664.test_pc_i440fx_kvm \ - tests/avocado/boot_linux.py:BootLinuxX8664.test_pc_q35_kvm \ - tests/avocado/boot_linux_console.py:BootLinuxConsole.test_x86_64_pc \ - tests/avocado/cpu_queries.py:QueryCPUModelExpansion.test \ - tests/avocado/empty_cpu_model.py:EmptyCPUModel.test \ - tests/avocado/hotplug_cpu.py:HotPlugCPU.test \ - tests/avocado/netdev-ethtool.py:NetDevEthtool.test_igb \ - tests/avocado/netdev-ethtool.py:NetDevEthtool.test_igb_nomsi \ - tests/avocado/info_usernet.py:InfoUsernet.test_hostfwd \ - tests/avocado/intel_iommu.py:IntelIOMMU.test_intel_iommu \ - tests/avocado/intel_iommu.py:IntelIOMMU.test_intel_iommu_pt \ - tests/avocado/intel_iommu.py:IntelIOMMU.test_intel_iommu_strict \ - tests/avocado/intel_iommu.py:IntelIOMMU.test_intel_iommu_strict_cm \ - tests/avocado/linux_initrd.py:LinuxInitrd.test_with_2gib_file_should_exit_error_msg_with_linux_v3_6 \ - tests/avocado/linux_initrd.py:LinuxInitrd.test_with_2gib_file_should_work_with_linux_v4_16 \ - tests/avocado/migration.py:Migration.test_migration_with_exec \ - tests/avocado/migration.py:Migration.test_migration_with_tcp_localhost \ - tests/avocado/migration.py:Migration.test_migration_with_unix \ - tests/avocado/pc_cpu_hotplug_props.py:OmittedCPUProps.test_no_die_id \ - tests/avocado/replay_kernel.py:ReplayKernelNormal.test_x86_64_pc \ - tests/avocado/reverse_debugging.py:ReverseDebugging_X86_64.test_x86_64_pc \ - tests/avocado/version.py:Version.test_qmp_human_info_version \ - tests/avocado/virtio_version.py:VirtioVersionCheck.test_conventional_devs \ - tests/avocado/virtio_version.py:VirtioVersionCheck.test_modern_only_devs \ - tests/avocado/vnc.py:Vnc.test_change_password \ - tests/avocado/vnc.py:Vnc.test_change_password_requires_a_password \ - tests/avocado/vnc.py:Vnc.test_no_vnc \ - tests/avocado/vnc.py:Vnc.test_no_vnc_change_password \ - tests/avocado/x86_cpu_model_versions.py:CascadelakeArchCapabilities.test_4_0 \ - tests/avocado/x86_cpu_model_versions.py:CascadelakeArchCapabilities.test_4_1 \ - tests/avocado/x86_cpu_model_versions.py:CascadelakeArchCapabilities.test_set_4_0 \ - tests/avocado/x86_cpu_model_versions.py:CascadelakeArchCapabilities.test_unset_4_1 \ - tests/avocado/x86_cpu_model_versions.py:CascadelakeArchCapabilities.test_v1_4_0 \ - tests/avocado/x86_cpu_model_versions.py:CascadelakeArchCapabilities.test_v1_set_4_0 \ - tests/avocado/x86_cpu_model_versions.py:CascadelakeArchCapabilities.test_v2_4_0 \ - tests/avocado/x86_cpu_model_versions.py:CascadelakeArchCapabilities.test_v2_unset_4_1 \ - tests/avocado/x86_cpu_model_versions.py:X86CPUModelAliases.test_4_0_alias_compatibility \ - tests/avocado/x86_cpu_model_versions.py:X86CPUModelAliases.test_4_1_alias \ - tests/avocado/x86_cpu_model_versions.py:X86CPUModelAliases.test_none_alias diff --git a/scripts/ci/org.centos/stream/README b/scripts/ci/org.centos/stream/README deleted file mode 100644 index e3eadfe3ea..0000000000 --- a/scripts/ci/org.centos/stream/README +++ /dev/null @@ -1,17 +0,0 @@ -This directory contains scripts for generating a build of QEMU that -closely matches the CentOS Stream[1] builds of the qemu-kvm package. - -To have the environment ready to configure, build QEMU and run tests, -please start with a CentOS Stream machine and: - - * apply the generic "build-environment.yml" playbook located at - scripts/ci/setup - - * apply the "build-environment.yml" in the directory following the - CentOS Stream version (such as "8"). - -This currently only covers CentOS Stream 8 environments and -packages[2]. - -[1] https://www.centos.org/centos-stream/ -[2] https://git.centos.org/rpms/qemu-kvm/commits/c8s-stream-rhel From cc1d2e04d516da0e1c2e4e99aedf86c5688bd845 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Mon, 3 Jun 2024 18:53:18 +0100 Subject: [PATCH 02/12] docs/devel: update references to centos to non-versioned container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit >From the website: "After May 31, 2024, CentOS Stream 8 will be archived and no further updates will be provided." We have updated a few bits but there are still references that need fixing. Rather than bump I've replaced them with references to the Debian image so we don't have to bump at the next update. Reviewed-by: Richard Henderson Reviewed-by: Thomas Huth Signed-off-by: Alex Bennée Message-Id: <20240603175328.3823123-3-alex.bennee@linaro.org> --- docs/devel/testing.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/devel/testing.rst b/docs/devel/testing.rst index fa28e3ecb2..23d3f44f52 100644 --- a/docs/devel/testing.rst +++ b/docs/devel/testing.rst @@ -387,9 +387,9 @@ make target): .. code:: - make docker-test-build@centos8 + make docker-test-build@debian -This will create a container instance using the ``centos8`` image (the image +This will create a container instance using the ``debian`` image (the image is downloaded and initialized automatically), in which the ``test-build`` job is executed. @@ -410,8 +410,8 @@ locally by using the ``NOCACHE`` build option: Images ~~~~~~ -Along with many other images, the ``centos8`` image is defined in a Dockerfile -in ``tests/docker/dockerfiles/``, called ``centos8.docker``. ``make docker-help`` +Along with many other images, the ``debian`` image is defined in a Dockerfile +in ``tests/docker/dockerfiles/``, called ``debian.docker``. ``make docker-help`` command will list all the available images. A ``.pre`` script can be added beside the ``.docker`` file, which will be From 5ed4e5a15ccf13c633c8b664097bc6f2d61d1109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Mon, 3 Jun 2024 18:53:19 +0100 Subject: [PATCH 03/12] tests/vm: update centos.aarch64 image to 9 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As Centos Stream 8 goes out of support we need to update. To do this powertools is replaced by crb and we don't over specify the python3 we want. Reviewed-by: Richard Henderson Reviewed-by: Thomas Huth Signed-off-by: Alex Bennée Message-Id: <20240603175328.3823123-4-alex.bennee@linaro.org> --- tests/vm/centos.aarch64 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/vm/centos.aarch64 b/tests/vm/centos.aarch64 index 3f58de1e64..fcf9e08c87 100755 --- a/tests/vm/centos.aarch64 +++ b/tests/vm/centos.aarch64 @@ -25,9 +25,9 @@ DEFAULT_CONFIG = { 'cpu' : "max", 'machine' : "virt,gic-version=max", 'install_cmds' : ( - "dnf config-manager --set-enabled powertools, " + "dnf config-manager --enable crb, " "dnf config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo, " - "dnf install -y make ninja-build git python38 gcc gcc-c++ flex bison "\ + "dnf install -y make ninja-build git python3 gcc gcc-c++ flex bison "\ "glib2-devel pixman-devel zlib-devel docker-ce.aarch64, " "systemctl enable docker, " ), @@ -38,10 +38,10 @@ DEFAULT_CONFIG = { class CentosAarch64VM(basevm.BaseVM): - name = "centos8.aarch64" + name = "centos9.aarch64" arch = "aarch64" - image_name = "CentOS-Stream-GenericCloud-8-20220125.1.aarch64.qcow2" - image_link = "https://cloud.centos.org/centos/8-stream/aarch64/images/" + image_name = "CentOS-Stream-GenericCloud-9-20230501.0.aarch64.qcow2" + image_link = "https://cloud.centos.org/centos/9-stream/aarch64/images/" image_link += image_name BUILD_SCRIPT = """ set -e; From 0f73539676719605e618a0d23326fdc85230963f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Mon, 3 Jun 2024 18:53:20 +0100 Subject: [PATCH 04/12] tests/vm: remove plain centos image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This isn't really used and we have lighter weight docker containers for testing this stuff directly. Reviewed-by: Thomas Huth Reviewed-by: Richard Henderson Signed-off-by: Alex Bennée Message-Id: <20240603175328.3823123-5-alex.bennee@linaro.org> --- tests/vm/Makefile.include | 1 - tests/vm/centos | 51 --------------------------------------- 2 files changed, 52 deletions(-) delete mode 100755 tests/vm/centos diff --git a/tests/vm/Makefile.include b/tests/vm/Makefile.include index ac56824a87..13ed80f72d 100644 --- a/tests/vm/Makefile.include +++ b/tests/vm/Makefile.include @@ -45,7 +45,6 @@ vm-help vm-test: @echo " vm-build-netbsd - Build QEMU in NetBSD VM" @echo " vm-build-openbsd - Build QEMU in OpenBSD VM" ifneq ($(GENISOIMAGE),) - @echo " vm-build-centos - Build QEMU in CentOS VM, with Docker" ifneq ($(EFI_AARCH64),) @echo " vm-build-ubuntu.aarch64 - Build QEMU in ubuntu aarch64 VM" @echo " vm-build-centos.aarch64 - Build QEMU in CentOS aarch64 VM" diff --git a/tests/vm/centos b/tests/vm/centos deleted file mode 100755 index d25c8f8b5b..0000000000 --- a/tests/vm/centos +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 -# -# CentOS 8 Stream image -# -# Copyright 2018, 2022 Red Hat Inc. -# -# Authors: -# Fam Zheng -# -# This code is licensed under the GPL version 2 or later. See -# the COPYING file in the top-level directory. -# - -import os -import sys -import subprocess -import basevm -import time - -class CentosVM(basevm.BaseVM): - name = "centos" - arch = "x86_64" - BUILD_SCRIPT = """ - set -e; - cd $(mktemp -d); - export SRC_ARCHIVE=/dev/vdb; - sudo chmod a+r $SRC_ARCHIVE; - tar -xf $SRC_ARCHIVE; - make docker-test-block@centos9 {verbose} J={jobs} NETWORK=1; - make docker-test-quick@centos9 {verbose} J={jobs} NETWORK=1; - """ - - def build_image(self, img): - cimg = self._download_with_cache("https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20220125.1.x86_64.qcow2") - img_tmp = img + ".tmp" - subprocess.check_call(['cp', '-f', cimg, img_tmp]) - self.exec_qemu_img("resize", img_tmp, "50G") - self.boot(img_tmp, extra_args = ["-cdrom", self.gen_cloud_init_iso()]) - self.wait_ssh() - self.ssh_root_check("touch /etc/cloud/cloud-init.disabled") - self.ssh_root_check("dnf update -y") - self.ssh_root_check("dnf install -y dnf-plugins-core") - self.ssh_root_check("dnf config-manager --set-enabled powertools") - self.ssh_root_check("dnf install -y podman make ninja-build git python3") - self.ssh_root("poweroff") - self.wait() - os.rename(img_tmp, img) - return 0 - -if __name__ == "__main__": - sys.exit(basevm.main(CentosVM)) From 053d5042ad8df5c4670bee61e175f0dc8046ee6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Mon, 3 Jun 2024 18:53:21 +0100 Subject: [PATCH 05/12] scripts/ci: remove CentOS bits from common build-environment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Although I've just removed the CentOS specific build-environment its probably a bit too confusing to have multiple distros mixed up in one place. Prior to moving clean-up what will be just for ubuntu. Reviewed-by: Richard Henderson Signed-off-by: Alex Bennée Message-Id: <20240603175328.3823123-6-alex.bennee@linaro.org> --- scripts/ci/setup/build-environment.yml | 98 -------------------------- 1 file changed, 98 deletions(-) diff --git a/scripts/ci/setup/build-environment.yml b/scripts/ci/setup/build-environment.yml index de0d866a1e..e82097b465 100644 --- a/scripts/ci/setup/build-environment.yml +++ b/scripts/ci/setup/build-environment.yml @@ -184,101 +184,3 @@ - ansible_facts['distribution_version'] == '22.04' - ansible_facts['architecture'] == 'aarch64' - - name: Enable EPEL repo on EL8 - dnf: - name: - - epel-release - state: present - when: - - ansible_facts['distribution_file_variety'] in ['RedHat', 'CentOS'] - - ansible_facts['distribution_major_version'] == '8' - - - name: Enable PowerTools repo on CentOS 8 - ini_file: - path: /etc/yum.repos.d/CentOS-Stream-PowerTools.repo - section: powertools - option: enabled - value: "1" - when: - - ansible_facts['distribution_file_variety'] == 'CentOS' - - ansible_facts['distribution_major_version'] == '8' - - - name: Install basic packages to build QEMU on EL8 - dnf: - # This list of packages start with tests/docker/dockerfiles/centos8.docker - # but only include files that are common to all distro variants and present - # in the standard repos (no add-ons) - name: - - bzip2 - - bzip2-devel - - capstone-devel - - dbus-daemon - - device-mapper-multipath-devel - - diffutils - - gcc - - gcc-c++ - - genisoimage - - gettext - - git - - glib2-devel - - glusterfs-api-devel - - gnutls-devel - - libaio-devel - - libcap-ng-devel - - libcurl-devel - - libepoxy-devel - - libfdt-devel - - libgcrypt-devel - - libiscsi-devel - - libpmem-devel - - librados-devel - - librbd-devel - - libseccomp-devel - - libssh-devel - - libxkbcommon-devel - - lzo-devel - - make - - mesa-libEGL-devel - - nettle-devel - - ninja-build - - nmap-ncat - - numactl-devel - - pixman-devel - - python38 - - python3-sphinx - - rdma-core-devel - - redhat-rpm-config - - snappy-devel - - spice-glib-devel - - systemd-devel - - systemtap-sdt-devel - - tar - - zlib-devel - state: present - when: - - ansible_facts['distribution_file_variety'] in ['RedHat', 'CentOS'] - - ansible_facts['distribution_version'] == '8' - - - name: Install packages only available on x86 and aarch64 - dnf: - # Spice server not available in ppc64le - name: - - spice-server - - spice-server-devel - state: present - when: - - ansible_facts['distribution_file_variety'] in ['RedHat', 'CentOS'] - - ansible_facts['distribution_version'] == '8' - - ansible_facts['architecture'] == 'aarch64' or ansible_facts['architecture'] == 'x86_64' - - - name: Check whether the Python runtime version is managed by alternatives - stat: - path: /etc/alternatives/python3 - register: python3 - - - name: Set default Python runtime to 3.8 on EL8 - command: alternatives --set python3 /usr/bin/python3.8 - when: - - ansible_facts['distribution_file_variety'] in ['RedHat', 'CentOS'] - - ansible_facts['distribution_version'] == '8' - - python3.stat.islnk and python3.stat.lnk_target != '/usr/bin/python3.8' From 0eb7fadcfdaca701105480f2215bd3e38e40b3da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Mon, 3 Jun 2024 18:53:22 +0100 Subject: [PATCH 06/12] docs/ci: clean-up references for consistency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document we have split up build-environment by distro and update the references that exist in the code base to be correct. Reviewed-by: Richard Henderson Signed-off-by: Alex Bennée Message-Id: <20240603175328.3823123-7-alex.bennee@linaro.org> --- .../custom-runners/ubuntu-22.04-aarch32.yml | 2 +- .../custom-runners/ubuntu-22.04-aarch64.yml | 2 +- .gitlab-ci.d/custom-runners/ubuntu-22.04-s390x.yml | 2 +- docs/devel/ci-runners.rst.inc | 13 ++++++------- scripts/ci/setup/{ => ubuntu}/build-environment.yml | 0 5 files changed, 9 insertions(+), 10 deletions(-) rename scripts/ci/setup/{ => ubuntu}/build-environment.yml (100%) diff --git a/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch32.yml b/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch32.yml index b8a0d75162..8727687e2b 100644 --- a/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch32.yml +++ b/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch32.yml @@ -1,5 +1,5 @@ # All ubuntu-22.04 jobs should run successfully in an environment -# setup by the scripts/ci/setup/qemu/build-environment.yml task +# setup by the scripts/ci/setup/ubuntu/build-environment.yml task # "Install basic packages to build QEMU on Ubuntu 22.04" ubuntu-22.04-aarch32-all: diff --git a/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml b/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml index 374b0956c3..263a3c2140 100644 --- a/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml +++ b/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml @@ -1,5 +1,5 @@ # All ubuntu-22.04 jobs should run successfully in an environment -# setup by the scripts/ci/setup/qemu/build-environment.yml task +# setup by the scripts/ci/setup/ubuntu/build-environment.yml task # "Install basic packages to build QEMU on Ubuntu 22.04" ubuntu-22.04-aarch64-all-linux-static: diff --git a/.gitlab-ci.d/custom-runners/ubuntu-22.04-s390x.yml b/.gitlab-ci.d/custom-runners/ubuntu-22.04-s390x.yml index 25935048e2..69ddd3e7d5 100644 --- a/.gitlab-ci.d/custom-runners/ubuntu-22.04-s390x.yml +++ b/.gitlab-ci.d/custom-runners/ubuntu-22.04-s390x.yml @@ -1,5 +1,5 @@ # All ubuntu-22.04 jobs should run successfully in an environment -# setup by the scripts/ci/setup/build-environment.yml task +# setup by the scripts/ci/setup/ubuntu/build-environment.yml task # "Install basic packages to build QEMU on Ubuntu 22.04" ubuntu-22.04-s390x-all-linux: diff --git a/docs/devel/ci-runners.rst.inc b/docs/devel/ci-runners.rst.inc index 7817001fb2..67b23d3719 100644 --- a/docs/devel/ci-runners.rst.inc +++ b/docs/devel/ci-runners.rst.inc @@ -41,19 +41,18 @@ those hosts. This would look like:: Build environment ~~~~~~~~~~~~~~~~~ -The ``scripts/ci/setup/build-environment.yml`` Ansible playbook will -set up machines with the environment needed to perform builds and run -QEMU tests. This playbook consists on the installation of various -required packages (and a general package update while at it). It -currently covers a number of different Linux distributions, but it can -be expanded to cover other systems. +The ``scripts/ci/setup/$DISTRO/build-environment.yml`` Ansible +playbook will set up machines with the environment needed to perform +builds and run QEMU tests. This playbook consists on the installation +of various required packages (and a general package update while at +it). The minimum required version of Ansible successfully tested in this playbook is 2.8.0 (a version check is embedded within the playbook itself). To run the playbook, execute:: cd scripts/ci/setup - ansible-playbook -i inventory build-environment.yml + ansible-playbook -i inventory $DISTRO/build-environment.yml Please note that most of the tasks in the playbook require superuser privileges, such as those from the ``root`` account or those obtained diff --git a/scripts/ci/setup/build-environment.yml b/scripts/ci/setup/ubuntu/build-environment.yml similarity index 100% rename from scripts/ci/setup/build-environment.yml rename to scripts/ci/setup/ubuntu/build-environment.yml From 8e3034914a51444a4e5db9b82a8cc711cc1f76ed Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Mon, 3 Jun 2024 18:53:23 +0100 Subject: [PATCH 07/12] tests/lcitool: Delete obsolete centos-stream-8.yml file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We've missed to delete this file when removing support for CentOS 8. Since the current upstream version of the lcitool removed support for CentOS 8 now, too, we have to remove the file before updating. Signed-off-by: Thomas Huth Reviewed-by: Daniel P. Berrangé Message-Id: <20240601070543.37786-2-thuth@redhat.com> Signed-off-by: Alex Bennée Message-Id: <20240603175328.3823123-8-alex.bennee@linaro.org> --- tests/lcitool/targets/centos-stream-8.yml | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 tests/lcitool/targets/centos-stream-8.yml diff --git a/tests/lcitool/targets/centos-stream-8.yml b/tests/lcitool/targets/centos-stream-8.yml deleted file mode 100644 index 6b11160fd1..0000000000 --- a/tests/lcitool/targets/centos-stream-8.yml +++ /dev/null @@ -1,3 +0,0 @@ -paths: - pip3: /usr/bin/pip3.8 - python: /usr/bin/python3.8 From 23ef50ae2d0c557483727db652a18e02b11d272f Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Mon, 3 Jun 2024 18:53:24 +0100 Subject: [PATCH 08/12] .gitlab-ci.d/buildtest.yml: Use -fno-sanitize=function in the clang-system job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The latest version of Clang (version 18 from Fedora 40) now reports bad function pointer casts as undefined behavior. Unfortunately, we are still doing this in quite a lot of places in the QEMU code and some of them are not easy to fix. So for the time being, temporarily switch this off in the failing clang-system job until all spots in the QEMU sources have been tackled. Signed-off-by: Thomas Huth Reviewed-by: Daniel P. Berrangé Message-Id: <20240601070543.37786-4-thuth@redhat.com> Signed-off-by: Alex Bennée Message-Id: <20240603175328.3823123-9-alex.bennee@linaro.org> --- .gitlab-ci.d/buildtest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.d/buildtest.yml b/.gitlab-ci.d/buildtest.yml index 91c57efded..0eec570310 100644 --- a/.gitlab-ci.d/buildtest.yml +++ b/.gitlab-ci.d/buildtest.yml @@ -432,6 +432,7 @@ clang-system: IMAGE: fedora CONFIGURE_ARGS: --cc=clang --cxx=clang++ --extra-cflags=-fsanitize=undefined --extra-cflags=-fno-sanitize-recover=undefined + --extra-cflags=-fno-sanitize=function TARGETS: alpha-softmmu arm-softmmu m68k-softmmu mips64-softmmu s390x-softmmu MAKE_CHECK_ARGS: check-qtest check-tcg From 06f3330bb0b04607c5b16cdcb516a76e5f3f7dc6 Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Mon, 3 Jun 2024 18:53:25 +0100 Subject: [PATCH 09/12] tests/lcitool: Bump to latest libvirt-ci and update Fedora and Alpine version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update to the latest version of lcitool. It dropped support for Fedora 38 and Alpine 3.18, so we have to update these to newer versions here, too. Python 3.12 dropped the "imp" module which we still need for running Avocado. Fortunately Fedora 40 still ships with a work-around package that we can use until somebody updates our Avocado to a newer version. Signed-off-by: Thomas Huth Message-Id: <20240601070543.37786-3-thuth@redhat.com> [AJB: regen on rebase] Reviewed-by: Daniel P. Berrangé Signed-off-by: Alex Bennée Message-Id: <20240603175328.3823123-10-alex.bennee@linaro.org> --- tests/docker/dockerfiles/alpine.docker | 4 ++-- tests/docker/dockerfiles/fedora-win64-cross.docker | 5 +++-- tests/docker/dockerfiles/fedora.docker | 5 +++-- tests/lcitool/libvirt-ci | 2 +- tests/lcitool/projects/qemu.yml | 1 + tests/lcitool/refresh | 6 +++--- 6 files changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/docker/dockerfiles/alpine.docker b/tests/docker/dockerfiles/alpine.docker index 554464f31e..b079a83fe2 100644 --- a/tests/docker/dockerfiles/alpine.docker +++ b/tests/docker/dockerfiles/alpine.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all alpine-318 qemu +# $ lcitool dockerfile --layers all alpine-319 qemu # # https://gitlab.com/libvirt/libvirt-ci -FROM docker.io/library/alpine:3.18 +FROM docker.io/library/alpine:3.19 RUN apk update && \ apk upgrade && \ diff --git a/tests/docker/dockerfiles/fedora-win64-cross.docker b/tests/docker/dockerfiles/fedora-win64-cross.docker index 0f78711876..fef846d5a6 100644 --- a/tests/docker/dockerfiles/fedora-win64-cross.docker +++ b/tests/docker/dockerfiles/fedora-win64-cross.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all --cross-arch mingw64 fedora-38 qemu,qemu-win-installer +# $ lcitool dockerfile --layers all --cross-arch mingw64 fedora-40 qemu,qemu-win-installer # # https://gitlab.com/libvirt/libvirt-ci -FROM registry.fedoraproject.org/fedora:38 +FROM registry.fedoraproject.org/fedora:40 RUN dnf install -y nosync && \ printf '#!/bin/sh\n\ @@ -51,6 +51,7 @@ exec "$@"\n' > /usr/bin/nosync && \ python3-pip \ python3-sphinx \ python3-sphinx_rtd_theme \ + python3-zombie-imp \ sed \ socat \ sparse \ diff --git a/tests/docker/dockerfiles/fedora.docker b/tests/docker/dockerfiles/fedora.docker index 098c894d10..44f239c088 100644 --- a/tests/docker/dockerfiles/fedora.docker +++ b/tests/docker/dockerfiles/fedora.docker @@ -1,10 +1,10 @@ # THIS FILE WAS AUTO-GENERATED # -# $ lcitool dockerfile --layers all fedora-38 qemu +# $ lcitool dockerfile --layers all fedora-40 qemu # # https://gitlab.com/libvirt/libvirt-ci -FROM registry.fedoraproject.org/fedora:38 +FROM registry.fedoraproject.org/fedora:40 RUN dnf install -y nosync && \ printf '#!/bin/sh\n\ @@ -110,6 +110,7 @@ exec "$@"\n' > /usr/bin/nosync && \ python3-pip \ python3-sphinx \ python3-sphinx_rtd_theme \ + python3-zombie-imp \ rdma-core-devel \ sed \ snappy-devel \ diff --git a/tests/lcitool/libvirt-ci b/tests/lcitool/libvirt-ci index cec6703971..0e9490cebc 160000 --- a/tests/lcitool/libvirt-ci +++ b/tests/lcitool/libvirt-ci @@ -1 +1 @@ -Subproject commit cec67039719becbfbab866f9c23574f389cf9559 +Subproject commit 0e9490cebc726ef772b6c9e27dac32e7ae99f9b2 diff --git a/tests/lcitool/projects/qemu.yml b/tests/lcitool/projects/qemu.yml index 7511ec7ccb..070d7f4706 100644 --- a/tests/lcitool/projects/qemu.yml +++ b/tests/lcitool/projects/qemu.yml @@ -89,6 +89,7 @@ packages: - pkg-config - pulseaudio - python3 + - python3-imp - python3-numpy - python3-opencv - python3-pillow diff --git a/tests/lcitool/refresh b/tests/lcitool/refresh index 789acefb75..9d8e9c6a4a 100755 --- a/tests/lcitool/refresh +++ b/tests/lcitool/refresh @@ -124,11 +124,11 @@ try: # # Standard native builds # - generate_dockerfile("alpine", "alpine-318") + generate_dockerfile("alpine", "alpine-319") generate_dockerfile("centos9", "centos-stream-9") generate_dockerfile("debian", "debian-12", trailer="".join(debian12_extras)) - generate_dockerfile("fedora", "fedora-38") + generate_dockerfile("fedora", "fedora-40") generate_dockerfile("opensuse-leap", "opensuse-leap-15") generate_dockerfile("ubuntu2204", "ubuntu-2204") @@ -191,7 +191,7 @@ try: trailer=cross_build("s390x-linux-gnu-", "s390x-softmmu,s390x-linux-user")) - generate_dockerfile("fedora-win64-cross", "fedora-38", + generate_dockerfile("fedora-win64-cross", "fedora-40", project='qemu,qemu-win-installer', cross="mingw64", trailer=cross_build("x86_64-w64-mingw32-", From 61d1e3cbde0e6e84de4edc1b944a61f352f95d57 Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Mon, 3 Jun 2024 18:53:26 +0100 Subject: [PATCH 10/12] tests/lcitool: Install mingw-w64-tools for the Windows cross-builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Beside g++ we also need the mingw-w64-tools for properly building the code in qga/vss-win32/ , so let's install that package now, too. Signed-off-by: Thomas Huth Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Daniel P. Berrangé Message-Id: <20240601070543.37786-5-thuth@redhat.com> Signed-off-by: Alex Bennée Message-Id: <20240603175328.3823123-11-alex.bennee@linaro.org> --- tests/docker/dockerfiles/fedora-win64-cross.docker | 1 + tests/lcitool/projects/qemu-win-installer.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/docker/dockerfiles/fedora-win64-cross.docker b/tests/docker/dockerfiles/fedora-win64-cross.docker index fef846d5a6..007e1574bd 100644 --- a/tests/docker/dockerfiles/fedora-win64-cross.docker +++ b/tests/docker/dockerfiles/fedora-win64-cross.docker @@ -75,6 +75,7 @@ ENV NINJA "/usr/bin/ninja" ENV PYTHON "/usr/bin/python3" RUN nosync dnf install -y \ + mingw-w64-tools \ mingw32-nsis \ mingw64-SDL2 \ mingw64-SDL2_image \ diff --git a/tests/lcitool/projects/qemu-win-installer.yml b/tests/lcitool/projects/qemu-win-installer.yml index 86aa22297c..f3663ba030 100644 --- a/tests/lcitool/projects/qemu-win-installer.yml +++ b/tests/lcitool/projects/qemu-win-installer.yml @@ -2,3 +2,4 @@ --- packages: - g++ + - mingw-w64-tools From 1417704564a50457fcef9f4ed11afbd9e9050bc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Mon, 3 Jun 2024 18:53:27 +0100 Subject: [PATCH 11/12] tests/lcitool: generate package lists for ansible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the new ability to output YAML we can build the package list for our ansible setup scripts. We will integrate them in the next commit. Signed-off-by: Alex Bennée Message-Id: <20240603175328.3823123-12-alex.bennee@linaro.org> --- .../ci/setup/ubuntu/ubuntu-2204-aarch64.yaml | 127 ++++++++++++++++++ .../setup/ubuntu/ubuntu-2204-armhf-cross.yml | 127 ++++++++++++++++++ .../ci/setup/ubuntu/ubuntu-2204-s390x.yaml | 125 +++++++++++++++++ tests/lcitool/refresh | 16 ++- 4 files changed, 394 insertions(+), 1 deletion(-) create mode 100644 scripts/ci/setup/ubuntu/ubuntu-2204-aarch64.yaml create mode 100644 scripts/ci/setup/ubuntu/ubuntu-2204-armhf-cross.yml create mode 100644 scripts/ci/setup/ubuntu/ubuntu-2204-s390x.yaml diff --git a/scripts/ci/setup/ubuntu/ubuntu-2204-aarch64.yaml b/scripts/ci/setup/ubuntu/ubuntu-2204-aarch64.yaml new file mode 100644 index 0000000000..8d7d8725fb --- /dev/null +++ b/scripts/ci/setup/ubuntu/ubuntu-2204-aarch64.yaml @@ -0,0 +1,127 @@ +# THIS FILE WAS AUTO-GENERATED +# +# $ lcitool variables --host-arch aarch64 ubuntu-2204 qemu +# +# https://gitlab.com/libvirt/libvirt-ci + +packages: + - bash + - bc + - bison + - bsdextrautils + - bzip2 + - ca-certificates + - ccache + - clang + - dbus + - debianutils + - diffutils + - exuberant-ctags + - findutils + - flex + - gcc + - gcovr + - gettext + - git + - hostname + - libaio-dev + - libasan6 + - libasound2-dev + - libattr1-dev + - libbpf-dev + - libbrlapi-dev + - libbz2-dev + - libc6-dev + - libcacard-dev + - libcap-ng-dev + - libcapstone-dev + - libcmocka-dev + - libcurl4-gnutls-dev + - libdaxctl-dev + - libdrm-dev + - libepoxy-dev + - libfdt-dev + - libffi-dev + - libfuse3-dev + - libgbm-dev + - libgcrypt20-dev + - libglib2.0-dev + - libglusterfs-dev + - libgnutls28-dev + - libgtk-3-dev + - libibumad-dev + - libibverbs-dev + - libiscsi-dev + - libjemalloc-dev + - libjpeg-turbo8-dev + - libjson-c-dev + - liblttng-ust-dev + - liblzo2-dev + - libncursesw5-dev + - libnfs-dev + - libnuma-dev + - libpam0g-dev + - libpcre2-dev + - libpipewire-0.3-dev + - libpixman-1-dev + - libpng-dev + - libpulse-dev + - librbd-dev + - librdmacm-dev + - libsasl2-dev + - libsdl2-dev + - libsdl2-image-dev + - libseccomp-dev + - libselinux1-dev + - libslirp-dev + - libsnappy-dev + - libsndio-dev + - libspice-protocol-dev + - libspice-server-dev + - libssh-dev + - libsystemd-dev + - libtasn1-6-dev + - libubsan1 + - libudev-dev + - liburing-dev + - libusb-1.0-0-dev + - libusbredirhost-dev + - libvdeplug-dev + - libvirglrenderer-dev + - libvte-2.91-dev + - libxen-dev + - libzstd-dev + - llvm + - locales + - make + - meson + - mtools + - multipath-tools + - ncat + - nettle-dev + - ninja-build + - openssh-client + - pkgconf + - python3 + - python3-numpy + - python3-opencv + - python3-pillow + - python3-pip + - python3-sphinx + - python3-sphinx-rtd-theme + - python3-tomli + - python3-venv + - python3-yaml + - rpm2cpio + - sed + - socat + - sparse + - swtpm + - systemtap-sdt-dev + - tar + - tesseract-ocr + - tesseract-ocr-eng + - xorriso + - zlib1g-dev + - zstd + diff --git a/scripts/ci/setup/ubuntu/ubuntu-2204-armhf-cross.yml b/scripts/ci/setup/ubuntu/ubuntu-2204-armhf-cross.yml new file mode 100644 index 0000000000..0cc34cd10b --- /dev/null +++ b/scripts/ci/setup/ubuntu/ubuntu-2204-armhf-cross.yml @@ -0,0 +1,127 @@ +# THIS FILE WAS AUTO-GENERATED +# +# $ lcitool variables --cross-arch armv7l ubuntu-2204 qemu +# +# https://gitlab.com/libvirt/libvirt-ci + +packages: + - bash + - bc + - bison + - bsdextrautils + - bzip2 + - ca-certificates + - ccache + - dbus + - debianutils + - diffutils + - exuberant-ctags + - findutils + - flex + - gcc + - gcovr + - gettext + - git + - hostname + - libglib2.0-dev + - libpcre2-dev + - libsndio-dev + - libspice-protocol-dev + - llvm + - locales + - make + - meson + - mtools + - ncat + - ninja-build + - openssh-client + - pkgconf + - python3 + - python3-numpy + - python3-opencv + - python3-pillow + - python3-pip + - python3-sphinx + - python3-sphinx-rtd-theme + - python3-tomli + - python3-venv + - python3-yaml + - rpm2cpio + - sed + - socat + - sparse + - swtpm + - tar + - tesseract-ocr + - tesseract-ocr-eng + - xorriso + - zstd + - gcc-arm-linux-gnueabihf + - libaio-dev:armhf + - libasan6:armhf + - libasound2-dev:armhf + - libattr1-dev:armhf + - libbpf-dev:armhf + - libbrlapi-dev:armhf + - libbz2-dev:armhf + - libc6-dev:armhf + - libcacard-dev:armhf + - libcap-ng-dev:armhf + - libcapstone-dev:armhf + - libcmocka-dev:armhf + - libcurl4-gnutls-dev:armhf + - libdaxctl-dev:armhf + - libdrm-dev:armhf + - libepoxy-dev:armhf + - libfdt-dev:armhf + - libffi-dev:armhf + - libfuse3-dev:armhf + - libgbm-dev:armhf + - libgcrypt20-dev:armhf + - libglib2.0-dev:armhf + - libglusterfs-dev:armhf + - libgnutls28-dev:armhf + - libgtk-3-dev:armhf + - libibumad-dev:armhf + - libibverbs-dev:armhf + - libiscsi-dev:armhf + - libjemalloc-dev:armhf + - libjpeg-turbo8-dev:armhf + - libjson-c-dev:armhf + - liblttng-ust-dev:armhf + - liblzo2-dev:armhf + - libncursesw5-dev:armhf + - libnfs-dev:armhf + - libnuma-dev:armhf + - libpam0g-dev:armhf + - libpipewire-0.3-dev:armhf + - libpixman-1-dev:armhf + - libpng-dev:armhf + - libpulse-dev:armhf + - librbd-dev:armhf + - librdmacm-dev:armhf + - libsasl2-dev:armhf + - libsdl2-dev:armhf + - libsdl2-image-dev:armhf + - libseccomp-dev:armhf + - libselinux1-dev:armhf + - libslirp-dev:armhf + - libsnappy-dev:armhf + - libspice-server-dev:armhf + - libssh-dev:armhf + - libsystemd-dev:armhf + - libtasn1-6-dev:armhf + - libubsan1:armhf + - libudev-dev:armhf + - liburing-dev:armhf + - libusb-1.0-0-dev:armhf + - libusbredirhost-dev:armhf + - libvdeplug-dev:armhf + - libvirglrenderer-dev:armhf + - libvte-2.91-dev:armhf + - libxen-dev:armhf + - libzstd-dev:armhf + - nettle-dev:armhf + - systemtap-sdt-dev:armhf + - zlib1g-dev:armhf + diff --git a/scripts/ci/setup/ubuntu/ubuntu-2204-s390x.yaml b/scripts/ci/setup/ubuntu/ubuntu-2204-s390x.yaml new file mode 100644 index 0000000000..16050a5058 --- /dev/null +++ b/scripts/ci/setup/ubuntu/ubuntu-2204-s390x.yaml @@ -0,0 +1,125 @@ +# THIS FILE WAS AUTO-GENERATED +# +# $ lcitool variables --host-arch s390x ubuntu-2204 qemu +# +# https://gitlab.com/libvirt/libvirt-ci + +packages: + - bash + - bc + - bison + - bsdextrautils + - bzip2 + - ca-certificates + - ccache + - clang + - dbus + - debianutils + - diffutils + - exuberant-ctags + - findutils + - flex + - gcc + - gcovr + - gettext + - git + - hostname + - libaio-dev + - libasan6 + - libasound2-dev + - libattr1-dev + - libbpf-dev + - libbrlapi-dev + - libbz2-dev + - libc6-dev + - libcacard-dev + - libcap-ng-dev + - libcapstone-dev + - libcmocka-dev + - libcurl4-gnutls-dev + - libdaxctl-dev + - libdrm-dev + - libepoxy-dev + - libfdt-dev + - libffi-dev + - libfuse3-dev + - libgbm-dev + - libgcrypt20-dev + - libglib2.0-dev + - libglusterfs-dev + - libgnutls28-dev + - libgtk-3-dev + - libibumad-dev + - libibverbs-dev + - libiscsi-dev + - libjemalloc-dev + - libjpeg-turbo8-dev + - libjson-c-dev + - liblttng-ust-dev + - liblzo2-dev + - libncursesw5-dev + - libnfs-dev + - libnuma-dev + - libpam0g-dev + - libpcre2-dev + - libpipewire-0.3-dev + - libpixman-1-dev + - libpng-dev + - libpulse-dev + - librbd-dev + - librdmacm-dev + - libsasl2-dev + - libsdl2-dev + - libsdl2-image-dev + - libseccomp-dev + - libselinux1-dev + - libslirp-dev + - libsnappy-dev + - libsndio-dev + - libspice-protocol-dev + - libssh-dev + - libsystemd-dev + - libtasn1-6-dev + - libubsan1 + - libudev-dev + - liburing-dev + - libusb-1.0-0-dev + - libusbredirhost-dev + - libvdeplug-dev + - libvirglrenderer-dev + - libvte-2.91-dev + - libzstd-dev + - llvm + - locales + - make + - meson + - mtools + - multipath-tools + - ncat + - nettle-dev + - ninja-build + - openssh-client + - pkgconf + - python3 + - python3-numpy + - python3-opencv + - python3-pillow + - python3-pip + - python3-sphinx + - python3-sphinx-rtd-theme + - python3-tomli + - python3-venv + - python3-yaml + - rpm2cpio + - sed + - socat + - sparse + - swtpm + - systemtap-sdt-dev + - tar + - tesseract-ocr + - tesseract-ocr-eng + - xorriso + - zlib1g-dev + - zstd + diff --git a/tests/lcitool/refresh b/tests/lcitool/refresh index 9d8e9c6a4a..b25e3ac4dd 100755 --- a/tests/lcitool/refresh +++ b/tests/lcitool/refresh @@ -80,7 +80,7 @@ def generate_dockerfile(host, target, project="qemu", cross=None, trailer=None): def generate_cirrus(target, trailer=None): filename = Path(src_dir, ".gitlab-ci.d", "cirrus", target + ".vars") - cmd = lcitool_cmd + ["variables", target, "qemu"] + cmd = lcitool_cmd + ["variables", "--format", "shell", target, "qemu"] generate(filename, cmd, trailer) @@ -90,6 +90,13 @@ def generate_pkglist(vm, target): generate(filename, cmd, None) +def generate_yaml(os, target, arch, trailer=None): + filename = Path(src_dir, "scripts", "ci", "setup", os, f"{target}-{arch}.yaml") + cmd = lcitool_cmd + ["variables", "--format", "yaml", "-a", + arch, target, "qemu"] + generate(filename, cmd, trailer) + + # Netmap still needs to be manually built as it is yet to be packaged # into a distro. We also add cscope and gtags which are used in the CI # test @@ -209,6 +216,13 @@ try: # generate_pkglist("freebsd", "freebsd-13") + # + # Ansible package lists + # + generate_yaml("ubuntu", "ubuntu-2204", "aarch64") + generate_yaml("ubuntu", "ubuntu-2204", "s390x") + + sys.exit(0) except Exception as ex: print(str(ex), file=sys.stderr) From c99064d03fc574254ab098562798c937a4761161 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Mon, 3 Jun 2024 18:53:28 +0100 Subject: [PATCH 12/12] scripts/ci: drive ubuntu/build-environment.yml from lcitool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now lcitool can write the package list for us we no longer need to duplicate the information directly in build-environment. Signed-off-by: Alex Bennée Message-Id: <20240603175328.3823123-13-alex.bennee@linaro.org> --- scripts/ci/setup/ubuntu/build-environment.yml | 131 +----------------- 1 file changed, 7 insertions(+), 124 deletions(-) diff --git a/scripts/ci/setup/ubuntu/build-environment.yml b/scripts/ci/setup/ubuntu/build-environment.yml index e82097b465..edf1900b3e 100644 --- a/scripts/ci/setup/ubuntu/build-environment.yml +++ b/scripts/ci/setup/ubuntu/build-environment.yml @@ -32,135 +32,18 @@ when: - ansible_facts['distribution'] == 'Ubuntu' - # lcitool variables -f json ubuntu-2204 qemu | jq -r '.pkgs[]' | xargs -n 1 echo "-" - - name: Install basic packages to build QEMU on Ubuntu 22.04 - package: - name: - - bash - - bc - - bison - - bsdextrautils - - bzip2 - - ca-certificates - - ccache - - clang - - dbus - - debianutils - - diffutils - - exuberant-ctags - - findutils - - flex - - g++ - - gcc - - gcovr - - genisoimage - - gettext - - git - - hostname - - libaio-dev - - libasan5 - - libasound2-dev - - libattr1-dev - - libbpf-dev - - libbrlapi-dev - - libbz2-dev - - libc6-dev - - libcacard-dev - - libcap-ng-dev - - libcapstone-dev - - libcmocka-dev - - libcurl4-gnutls-dev - - libdaxctl-dev - - libdrm-dev - - libepoxy-dev - - libfdt-dev - - libffi-dev - - libgbm-dev - - libgcrypt20-dev - - libglib2.0-dev - - libglusterfs-dev - - libgnutls28-dev - - libgtk-3-dev - - libibumad-dev - - libibverbs-dev - - libiscsi-dev - - libjemalloc-dev - - libjpeg-turbo8-dev - - libjson-c-dev - - liblttng-ust-dev - - liblzo2-dev - - libncursesw5-dev - - libnfs-dev - - libnuma-dev - - libpam0g-dev - - libpcre2-dev - - libpixman-1-dev - - libpng-dev - - libpulse-dev - - librbd-dev - - librdmacm-dev - - libsasl2-dev - - libsdl2-dev - - libsdl2-image-dev - - libseccomp-dev - - libslirp-dev - - libsnappy-dev - - libspice-protocol-dev - - libssh-dev - - libsystemd-dev - - libtasn1-6-dev - - libubsan1 - - libudev-dev - - liburing-dev - - libusb-1.0-0-dev - - libusbredirhost-dev - - libvdeplug-dev - - libvirglrenderer-dev - - libvte-2.91-dev - - libxml2-dev - - libzstd-dev - - llvm - - locales - - make - - meson - - multipath-tools - - ncat - - nettle-dev - - ninja-build - - openssh-client - - pkgconf - - python3 - - python3-numpy - - python3-opencv - - python3-pillow - - python3-pip - - python3-sphinx - - python3-sphinx-rtd-theme - - python3-venv - - python3-yaml - - rpm2cpio - - sed - - sparse - - systemtap-sdt-dev - - tar - - tesseract-ocr - - tesseract-ocr-eng - - texinfo - - xfslibs-dev - - zlib1g-dev - state: present + # the package lists are updated by "make lcitool-refresh" + - name: Include package lists based on OS and architecture + include_vars: + file: "ubuntu-2204-{{ ansible_facts['architecture'] }}.yaml" when: - ansible_facts['distribution'] == 'Ubuntu' - ansible_facts['distribution_version'] == '22.04' + - ansible_facts['architecture'] == 'aarch64' or ansible_facts['architecture'] == 'x86_64' - # not all packages are available for all architectures - - name: Install additional packages to build QEMU on Ubuntu 22.04 + - name: Install packages for QEMU on Ubuntu 22.04 package: - name: - - libpmem-dev - - libspice-server-dev - - libxen-dev - state: present + name: "{{ packages }}" when: - ansible_facts['distribution'] == 'Ubuntu' - ansible_facts['distribution_version'] == '22.04'