From a4b201d1ace0d25219491ac3e14930e9539ae762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Wed, 19 May 2021 20:54:55 +0200 Subject: [PATCH 01/18] gitlab: Replace YAML anchors by extends (acceptance_test_job) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 'extends' is an alternative to using YAML anchors and is a little more flexible and readable. See: https://docs.gitlab.com/ee/ci/yaml/#extends Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Wainer dos Santos Moschetta Reviewed-by: Willian Rampazzo Message-Id: <20210519185504.2198573-2-f4bug@amsat.org> Signed-off-by: Thomas Huth --- .gitlab-ci.yml | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1c46392b2f..bfdbf9ddde 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,7 +52,8 @@ include: # Avoid recompiling by hiding ninja with NINJA=":" - make NINJA=":" $MAKE_CHECK_ARGS -.acceptance_template: &acceptance_definition +.acceptance_test_job_template: + extends: .native_test_job_template cache: key: "${CI_JOB_NAME}-cache" paths: @@ -108,14 +109,13 @@ check-system-alpine: MAKE_CHECK_ARGS: check acceptance-system-alpine: - extends: .native_test_job_template + extends: .acceptance_test_job_template needs: - job: build-system-alpine artifacts: true variables: IMAGE: alpine MAKE_CHECK_ARGS: check-acceptance - <<: *acceptance_definition build-system-ubuntu: extends: .native_build_job_template @@ -142,14 +142,13 @@ check-system-ubuntu: MAKE_CHECK_ARGS: check acceptance-system-ubuntu: - extends: .native_test_job_template + extends: .acceptance_test_job_template needs: - job: build-system-ubuntu artifacts: true variables: IMAGE: ubuntu2004 MAKE_CHECK_ARGS: check-acceptance - <<: *acceptance_definition build-system-debian: extends: .native_build_job_template @@ -176,14 +175,13 @@ check-system-debian: MAKE_CHECK_ARGS: check acceptance-system-debian: - extends: .native_test_job_template + extends: .acceptance_test_job_template needs: - job: build-system-debian artifacts: true variables: IMAGE: debian-amd64 MAKE_CHECK_ARGS: check-acceptance - <<: *acceptance_definition build-system-fedora: extends: .native_build_job_template @@ -211,14 +209,13 @@ check-system-fedora: MAKE_CHECK_ARGS: check acceptance-system-fedora: - extends: .native_test_job_template + extends: .acceptance_test_job_template needs: - job: build-system-fedora artifacts: true variables: IMAGE: fedora MAKE_CHECK_ARGS: check-acceptance - <<: *acceptance_definition build-system-centos: extends: .native_build_job_template @@ -246,14 +243,13 @@ check-system-centos: MAKE_CHECK_ARGS: check acceptance-system-centos: - extends: .native_test_job_template + extends: .acceptance_test_job_template needs: - job: build-system-centos artifacts: true variables: IMAGE: centos8 MAKE_CHECK_ARGS: check-acceptance - <<: *acceptance_definition build-system-opensuse: extends: .native_build_job_template @@ -279,14 +275,13 @@ check-system-opensuse: MAKE_CHECK_ARGS: check acceptance-system-opensuse: - extends: .native_test_job_template + extends: .acceptance_test_job_template needs: - job: build-system-opensuse artifacts: true variables: IMAGE: opensuse-leap MAKE_CHECK_ARGS: check-acceptance - <<: *acceptance_definition build-disabled: @@ -535,14 +530,13 @@ check-cfi-aarch64: MAKE_CHECK_ARGS: check acceptance-cfi-aarch64: - extends: .native_test_job_template + extends: .acceptance_test_job_template needs: - job: build-cfi-aarch64 artifacts: true variables: IMAGE: fedora MAKE_CHECK_ARGS: check-acceptance - <<: *acceptance_definition build-cfi-ppc64-s390x: extends: .native_build_job_template @@ -572,14 +566,13 @@ check-cfi-ppc64-s390x: MAKE_CHECK_ARGS: check acceptance-cfi-ppc64-s390x: - extends: .native_test_job_template + extends: .acceptance_test_job_template needs: - job: build-cfi-ppc64-s390x artifacts: true variables: IMAGE: fedora MAKE_CHECK_ARGS: check-acceptance - <<: *acceptance_definition build-cfi-x86_64: extends: .native_build_job_template @@ -609,14 +602,13 @@ check-cfi-x86_64: MAKE_CHECK_ARGS: check acceptance-cfi-x86_64: - extends: .native_test_job_template + extends: .acceptance_test_job_template needs: - job: build-cfi-x86_64 artifacts: true variables: IMAGE: fedora MAKE_CHECK_ARGS: check-acceptance - <<: *acceptance_definition tsan-build: extends: .native_build_job_template From e4af2160e17ab48f49b6c18cf39de8562b390768 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sun, 2 May 2021 16:57:26 -0700 Subject: [PATCH 02/18] gitlab: Rename ACCEL_CONFIGURE_OPTS to EXTRA_CONFIGURE_OPTS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Suggested-by: Thomas Huth Signed-off-by: Richard Henderson Reviewed-by: Willian Rampazzo Message-Id: <20210502235727.1979457-26-richard.henderson@linaro.org> Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Philippe Mathieu-Daudé --- .gitlab-ci.d/crossbuilds.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.d/crossbuilds.yml b/.gitlab-ci.d/crossbuilds.yml index e44e4b49a2..2198abb400 100644 --- a/.gitlab-ci.d/crossbuilds.yml +++ b/.gitlab-ci.d/crossbuilds.yml @@ -16,7 +16,7 @@ # # Set the $ACCEL variable to select the specific accelerator (default to # KVM), and set extra options (such disabling other accelerators) via the -# $ACCEL_CONFIGURE_OPTS variable. +# $EXTRA_CONFIGURE_OPTS variable. .cross_accel_build_job: stage: build image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest @@ -26,7 +26,7 @@ - cd build - PKG_CONFIG_PATH=$PKG_CONFIG_PATH ../configure --enable-werror --disable-docs $QEMU_CONFIGURE_OPTS - --disable-tools --enable-${ACCEL:-kvm} $ACCEL_CONFIGURE_OPTS + --disable-tools --enable-${ACCEL:-kvm} $EXTRA_CONFIGURE_OPTS - make -j$(expr $(nproc) + 1) all check-build .cross_user_build_job: @@ -174,7 +174,7 @@ cross-s390x-kvm-only: job: s390x-debian-cross-container variables: IMAGE: debian-s390x-cross - ACCEL_CONFIGURE_OPTS: --disable-tcg + EXTRA_CONFIGURE_OPTS: --disable-tcg cross-mips64el-kvm-only: extends: .cross_accel_build_job @@ -182,7 +182,7 @@ cross-mips64el-kvm-only: job: mips64el-debian-cross-container variables: IMAGE: debian-mips64el-cross - ACCEL_CONFIGURE_OPTS: --disable-tcg --target-list=mips64el-softmmu + EXTRA_CONFIGURE_OPTS: --disable-tcg --target-list=mips64el-softmmu cross-win32-system: extends: .cross_system_build_job @@ -205,7 +205,7 @@ cross-amd64-xen-only: variables: IMAGE: debian-amd64-cross ACCEL: xen - ACCEL_CONFIGURE_OPTS: --disable-tcg --disable-kvm + EXTRA_CONFIGURE_OPTS: --disable-tcg --disable-kvm cross-arm64-xen-only: extends: .cross_accel_build_job @@ -214,4 +214,4 @@ cross-arm64-xen-only: variables: IMAGE: debian-arm64-cross ACCEL: xen - ACCEL_CONFIGURE_OPTS: --disable-tcg --disable-kvm + EXTRA_CONFIGURE_OPTS: --disable-tcg --disable-kvm From 840cafced12f962e7447de4333a55b67b44030a1 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sun, 2 May 2021 16:57:27 -0700 Subject: [PATCH 03/18] gitlab: Enable cross-i386 builds of TCI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We're currently only testing TCI with a 64-bit host -- also test with a 32-bit host. Enable a selection of softmmu and user-only targets, 32-bit LE, 64-bit LE, 32-bit BE, as there are ifdefs for each. Signed-off-by: Richard Henderson Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Willian Rampazzo Acked-by: Thomas Huth Message-Id: <20210502235727.1979457-27-richard.henderson@linaro.org> Signed-off-by: Philippe Mathieu-Daudé --- .gitlab-ci.d/crossbuilds.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.d/crossbuilds.yml b/.gitlab-ci.d/crossbuilds.yml index 2198abb400..edf3a62563 100644 --- a/.gitlab-ci.d/crossbuilds.yml +++ b/.gitlab-ci.d/crossbuilds.yml @@ -27,7 +27,7 @@ - PKG_CONFIG_PATH=$PKG_CONFIG_PATH ../configure --enable-werror --disable-docs $QEMU_CONFIGURE_OPTS --disable-tools --enable-${ACCEL:-kvm} $EXTRA_CONFIGURE_OPTS - - make -j$(expr $(nproc) + 1) all check-build + - make -j$(expr $(nproc) + 1) all check-build $MAKE_CHECK_ARGS .cross_user_build_job: stage: build @@ -98,6 +98,15 @@ cross-i386-user: IMAGE: fedora-i386-cross MAKE_CHECK_ARGS: check +cross-i386-tci: + extends: .cross_accel_build_job + timeout: 60m + variables: + IMAGE: fedora-i386-cross + ACCEL: tcg-interpreter + EXTRA_CONFIGURE_OPTS: --target-list=i386-softmmu,i386-linux-user,aarch64-softmmu,aarch64-linux-user,ppc-softmmu,ppc-linux-user + MAKE_CHECK_ARGS: check check-tcg + cross-mips-system: extends: .cross_system_build_job needs: From fb3c0334dedc09994359b213df303440a8ee6edd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Wed, 19 May 2021 20:54:57 +0200 Subject: [PATCH 04/18] gitlab: Extract container job template to container-template.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the container job template to a new file (container-template.yml) to be able to reuse it without having to run all the jobs included, which are mainly useful for mainstream CI. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Willian Rampazzo Message-Id: <20210519185504.2198573-4-f4bug@amsat.org> Signed-off-by: Thomas Huth --- .gitlab-ci.d/container-template.yml | 21 +++++++++++++++++++++ .gitlab-ci.d/containers.yml | 23 ++--------------------- 2 files changed, 23 insertions(+), 21 deletions(-) create mode 100644 .gitlab-ci.d/container-template.yml diff --git a/.gitlab-ci.d/container-template.yml b/.gitlab-ci.d/container-template.yml new file mode 100644 index 0000000000..1baecd9460 --- /dev/null +++ b/.gitlab-ci.d/container-template.yml @@ -0,0 +1,21 @@ +.container_job_template: + image: docker:stable + stage: containers + services: + - docker:dind + before_script: + - export TAG="$CI_REGISTRY_IMAGE/qemu/$NAME:latest" + - export COMMON_TAG="$CI_REGISTRY/qemu-project/qemu/$NAME:latest" + - apk add python3 + - docker info + - docker login $CI_REGISTRY -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" + script: + - echo "TAG:$TAG" + - echo "COMMON_TAG:$COMMON_TAG" + - ./tests/docker/docker.py --engine docker build + -t "qemu/$NAME" -f "tests/docker/dockerfiles/$NAME.docker" + -r $CI_REGISTRY/qemu-project/qemu + - docker tag "qemu/$NAME" "$TAG" + - docker push "$TAG" + after_script: + - docker logout diff --git a/.gitlab-ci.d/containers.yml b/.gitlab-ci.d/containers.yml index 7b7ca3790d..2b06763d73 100644 --- a/.gitlab-ci.d/containers.yml +++ b/.gitlab-ci.d/containers.yml @@ -1,24 +1,5 @@ -.container_job_template: - image: docker:stable - stage: containers - services: - - docker:dind - before_script: - - export TAG="$CI_REGISTRY_IMAGE/qemu/$NAME:latest" - - export COMMON_TAG="$CI_REGISTRY/qemu-project/qemu/$NAME:latest" - - apk add python3 - - docker info - - docker login $CI_REGISTRY -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" - script: - - echo "TAG:$TAG" - - echo "COMMON_TAG:$COMMON_TAG" - - ./tests/docker/docker.py --engine docker build - -t "qemu/$NAME" -f "tests/docker/dockerfiles/$NAME.docker" - -r $CI_REGISTRY/qemu-project/qemu - - docker tag "qemu/$NAME" "$TAG" - - docker push "$TAG" - after_script: - - docker logout +include: + - local: '/.gitlab-ci.d/container-template.yml' amd64-alpine-container: extends: .container_job_template From ecac85fe49793fc4102b68f03ef18d74c4c77dee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Wed, 19 May 2021 20:54:58 +0200 Subject: [PATCH 05/18] gitlab: Extract crossbuild job templates to crossbuild-template.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the crossbuild job templates to a new file (crossbuild-template.yml) to be able to reuse them without having to run all the jobs included, which are mainly useful for mainstream CI. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Willian Rampazzo Message-Id: <20210519185504.2198573-5-f4bug@amsat.org> Signed-off-by: Thomas Huth --- .gitlab-ci.d/crossbuild-template.yml | 41 ++++++++++++++++++++++++++ .gitlab-ci.d/crossbuilds.yml | 43 ++-------------------------- 2 files changed, 43 insertions(+), 41 deletions(-) create mode 100644 .gitlab-ci.d/crossbuild-template.yml diff --git a/.gitlab-ci.d/crossbuild-template.yml b/.gitlab-ci.d/crossbuild-template.yml new file mode 100644 index 0000000000..1be541174c --- /dev/null +++ b/.gitlab-ci.d/crossbuild-template.yml @@ -0,0 +1,41 @@ +.cross_system_build_job: + stage: build + image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest + timeout: 80m + script: + - mkdir build + - cd build + - PKG_CONFIG_PATH=$PKG_CONFIG_PATH + ../configure --enable-werror --disable-docs $QEMU_CONFIGURE_OPTS + --disable-user --target-list-exclude="arm-softmmu cris-softmmu + i386-softmmu microblaze-softmmu mips-softmmu mipsel-softmmu + mips64-softmmu ppc-softmmu sh4-softmmu xtensa-softmmu" + - make -j$(expr $(nproc) + 1) all check-build $MAKE_CHECK_ARGS + +# Job to cross-build specific accelerators. +# +# Set the $ACCEL variable to select the specific accelerator (default to +# KVM), and set extra options (such disabling other accelerators) via the +# $EXTRA_CONFIGURE_OPTS variable. +.cross_accel_build_job: + stage: build + image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest + timeout: 30m + script: + - mkdir build + - cd build + - PKG_CONFIG_PATH=$PKG_CONFIG_PATH + ../configure --enable-werror --disable-docs $QEMU_CONFIGURE_OPTS + --disable-tools --enable-${ACCEL:-kvm} $EXTRA_CONFIGURE_OPTS + - make -j$(expr $(nproc) + 1) all check-build $MAKE_CHECK_ARGS + +.cross_user_build_job: + stage: build + image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest + script: + - mkdir build + - cd build + - PKG_CONFIG_PATH=$PKG_CONFIG_PATH + ../configure --enable-werror --disable-docs $QEMU_CONFIGURE_OPTS + --disable-system + - make -j$(expr $(nproc) + 1) all check-build $MAKE_CHECK_ARGS diff --git a/.gitlab-ci.d/crossbuilds.yml b/.gitlab-ci.d/crossbuilds.yml index edf3a62563..6b3865c9e8 100644 --- a/.gitlab-ci.d/crossbuilds.yml +++ b/.gitlab-ci.d/crossbuilds.yml @@ -1,44 +1,5 @@ -.cross_system_build_job: - stage: build - image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest - timeout: 80m - script: - - mkdir build - - cd build - - PKG_CONFIG_PATH=$PKG_CONFIG_PATH - ../configure --enable-werror --disable-docs $QEMU_CONFIGURE_OPTS - --disable-user --target-list-exclude="arm-softmmu cris-softmmu - i386-softmmu microblaze-softmmu mips-softmmu mipsel-softmmu - mips64-softmmu ppc-softmmu sh4-softmmu xtensa-softmmu" - - make -j$(expr $(nproc) + 1) all check-build $MAKE_CHECK_ARGS - -# Job to cross-build specific accelerators. -# -# Set the $ACCEL variable to select the specific accelerator (default to -# KVM), and set extra options (such disabling other accelerators) via the -# $EXTRA_CONFIGURE_OPTS variable. -.cross_accel_build_job: - stage: build - image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest - timeout: 30m - script: - - mkdir build - - cd build - - PKG_CONFIG_PATH=$PKG_CONFIG_PATH - ../configure --enable-werror --disable-docs $QEMU_CONFIGURE_OPTS - --disable-tools --enable-${ACCEL:-kvm} $EXTRA_CONFIGURE_OPTS - - make -j$(expr $(nproc) + 1) all check-build $MAKE_CHECK_ARGS - -.cross_user_build_job: - stage: build - image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest - script: - - mkdir build - - cd build - - PKG_CONFIG_PATH=$PKG_CONFIG_PATH - ../configure --enable-werror --disable-docs $QEMU_CONFIGURE_OPTS - --disable-system - - make -j$(expr $(nproc) + 1) all check-build $MAKE_CHECK_ARGS +include: + - local: '/.gitlab-ci.d/crossbuild-template.yml' cross-armel-system: extends: .cross_system_build_job From f659e6584a2022473af14ac4d89f656515c25ea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Wed, 19 May 2021 20:54:59 +0200 Subject: [PATCH 06/18] gitlab: Extract DCO/style check jobs to static_checks.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the DCO / checkpatch jobs to a new file (static_checks.yml) to be able to run them without having to run all the jobs included in the default .gitlab-ci.yml, which are mainly useful for the mainstream CI. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Willian Rampazzo Message-Id: <20210519185504.2198573-6-f4bug@amsat.org> Signed-off-by: Thomas Huth --- .gitlab-ci.d/static_checks.yml | 24 ++++++++++++++++++++++++ .gitlab-ci.yml | 26 +------------------------- 2 files changed, 25 insertions(+), 25 deletions(-) create mode 100644 .gitlab-ci.d/static_checks.yml diff --git a/.gitlab-ci.d/static_checks.yml b/.gitlab-ci.d/static_checks.yml new file mode 100644 index 0000000000..f695627b7c --- /dev/null +++ b/.gitlab-ci.d/static_checks.yml @@ -0,0 +1,24 @@ +check-patch: + stage: build + image: $CI_REGISTRY_IMAGE/qemu/centos8:latest + needs: + job: amd64-centos8-container + script: .gitlab-ci.d/check-patch.py + except: + variables: + - $CI_PROJECT_NAMESPACE == 'qemu-project' && $CI_COMMIT_BRANCH == 'master' + variables: + GIT_DEPTH: 1000 + allow_failure: true + +check-dco: + stage: build + image: $CI_REGISTRY_IMAGE/qemu/centos8:latest + needs: + job: amd64-centos8-container + script: .gitlab-ci.d/check-dco.py + except: + variables: + - $CI_PROJECT_NAMESPACE == 'qemu-project' && $CI_COMMIT_BRANCH == 'master' + variables: + GIT_DEPTH: 1000 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bfdbf9ddde..369963a949 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,6 +12,7 @@ include: - local: '/.gitlab-ci.d/opensbi.yml' - local: '/.gitlab-ci.d/containers.yml' - local: '/.gitlab-ci.d/crossbuilds.yml' + - local: '/.gitlab-ci.d/static_checks.yml' .native_build_job_template: stage: build @@ -771,31 +772,6 @@ build-without-default-features: --target-list-exclude=arm-softmmu,i386-softmmu,mipsel-softmmu,mips64-softmmu,ppc-softmmu MAKE_CHECK_ARGS: check-unit -check-patch: - stage: build - image: $CI_REGISTRY_IMAGE/qemu/centos8:latest - needs: - job: amd64-centos8-container - script: .gitlab-ci.d/check-patch.py - except: - variables: - - $CI_PROJECT_NAMESPACE == 'qemu-project' && $CI_COMMIT_BRANCH == 'master' - variables: - GIT_DEPTH: 1000 - allow_failure: true - -check-dco: - stage: build - image: $CI_REGISTRY_IMAGE/qemu/centos8:latest - needs: - job: amd64-centos8-container - script: .gitlab-ci.d/check-dco.py - except: - variables: - - $CI_PROJECT_NAMESPACE == 'qemu-project' && $CI_COMMIT_BRANCH == 'master' - variables: - GIT_DEPTH: 1000 - build-libvhost-user: stage: build image: $CI_REGISTRY_IMAGE/qemu/fedora:latest From 07df3dfafa94460082f704dab674a7911c844ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Wed, 19 May 2021 20:55:00 +0200 Subject: [PATCH 07/18] gitlab: Extract build stages to stages.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the build stages used by our job templates to a new file (stages.yml) to be able to include it with the other templates, without having to run all the jobs included in the default .gitlab-ci.yml, which are mainly useful for mainstream CI. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Willian Rampazzo Message-Id: <20210519185504.2198573-7-f4bug@amsat.org> Signed-off-by: Thomas Huth --- .gitlab-ci.d/stages.yml | 8 ++++++++ .gitlab-ci.yml | 10 +--------- 2 files changed, 9 insertions(+), 9 deletions(-) create mode 100644 .gitlab-ci.d/stages.yml diff --git a/.gitlab-ci.d/stages.yml b/.gitlab-ci.d/stages.yml new file mode 100644 index 0000000000..f50826018d --- /dev/null +++ b/.gitlab-ci.d/stages.yml @@ -0,0 +1,8 @@ +# Currently we have two build stages after our containers are built: +# - build (for traditional build and test or first stage build) +# - test (for test stages, using build artefacts from a build stage) +stages: + - containers + - containers-layer2 + - build + - test diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 369963a949..2be8097f19 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,13 +1,5 @@ -# Currently we have two build stages after our containers are built: -# - build (for traditional build and test or first stage build) -# - test (for test stages, using build artefacts from a build stage) -stages: - - containers - - containers-layer2 - - build - - test - include: + - local: '/.gitlab-ci.d/stages.yml' - local: '/.gitlab-ci.d/edk2.yml' - local: '/.gitlab-ci.d/opensbi.yml' - local: '/.gitlab-ci.d/containers.yml' From 6cd5251b407ab1ec7f5b8dbdc5d030ca48e4ed5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Wed, 19 May 2021 20:55:01 +0200 Subject: [PATCH 08/18] gitlab: Extract default build/test jobs templates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To be able to reuse the mainstream build/test jobs templates, extract them into a new file (buildtest-template.yml). Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Willian Rampazzo Message-Id: <20210519185504.2198573-8-f4bug@amsat.org> [thuth: Keep the "acceptance_test_job_template" name for now] Signed-off-by: Thomas Huth --- .gitlab-ci.d/buildtest-template.yml | 69 ++++++++++++++++++++++++++++ .gitlab-ci.yml | 71 +---------------------------- 2 files changed, 70 insertions(+), 70 deletions(-) create mode 100644 .gitlab-ci.d/buildtest-template.yml diff --git a/.gitlab-ci.d/buildtest-template.yml b/.gitlab-ci.d/buildtest-template.yml new file mode 100644 index 0000000000..530d30746c --- /dev/null +++ b/.gitlab-ci.d/buildtest-template.yml @@ -0,0 +1,69 @@ +.native_build_job_template: + stage: build + image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest + before_script: + - JOBS=$(expr $(nproc) + 1) + script: + - if test -n "$LD_JOBS"; + then + scripts/git-submodule.sh update meson ; + fi + - mkdir build + - cd build + - if test -n "$TARGETS"; + then + ../configure --enable-werror --disable-docs ${LD_JOBS:+--meson=git} $CONFIGURE_ARGS --target-list="$TARGETS" ; + else + ../configure --enable-werror --disable-docs ${LD_JOBS:+--meson=git} $CONFIGURE_ARGS ; + fi || { cat config.log meson-logs/meson-log.txt && exit 1; } + - if test -n "$LD_JOBS"; + then + ../meson/meson.py configure . -Dbackend_max_links="$LD_JOBS" ; + fi || exit 1; + - make -j"$JOBS" + - if test -n "$MAKE_CHECK_ARGS"; + then + make -j"$JOBS" $MAKE_CHECK_ARGS ; + fi + +.native_test_job_template: + stage: test + image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest + script: + - scripts/git-submodule.sh update + $(sed -n '/GIT_SUBMODULES=/ s/.*=// p' build/config-host.mak) + - cd build + - find . -type f -exec touch {} + + # Avoid recompiling by hiding ninja with NINJA=":" + - make NINJA=":" $MAKE_CHECK_ARGS + +.acceptance_test_job_template: + extends: .native_test_job_template + cache: + key: "${CI_JOB_NAME}-cache" + paths: + - ${CI_PROJECT_DIR}/avocado-cache + policy: pull-push + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" + when: always + expire_in: 2 days + paths: + - build/tests/results/latest/results.xml + - build/tests/results/latest/test-results + reports: + junit: build/tests/results/latest/results.xml + before_script: + - mkdir -p ~/.config/avocado + - echo "[datadir.paths]" > ~/.config/avocado/avocado.conf + - echo "cache_dirs = ['${CI_PROJECT_DIR}/avocado-cache']" + >> ~/.config/avocado/avocado.conf + - echo -e '[job.output.testlogs]\nstatuses = ["FAIL", "INTERRUPT"]' + >> ~/.config/avocado/avocado.conf + - if [ -d ${CI_PROJECT_DIR}/avocado-cache ]; then + du -chs ${CI_PROJECT_DIR}/avocado-cache ; + fi + - export AVOCADO_ALLOW_UNTRUSTED_CODE=1 + after_script: + - cd build + - du -chs ${CI_PROJECT_DIR}/avocado-cache diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2be8097f19..ccd11e1625 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,78 +4,9 @@ include: - local: '/.gitlab-ci.d/opensbi.yml' - local: '/.gitlab-ci.d/containers.yml' - local: '/.gitlab-ci.d/crossbuilds.yml' + - local: '/.gitlab-ci.d/buildtest-template.yml' - local: '/.gitlab-ci.d/static_checks.yml' -.native_build_job_template: - stage: build - image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest - before_script: - - JOBS=$(expr $(nproc) + 1) - script: - - if test -n "$LD_JOBS"; - then - scripts/git-submodule.sh update meson ; - fi - - mkdir build - - cd build - - if test -n "$TARGETS"; - then - ../configure --enable-werror --disable-docs ${LD_JOBS:+--meson=git} $CONFIGURE_ARGS --target-list="$TARGETS" ; - else - ../configure --enable-werror --disable-docs ${LD_JOBS:+--meson=git} $CONFIGURE_ARGS ; - fi || { cat config.log meson-logs/meson-log.txt && exit 1; } - - if test -n "$LD_JOBS"; - then - ../meson/meson.py configure . -Dbackend_max_links="$LD_JOBS" ; - fi || exit 1; - - make -j"$JOBS" - - if test -n "$MAKE_CHECK_ARGS"; - then - make -j"$JOBS" $MAKE_CHECK_ARGS ; - fi - -.native_test_job_template: - stage: test - image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest - script: - - scripts/git-submodule.sh update - $(sed -n '/GIT_SUBMODULES=/ s/.*=// p' build/config-host.mak) - - cd build - - find . -type f -exec touch {} + - # Avoid recompiling by hiding ninja with NINJA=":" - - make NINJA=":" $MAKE_CHECK_ARGS - -.acceptance_test_job_template: - extends: .native_test_job_template - cache: - key: "${CI_JOB_NAME}-cache" - paths: - - ${CI_PROJECT_DIR}/avocado-cache - policy: pull-push - artifacts: - name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" - when: always - expire_in: 2 days - paths: - - build/tests/results/latest/results.xml - - build/tests/results/latest/test-results - reports: - junit: build/tests/results/latest/results.xml - before_script: - - mkdir -p ~/.config/avocado - - echo "[datadir.paths]" > ~/.config/avocado/avocado.conf - - echo "cache_dirs = ['${CI_PROJECT_DIR}/avocado-cache']" - >> ~/.config/avocado/avocado.conf - - echo -e '[job.output.testlogs]\nstatuses = ["FAIL", "INTERRUPT"]' - >> ~/.config/avocado/avocado.conf - - if [ -d ${CI_PROJECT_DIR}/avocado-cache ]; then - du -chs ${CI_PROJECT_DIR}/avocado-cache ; - fi - - export AVOCADO_ALLOW_UNTRUSTED_CODE=1 - after_script: - - cd build - - du -chs ${CI_PROJECT_DIR}/avocado-cache - build-system-alpine: extends: .native_build_job_template needs: From 6b505a4e4f0dbda681c9dfbaca2fa824c341981e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Wed, 19 May 2021 20:55:03 +0200 Subject: [PATCH 09/18] gitlab: Extract core container jobs to container-core.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is not possible to use the previously extracted templates without this set of core containers. Extract them into a new file (container-core.yml) to be able to build them without having to build all the other containers by default. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Thomas Huth Reviewed-by: Willian Rampazzo Message-Id: <20210519185504.2198573-10-f4bug@amsat.org> Signed-off-by: Thomas Huth --- .gitlab-ci.d/container-core.yml | 17 +++++++++++++++++ .gitlab-ci.d/containers.yml | 17 +---------------- 2 files changed, 18 insertions(+), 16 deletions(-) create mode 100644 .gitlab-ci.d/container-core.yml diff --git a/.gitlab-ci.d/container-core.yml b/.gitlab-ci.d/container-core.yml new file mode 100644 index 0000000000..e8dd1f476a --- /dev/null +++ b/.gitlab-ci.d/container-core.yml @@ -0,0 +1,17 @@ +include: + - local: '/.gitlab-ci.d/container-template.yml' + +amd64-centos8-container: + extends: .container_job_template + variables: + NAME: centos8 + +amd64-fedora-container: + extends: .container_job_template + variables: + NAME: fedora + +amd64-debian10-container: + extends: .container_job_template + variables: + NAME: debian10 diff --git a/.gitlab-ci.d/containers.yml b/.gitlab-ci.d/containers.yml index 2b06763d73..0acc8381ac 100644 --- a/.gitlab-ci.d/containers.yml +++ b/.gitlab-ci.d/containers.yml @@ -1,5 +1,5 @@ include: - - local: '/.gitlab-ci.d/container-template.yml' + - local: '/.gitlab-ci.d/container-core.yml' amd64-alpine-container: extends: .container_job_template @@ -11,16 +11,6 @@ amd64-centos7-container: variables: NAME: centos7 -amd64-centos8-container: - extends: .container_job_template - variables: - NAME: centos8 - -amd64-debian10-container: - extends: .container_job_template - variables: - NAME: debian10 - amd64-debian11-container: extends: .container_job_template variables: @@ -211,11 +201,6 @@ cris-fedora-cross-container: variables: NAME: fedora-cris-cross -amd64-fedora-container: - extends: .container_job_template - variables: - NAME: fedora - i386-fedora-cross-container: extends: .container_job_template variables: From dcaf5e8f2fab7426194f3d656cf272accd28518b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Fri, 14 May 2021 13:04:04 +0100 Subject: [PATCH 10/18] gitlab: Drop linux user build job for CentOS 7 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It has been over two years since RHEL-8 was released, and thus per the platform build policy, we no longer need to support RHEL-7 as a build target. The build-user-centos7 job was to detect a failure specific to CentOS 7 and there are already other linux user jobs for other platforms. Thus we can drop this job rather than move it to CentOS 8. Signed-off-by: Daniel P. Berrangé Reviewed-by: Willian Rampazzo Reviewed-by: Thomas Huth Message-Id: <20210514120415.1368922-2-berrange@redhat.com> Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Philippe Mathieu-Daudé --- .gitlab-ci.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ccd11e1625..09ac74fc50 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -373,15 +373,6 @@ build-user-plugins: MAKE_CHECK_ARGS: check-tcg timeout: 1h 30m -build-user-centos7: - extends: .native_build_job_template - needs: - job: amd64-centos7-container - variables: - IMAGE: centos7 - CONFIGURE_ARGS: --disable-system --disable-tools --disable-docs - MAKE_CHECK_ARGS: check-tcg - build-some-softmmu-plugins: extends: .native_build_job_template needs: From f2f5c3e62907088df2c0540f15c00ffc5a06211b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Tue, 11 May 2021 09:10:23 +0200 Subject: [PATCH 11/18] gitlab: Extract all default build/test jobs to buildtest.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the build/test jobs run by default on the mainstream CI into a new file (buildtest.yml). Reviewed-by: Willian Rampazzo Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Alex Bennée Message-Id: <20210511072952.2813358-9-f4bug@amsat.org> --- .gitlab-ci.d/buildtest.yml | 730 +++++++++++++++++++++++++++++++++++++ .gitlab-ci.yml | 730 +------------------------------------ 2 files changed, 731 insertions(+), 729 deletions(-) create mode 100644 .gitlab-ci.d/buildtest.yml diff --git a/.gitlab-ci.d/buildtest.yml b/.gitlab-ci.d/buildtest.yml new file mode 100644 index 0000000000..807040c1c7 --- /dev/null +++ b/.gitlab-ci.d/buildtest.yml @@ -0,0 +1,730 @@ +include: + - local: '/.gitlab-ci.d/buildtest-template.yml' + +build-system-alpine: + extends: .native_build_job_template + needs: + - job: amd64-alpine-container + variables: + IMAGE: alpine + TARGETS: aarch64-softmmu alpha-softmmu cris-softmmu hppa-softmmu + microblazeel-softmmu mips64el-softmmu + MAKE_CHECK_ARGS: check-build + CONFIGURE_ARGS: --enable-docs --enable-trace-backends=log,simple,syslog + artifacts: + expire_in: 2 days + paths: + - .git-submodule-status + - build + +check-system-alpine: + extends: .native_test_job_template + needs: + - job: build-system-alpine + artifacts: true + variables: + IMAGE: alpine + MAKE_CHECK_ARGS: check + +acceptance-system-alpine: + extends: .acceptance_test_job_template + needs: + - job: build-system-alpine + artifacts: true + variables: + IMAGE: alpine + MAKE_CHECK_ARGS: check-acceptance + +build-system-ubuntu: + extends: .native_build_job_template + needs: + job: amd64-ubuntu2004-container + variables: + IMAGE: ubuntu2004 + CONFIGURE_ARGS: --enable-docs --enable-fdt=system --enable-slirp=system + TARGETS: aarch64-softmmu alpha-softmmu cris-softmmu hppa-softmmu + microblazeel-softmmu mips64el-softmmu + MAKE_CHECK_ARGS: check-build + artifacts: + expire_in: 2 days + paths: + - build + +check-system-ubuntu: + extends: .native_test_job_template + needs: + - job: build-system-ubuntu + artifacts: true + variables: + IMAGE: ubuntu2004 + MAKE_CHECK_ARGS: check + +acceptance-system-ubuntu: + extends: .acceptance_test_job_template + needs: + - job: build-system-ubuntu + artifacts: true + variables: + IMAGE: ubuntu2004 + MAKE_CHECK_ARGS: check-acceptance + +build-system-debian: + extends: .native_build_job_template + needs: + job: amd64-debian-container + variables: + IMAGE: debian-amd64 + CONFIGURE_ARGS: --enable-fdt=system + TARGETS: arm-softmmu avr-softmmu i386-softmmu mipsel-softmmu + riscv64-softmmu sh4eb-softmmu sparc-softmmu xtensaeb-softmmu + MAKE_CHECK_ARGS: check-build + artifacts: + expire_in: 2 days + paths: + - build + +check-system-debian: + extends: .native_test_job_template + needs: + - job: build-system-debian + artifacts: true + variables: + IMAGE: debian-amd64 + MAKE_CHECK_ARGS: check + +acceptance-system-debian: + extends: .acceptance_test_job_template + needs: + - job: build-system-debian + artifacts: true + variables: + IMAGE: debian-amd64 + MAKE_CHECK_ARGS: check-acceptance + +build-system-fedora: + extends: .native_build_job_template + needs: + job: amd64-fedora-container + variables: + IMAGE: fedora + CONFIGURE_ARGS: --disable-gcrypt --enable-nettle --enable-docs + --enable-fdt=system --enable-slirp=system --enable-capstone=system + TARGETS: tricore-softmmu microblaze-softmmu mips-softmmu + xtensa-softmmu m68k-softmmu riscv32-softmmu ppc-softmmu sparc64-softmmu + MAKE_CHECK_ARGS: check-build + artifacts: + expire_in: 2 days + paths: + - build + +check-system-fedora: + extends: .native_test_job_template + needs: + - job: build-system-fedora + artifacts: true + variables: + IMAGE: fedora + MAKE_CHECK_ARGS: check + +acceptance-system-fedora: + extends: .acceptance_test_job_template + needs: + - job: build-system-fedora + artifacts: true + variables: + IMAGE: fedora + MAKE_CHECK_ARGS: check-acceptance + +build-system-centos: + extends: .native_build_job_template + needs: + job: amd64-centos8-container + variables: + IMAGE: centos8 + CONFIGURE_ARGS: --disable-nettle --enable-gcrypt --enable-fdt=system + --enable-modules --enable-trace-backends=dtrace + TARGETS: ppc64-softmmu or1k-softmmu s390x-softmmu + x86_64-softmmu rx-softmmu sh4-softmmu nios2-softmmu + MAKE_CHECK_ARGS: check-build + artifacts: + expire_in: 2 days + paths: + - build + +check-system-centos: + extends: .native_test_job_template + needs: + - job: build-system-centos + artifacts: true + variables: + IMAGE: centos8 + MAKE_CHECK_ARGS: check + +acceptance-system-centos: + extends: .acceptance_test_job_template + needs: + - job: build-system-centos + artifacts: true + variables: + IMAGE: centos8 + MAKE_CHECK_ARGS: check-acceptance + +build-system-opensuse: + extends: .native_build_job_template + needs: + job: amd64-opensuse-leap-container + variables: + IMAGE: opensuse-leap + CONFIGURE_ARGS: --enable-fdt=system + TARGETS: s390x-softmmu x86_64-softmmu aarch64-softmmu + MAKE_CHECK_ARGS: check-build + artifacts: + expire_in: 2 days + paths: + - build + +check-system-opensuse: + extends: .native_test_job_template + needs: + - job: build-system-opensuse + artifacts: true + variables: + IMAGE: opensuse-leap + MAKE_CHECK_ARGS: check + +acceptance-system-opensuse: + extends: .acceptance_test_job_template + needs: + - job: build-system-opensuse + artifacts: true + variables: + IMAGE: opensuse-leap + MAKE_CHECK_ARGS: check-acceptance + + +build-disabled: + extends: .native_build_job_template + needs: + job: amd64-fedora-container + variables: + IMAGE: fedora + CONFIGURE_ARGS: + --disable-attr + --disable-auth-pam + --disable-avx2 + --disable-bochs + --disable-brlapi + --disable-bzip2 + --disable-cap-ng + --disable-capstone + --disable-cloop + --disable-coroutine-pool + --disable-curl + --disable-curses + --disable-dmg + --disable-docs + --disable-gcrypt + --disable-glusterfs + --disable-gnutls + --disable-gtk + --disable-guest-agent + --disable-iconv + --disable-keyring + --disable-kvm + --disable-libiscsi + --disable-libpmem + --disable-libssh + --disable-libudev + --disable-libusb + --disable-libxml2 + --disable-linux-aio + --disable-live-block-migration + --disable-lzo + --disable-malloc-trim + --disable-mpath + --disable-nettle + --disable-numa + --disable-opengl + --disable-parallels + --disable-pie + --disable-qcow1 + --disable-qed + --disable-qom-cast-debug + --disable-rbd + --disable-rdma + --disable-replication + --disable-sdl + --disable-seccomp + --disable-slirp + --disable-smartcard + --disable-snappy + --disable-sparse + --disable-spice + --disable-strip + --disable-tpm + --disable-usb-redir + --disable-vdi + --disable-vhost-crypto + --disable-vhost-net + --disable-vhost-scsi + --disable-vhost-kernel + --disable-vhost-user + --disable-vhost-vdpa + --disable-vhost-vsock + --disable-virglrenderer + --disable-vnc + --disable-vte + --disable-vvfat + --disable-xen + --disable-zstd + TARGETS: arm-softmmu i386-softmmu ppc64-softmmu mips64-softmmu + s390x-softmmu i386-linux-user + MAKE_CHECK_ARGS: check-qtest SPEED=slow + +# This jobs explicitly disable TCG (--disable-tcg), KVM is detected by +# the configure script. The container doesn't contain Xen headers so +# Xen accelerator is not detected / selected. As result it build the +# i386-softmmu and x86_64-softmmu with KVM being the single accelerator +# available. +# Also use a different coroutine implementation (which is only really of +# interest to KVM users, i.e. with TCG disabled) +build-tcg-disabled: + extends: .native_build_job_template + needs: + job: amd64-centos8-container + variables: + IMAGE: centos8 + script: + - mkdir build + - cd build + - ../configure --disable-tcg --audio-drv-list="" --with-coroutine=ucontext + || { cat config.log meson-logs/meson-log.txt && exit 1; } + - make -j"$JOBS" + - make check-unit + - make check-qapi-schema + - cd tests/qemu-iotests/ + - ./check -raw 001 002 003 004 005 008 009 010 011 012 021 025 032 033 048 + 052 063 077 086 101 104 106 113 148 150 151 152 157 159 160 163 + 170 171 183 184 192 194 197 208 215 221 222 226 227 236 253 277 + - ./check -qcow2 028 051 056 057 058 065 068 082 085 091 095 096 102 122 + 124 132 139 142 144 145 151 152 155 157 165 194 196 197 200 202 + 208 209 215 216 218 222 227 234 246 247 248 250 254 255 257 258 + 260 261 262 263 264 270 272 273 277 279 + +build-user: + extends: .native_build_job_template + needs: + job: amd64-debian-user-cross-container + variables: + IMAGE: debian-all-test-cross + CONFIGURE_ARGS: --disable-tools --disable-system + MAKE_CHECK_ARGS: check-tcg + +build-user-static: + extends: .native_build_job_template + needs: + job: amd64-debian-user-cross-container + variables: + IMAGE: debian-all-test-cross + CONFIGURE_ARGS: --disable-tools --disable-system --static + MAKE_CHECK_ARGS: check-tcg + +# Because the hexagon cross-compiler takes so long to build we don't rely +# on the CI system to build it and hence this job has an optional dependency +# declared. The image is manually uploaded. +build-user-hexagon: + extends: .native_build_job_template + needs: + job: hexagon-cross-container + optional: true + variables: + IMAGE: debian-hexagon-cross + TARGETS: hexagon-linux-user + CONFIGURE_ARGS: --disable-tools --disable-docs --enable-debug-tcg + MAKE_CHECK_ARGS: check-tcg + +# Only build the softmmu targets we have check-tcg tests for +build-some-softmmu: + extends: .native_build_job_template + needs: + job: amd64-debian-user-cross-container + variables: + IMAGE: debian-all-test-cross + CONFIGURE_ARGS: --disable-tools --enable-debug + TARGETS: xtensa-softmmu arm-softmmu aarch64-softmmu alpha-softmmu + MAKE_CHECK_ARGS: check-tcg + +# Run check-tcg against linux-user (with plugins) +# we skip sparc64-linux-user until it has been fixed somewhat +# we skip cris-linux-user as it doesn't use the common run loop +build-user-plugins: + extends: .native_build_job_template + needs: + job: amd64-debian-user-cross-container + variables: + IMAGE: debian-all-test-cross + CONFIGURE_ARGS: --disable-tools --disable-system --enable-plugins --enable-debug-tcg --target-list-exclude=sparc64-linux-user,cris-linux-user + MAKE_CHECK_ARGS: check-tcg + timeout: 1h 30m + +build-some-softmmu-plugins: + extends: .native_build_job_template + needs: + job: amd64-debian-user-cross-container + variables: + IMAGE: debian-all-test-cross + CONFIGURE_ARGS: --disable-tools --disable-user --enable-plugins --enable-debug-tcg + TARGETS: xtensa-softmmu arm-softmmu aarch64-softmmu alpha-softmmu + MAKE_CHECK_ARGS: check-tcg + +clang-system: + extends: .native_build_job_template + needs: + job: amd64-fedora-container + variables: + IMAGE: fedora + CONFIGURE_ARGS: --cc=clang --cxx=clang++ + --extra-cflags=-fsanitize=undefined --extra-cflags=-fno-sanitize-recover=undefined + TARGETS: alpha-softmmu arm-softmmu m68k-softmmu mips64-softmmu + ppc-softmmu s390x-softmmu + MAKE_CHECK_ARGS: check-qtest check-tcg + +clang-user: + extends: .native_build_job_template + needs: + job: amd64-debian-user-cross-container + variables: + IMAGE: debian-all-test-cross + CONFIGURE_ARGS: --cc=clang --cxx=clang++ --disable-system + --target-list-exclude=microblazeel-linux-user,aarch64_be-linux-user,i386-linux-user,m68k-linux-user,mipsn32el-linux-user,xtensaeb-linux-user + --extra-cflags=-fsanitize=undefined --extra-cflags=-fno-sanitize-recover=undefined + MAKE_CHECK_ARGS: check-unit check-tcg + +# Set LD_JOBS=1 because this requires LTO and ld consumes a large amount of memory. +# On gitlab runners, default value sometimes end up calling 2 lds concurrently and +# triggers an Out-Of-Memory error +# +# Since slirp callbacks are used in QEMU Timers, slirp needs to be compiled together +# with QEMU and linked as a static library to avoid false positives in CFI checks. +# This can be accomplished by using -enable-slirp=git, which avoids the use of +# a system-wide version of the library +# +# Split in three sets of build/check/acceptance to limit the execution time of each +# job +build-cfi-aarch64: + extends: .native_build_job_template + needs: + - job: amd64-fedora-container + variables: + LD_JOBS: 1 + AR: llvm-ar + IMAGE: fedora + CONFIGURE_ARGS: --cc=clang --cxx=clang++ --enable-cfi --enable-cfi-debug + --enable-safe-stack --enable-slirp=git + TARGETS: aarch64-softmmu + MAKE_CHECK_ARGS: check-build + timeout: 70m + artifacts: + expire_in: 2 days + paths: + - build + +check-cfi-aarch64: + extends: .native_test_job_template + needs: + - job: build-cfi-aarch64 + artifacts: true + variables: + IMAGE: fedora + MAKE_CHECK_ARGS: check + +acceptance-cfi-aarch64: + extends: .acceptance_test_job_template + needs: + - job: build-cfi-aarch64 + artifacts: true + variables: + IMAGE: fedora + MAKE_CHECK_ARGS: check-acceptance + +build-cfi-ppc64-s390x: + extends: .native_build_job_template + needs: + - job: amd64-fedora-container + variables: + LD_JOBS: 1 + AR: llvm-ar + IMAGE: fedora + CONFIGURE_ARGS: --cc=clang --cxx=clang++ --enable-cfi --enable-cfi-debug + --enable-safe-stack --enable-slirp=git + TARGETS: ppc64-softmmu s390x-softmmu + MAKE_CHECK_ARGS: check-build + timeout: 70m + artifacts: + expire_in: 2 days + paths: + - build + +check-cfi-ppc64-s390x: + extends: .native_test_job_template + needs: + - job: build-cfi-ppc64-s390x + artifacts: true + variables: + IMAGE: fedora + MAKE_CHECK_ARGS: check + +acceptance-cfi-ppc64-s390x: + extends: .acceptance_test_job_template + needs: + - job: build-cfi-ppc64-s390x + artifacts: true + variables: + IMAGE: fedora + MAKE_CHECK_ARGS: check-acceptance + +build-cfi-x86_64: + extends: .native_build_job_template + needs: + - job: amd64-fedora-container + variables: + LD_JOBS: 1 + AR: llvm-ar + IMAGE: fedora + CONFIGURE_ARGS: --cc=clang --cxx=clang++ --enable-cfi --enable-cfi-debug + --enable-safe-stack --enable-slirp=git + TARGETS: x86_64-softmmu + MAKE_CHECK_ARGS: check-build + timeout: 70m + artifacts: + expire_in: 2 days + paths: + - build + +check-cfi-x86_64: + extends: .native_test_job_template + needs: + - job: build-cfi-x86_64 + artifacts: true + variables: + IMAGE: fedora + MAKE_CHECK_ARGS: check + +acceptance-cfi-x86_64: + extends: .acceptance_test_job_template + needs: + - job: build-cfi-x86_64 + artifacts: true + variables: + IMAGE: fedora + MAKE_CHECK_ARGS: check-acceptance + +tsan-build: + extends: .native_build_job_template + needs: + job: amd64-ubuntu2004-container + variables: + IMAGE: ubuntu2004 + CONFIGURE_ARGS: --enable-tsan --cc=clang-10 --cxx=clang++-10 + --enable-trace-backends=ust --enable-fdt=system --enable-slirp=system + TARGETS: x86_64-softmmu ppc64-softmmu riscv64-softmmu x86_64-linux-user + MAKE_CHECK_ARGS: bench V=1 + +# These targets are on the way out +build-deprecated: + extends: .native_build_job_template + needs: + job: amd64-debian-user-cross-container + variables: + IMAGE: debian-all-test-cross + CONFIGURE_ARGS: --disable-tools + MAKE_CHECK_ARGS: build-tcg + TARGETS: ppc64abi32-linux-user + artifacts: + expire_in: 2 days + paths: + - build + +# We split the check-tcg step as test failures are expected but we still +# want to catch the build breaking. +check-deprecated: + extends: .native_test_job_template + needs: + - job: build-deprecated + artifacts: true + variables: + IMAGE: debian-all-test-cross + MAKE_CHECK_ARGS: check-tcg + allow_failure: true + +# gprof/gcov are GCC features +gprof-gcov: + extends: .native_build_job_template + needs: + job: amd64-ubuntu2004-container + variables: + IMAGE: ubuntu2004 + CONFIGURE_ARGS: --enable-gprof --enable-gcov + MAKE_CHECK_ARGS: check + TARGETS: aarch64-softmmu ppc64-softmmu s390x-softmmu x86_64-softmmu + timeout: 70m + after_script: + - ${CI_PROJECT_DIR}/scripts/ci/coverage-summary.sh + +build-oss-fuzz: + extends: .native_build_job_template + needs: + job: amd64-fedora-container + variables: + IMAGE: fedora + script: + - mkdir build-oss-fuzz + - CC="clang" CXX="clang++" CFLAGS="-fsanitize=address" + ./scripts/oss-fuzz/build.sh + - export ASAN_OPTIONS="fast_unwind_on_malloc=0" + - for fuzzer in $(find ./build-oss-fuzz/DEST_DIR/ -executable -type f + | grep -v slirp); do + grep "LLVMFuzzerTestOneInput" ${fuzzer} > /dev/null 2>&1 || continue ; + echo Testing ${fuzzer} ... ; + "${fuzzer}" -runs=1 -seed=1 || exit 1 ; + done + # Unrelated to fuzzer: run some tests with -fsanitize=address + - cd build-oss-fuzz && make check-qtest-i386 check-unit + +build-tci: + extends: .native_build_job_template + needs: + job: amd64-debian-user-cross-container + variables: + IMAGE: debian-all-test-cross + script: + - TARGETS="aarch64 alpha arm hppa m68k microblaze ppc64 s390x x86_64" + - mkdir build + - cd build + - ../configure --enable-tcg-interpreter + --target-list="$(for tg in $TARGETS; do echo -n ${tg}'-softmmu '; done)" || { cat config.log meson-logs/meson-log.txt && exit 1; } + - make -j"$JOBS" + - make tests/qtest/boot-serial-test tests/qtest/cdrom-test tests/qtest/pxe-test + - for tg in $TARGETS ; do + export QTEST_QEMU_BINARY="./qemu-system-${tg}" ; + ./tests/qtest/boot-serial-test || exit 1 ; + ./tests/qtest/cdrom-test || exit 1 ; + done + - QTEST_QEMU_BINARY="./qemu-system-x86_64" ./tests/qtest/pxe-test + - QTEST_QEMU_BINARY="./qemu-system-s390x" ./tests/qtest/pxe-test -m slow + - make check-tcg + +# Alternate coroutines implementations are only really of interest to KVM users +# However we can't test against KVM on Gitlab-CI so we can only run unit tests +build-coroutine-sigaltstack: + extends: .native_build_job_template + needs: + job: amd64-ubuntu2004-container + variables: + IMAGE: ubuntu2004 + CONFIGURE_ARGS: --with-coroutine=sigaltstack --disable-tcg + --enable-trace-backends=ftrace + MAKE_CHECK_ARGS: check-unit + +# Most jobs test latest gcrypt or nettle builds +# +# These jobs test old gcrypt and nettle from RHEL7 +# which had some API differences. +crypto-old-nettle: + extends: .native_build_job_template + needs: + job: amd64-centos7-container + variables: + IMAGE: centos7 + TARGETS: x86_64-softmmu x86_64-linux-user + CONFIGURE_ARGS: --disable-gcrypt --enable-nettle + MAKE_CHECK_ARGS: check + +crypto-old-gcrypt: + extends: .native_build_job_template + needs: + job: amd64-centos7-container + variables: + IMAGE: centos7 + TARGETS: x86_64-softmmu x86_64-linux-user + CONFIGURE_ARGS: --disable-nettle --enable-gcrypt + MAKE_CHECK_ARGS: check + +crypto-only-gnutls: + extends: .native_build_job_template + needs: + job: amd64-centos7-container + variables: + IMAGE: centos7 + TARGETS: x86_64-softmmu x86_64-linux-user + CONFIGURE_ARGS: --disable-nettle --disable-gcrypt --enable-gnutls + MAKE_CHECK_ARGS: check + + +# Check our reduced build configurations +build-without-default-devices: + extends: .native_build_job_template + needs: + job: amd64-centos8-container + variables: + IMAGE: centos8 + CONFIGURE_ARGS: --without-default-devices --disable-user + +build-without-default-features: + extends: .native_build_job_template + needs: + job: amd64-debian-container + variables: + IMAGE: debian-amd64 + CONFIGURE_ARGS: --without-default-features --disable-user + --target-list-exclude=arm-softmmu,i386-softmmu,mipsel-softmmu,mips64-softmmu,ppc-softmmu + MAKE_CHECK_ARGS: check-unit + +build-libvhost-user: + stage: build + image: $CI_REGISTRY_IMAGE/qemu/fedora:latest + needs: + job: amd64-fedora-container + before_script: + - dnf install -y meson ninja-build + script: + - mkdir subprojects/libvhost-user/build + - cd subprojects/libvhost-user/build + - meson + - ninja + +# No targets are built here, just tools, docs, and unit tests. This +# also feeds into the eventual documentation deployment steps later +build-tools-and-docs-debian: + extends: .native_build_job_template + needs: + job: amd64-debian-container + variables: + IMAGE: debian-amd64 + MAKE_CHECK_ARGS: check-unit check-softfloat ctags TAGS cscope + CONFIGURE_ARGS: --disable-system --disable-user --enable-docs --enable-tools + artifacts: + expire_in: 2 days + paths: + - build + +# Prepare for GitLab pages deployment. Anything copied into the +# "public" directory will be deployed to $USER.gitlab.io/$PROJECT +pages: + image: $CI_REGISTRY_IMAGE/qemu/debian-amd64:latest + stage: test + needs: + - job: build-tools-and-docs-debian + script: + - mkdir -p public + # HTML-ised source tree + - make gtags + - htags -anT --tree-view=filetree -m qemu_init + -t "Welcome to the QEMU sourcecode" + - mv HTML public/src + # Project documentation + - make -C build install DESTDIR=$(pwd)/temp-install + - mv temp-install/usr/local/share/doc/qemu/* public/ + artifacts: + paths: + - public diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 09ac74fc50..aea58813a2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,733 +4,5 @@ include: - local: '/.gitlab-ci.d/opensbi.yml' - local: '/.gitlab-ci.d/containers.yml' - local: '/.gitlab-ci.d/crossbuilds.yml' - - local: '/.gitlab-ci.d/buildtest-template.yml' + - local: '/.gitlab-ci.d/buildtest.yml' - local: '/.gitlab-ci.d/static_checks.yml' - -build-system-alpine: - extends: .native_build_job_template - needs: - - job: amd64-alpine-container - variables: - IMAGE: alpine - TARGETS: aarch64-softmmu alpha-softmmu cris-softmmu hppa-softmmu - microblazeel-softmmu mips64el-softmmu - MAKE_CHECK_ARGS: check-build - CONFIGURE_ARGS: --enable-docs --enable-trace-backends=log,simple,syslog - artifacts: - expire_in: 2 days - paths: - - .git-submodule-status - - build - -check-system-alpine: - extends: .native_test_job_template - needs: - - job: build-system-alpine - artifacts: true - variables: - IMAGE: alpine - MAKE_CHECK_ARGS: check - -acceptance-system-alpine: - extends: .acceptance_test_job_template - needs: - - job: build-system-alpine - artifacts: true - variables: - IMAGE: alpine - MAKE_CHECK_ARGS: check-acceptance - -build-system-ubuntu: - extends: .native_build_job_template - needs: - job: amd64-ubuntu2004-container - variables: - IMAGE: ubuntu2004 - CONFIGURE_ARGS: --enable-docs --enable-fdt=system --enable-slirp=system - TARGETS: aarch64-softmmu alpha-softmmu cris-softmmu hppa-softmmu - microblazeel-softmmu mips64el-softmmu - MAKE_CHECK_ARGS: check-build - artifacts: - expire_in: 2 days - paths: - - build - -check-system-ubuntu: - extends: .native_test_job_template - needs: - - job: build-system-ubuntu - artifacts: true - variables: - IMAGE: ubuntu2004 - MAKE_CHECK_ARGS: check - -acceptance-system-ubuntu: - extends: .acceptance_test_job_template - needs: - - job: build-system-ubuntu - artifacts: true - variables: - IMAGE: ubuntu2004 - MAKE_CHECK_ARGS: check-acceptance - -build-system-debian: - extends: .native_build_job_template - needs: - job: amd64-debian-container - variables: - IMAGE: debian-amd64 - CONFIGURE_ARGS: --enable-fdt=system - TARGETS: arm-softmmu avr-softmmu i386-softmmu mipsel-softmmu - riscv64-softmmu sh4eb-softmmu sparc-softmmu xtensaeb-softmmu - MAKE_CHECK_ARGS: check-build - artifacts: - expire_in: 2 days - paths: - - build - -check-system-debian: - extends: .native_test_job_template - needs: - - job: build-system-debian - artifacts: true - variables: - IMAGE: debian-amd64 - MAKE_CHECK_ARGS: check - -acceptance-system-debian: - extends: .acceptance_test_job_template - needs: - - job: build-system-debian - artifacts: true - variables: - IMAGE: debian-amd64 - MAKE_CHECK_ARGS: check-acceptance - -build-system-fedora: - extends: .native_build_job_template - needs: - job: amd64-fedora-container - variables: - IMAGE: fedora - CONFIGURE_ARGS: --disable-gcrypt --enable-nettle --enable-docs - --enable-fdt=system --enable-slirp=system --enable-capstone=system - TARGETS: tricore-softmmu microblaze-softmmu mips-softmmu - xtensa-softmmu m68k-softmmu riscv32-softmmu ppc-softmmu sparc64-softmmu - MAKE_CHECK_ARGS: check-build - artifacts: - expire_in: 2 days - paths: - - build - -check-system-fedora: - extends: .native_test_job_template - needs: - - job: build-system-fedora - artifacts: true - variables: - IMAGE: fedora - MAKE_CHECK_ARGS: check - -acceptance-system-fedora: - extends: .acceptance_test_job_template - needs: - - job: build-system-fedora - artifacts: true - variables: - IMAGE: fedora - MAKE_CHECK_ARGS: check-acceptance - -build-system-centos: - extends: .native_build_job_template - needs: - job: amd64-centos8-container - variables: - IMAGE: centos8 - CONFIGURE_ARGS: --disable-nettle --enable-gcrypt --enable-fdt=system - --enable-modules --enable-trace-backends=dtrace - TARGETS: ppc64-softmmu or1k-softmmu s390x-softmmu - x86_64-softmmu rx-softmmu sh4-softmmu nios2-softmmu - MAKE_CHECK_ARGS: check-build - artifacts: - expire_in: 2 days - paths: - - build - -check-system-centos: - extends: .native_test_job_template - needs: - - job: build-system-centos - artifacts: true - variables: - IMAGE: centos8 - MAKE_CHECK_ARGS: check - -acceptance-system-centos: - extends: .acceptance_test_job_template - needs: - - job: build-system-centos - artifacts: true - variables: - IMAGE: centos8 - MAKE_CHECK_ARGS: check-acceptance - -build-system-opensuse: - extends: .native_build_job_template - needs: - job: amd64-opensuse-leap-container - variables: - IMAGE: opensuse-leap - CONFIGURE_ARGS: --enable-fdt=system - TARGETS: s390x-softmmu x86_64-softmmu aarch64-softmmu - MAKE_CHECK_ARGS: check-build - artifacts: - expire_in: 2 days - paths: - - build - -check-system-opensuse: - extends: .native_test_job_template - needs: - - job: build-system-opensuse - artifacts: true - variables: - IMAGE: opensuse-leap - MAKE_CHECK_ARGS: check - -acceptance-system-opensuse: - extends: .acceptance_test_job_template - needs: - - job: build-system-opensuse - artifacts: true - variables: - IMAGE: opensuse-leap - MAKE_CHECK_ARGS: check-acceptance - - -build-disabled: - extends: .native_build_job_template - needs: - job: amd64-fedora-container - variables: - IMAGE: fedora - CONFIGURE_ARGS: - --disable-attr - --disable-auth-pam - --disable-avx2 - --disable-bochs - --disable-brlapi - --disable-bzip2 - --disable-cap-ng - --disable-capstone - --disable-cloop - --disable-coroutine-pool - --disable-curl - --disable-curses - --disable-dmg - --disable-docs - --disable-gcrypt - --disable-glusterfs - --disable-gnutls - --disable-gtk - --disable-guest-agent - --disable-iconv - --disable-keyring - --disable-kvm - --disable-libiscsi - --disable-libpmem - --disable-libssh - --disable-libudev - --disable-libusb - --disable-libxml2 - --disable-linux-aio - --disable-live-block-migration - --disable-lzo - --disable-malloc-trim - --disable-mpath - --disable-nettle - --disable-numa - --disable-opengl - --disable-parallels - --disable-pie - --disable-qcow1 - --disable-qed - --disable-qom-cast-debug - --disable-rbd - --disable-rdma - --disable-replication - --disable-sdl - --disable-seccomp - --disable-slirp - --disable-smartcard - --disable-snappy - --disable-sparse - --disable-spice - --disable-strip - --disable-tpm - --disable-usb-redir - --disable-vdi - --disable-vhost-crypto - --disable-vhost-net - --disable-vhost-scsi - --disable-vhost-kernel - --disable-vhost-user - --disable-vhost-vdpa - --disable-vhost-vsock - --disable-virglrenderer - --disable-vnc - --disable-vte - --disable-vvfat - --disable-xen - --disable-zstd - TARGETS: arm-softmmu i386-softmmu ppc64-softmmu mips64-softmmu - s390x-softmmu i386-linux-user - MAKE_CHECK_ARGS: check-qtest SPEED=slow - -# This jobs explicitly disable TCG (--disable-tcg), KVM is detected by -# the configure script. The container doesn't contain Xen headers so -# Xen accelerator is not detected / selected. As result it build the -# i386-softmmu and x86_64-softmmu with KVM being the single accelerator -# available. -# Also use a different coroutine implementation (which is only really of -# interest to KVM users, i.e. with TCG disabled) -build-tcg-disabled: - extends: .native_build_job_template - needs: - job: amd64-centos8-container - variables: - IMAGE: centos8 - script: - - mkdir build - - cd build - - ../configure --disable-tcg --audio-drv-list="" --with-coroutine=ucontext - || { cat config.log meson-logs/meson-log.txt && exit 1; } - - make -j"$JOBS" - - make check-unit - - make check-qapi-schema - - cd tests/qemu-iotests/ - - ./check -raw 001 002 003 004 005 008 009 010 011 012 021 025 032 033 048 - 052 063 077 086 101 104 106 113 148 150 151 152 157 159 160 163 - 170 171 183 184 192 194 197 208 215 221 222 226 227 236 253 277 - - ./check -qcow2 028 051 056 057 058 065 068 082 085 091 095 096 102 122 - 124 132 139 142 144 145 151 152 155 157 165 194 196 197 200 202 - 208 209 215 216 218 222 227 234 246 247 248 250 254 255 257 258 - 260 261 262 263 264 270 272 273 277 279 - -build-user: - extends: .native_build_job_template - needs: - job: amd64-debian-user-cross-container - variables: - IMAGE: debian-all-test-cross - CONFIGURE_ARGS: --disable-tools --disable-system - MAKE_CHECK_ARGS: check-tcg - -build-user-static: - extends: .native_build_job_template - needs: - job: amd64-debian-user-cross-container - variables: - IMAGE: debian-all-test-cross - CONFIGURE_ARGS: --disable-tools --disable-system --static - MAKE_CHECK_ARGS: check-tcg - -# Because the hexagon cross-compiler takes so long to build we don't rely -# on the CI system to build it and hence this job has an optional dependency -# declared. The image is manually uploaded. -build-user-hexagon: - extends: .native_build_job_template - needs: - job: hexagon-cross-container - optional: true - variables: - IMAGE: debian-hexagon-cross - TARGETS: hexagon-linux-user - CONFIGURE_ARGS: --disable-tools --disable-docs --enable-debug-tcg - MAKE_CHECK_ARGS: check-tcg - -# Only build the softmmu targets we have check-tcg tests for -build-some-softmmu: - extends: .native_build_job_template - needs: - job: amd64-debian-user-cross-container - variables: - IMAGE: debian-all-test-cross - CONFIGURE_ARGS: --disable-tools --enable-debug - TARGETS: xtensa-softmmu arm-softmmu aarch64-softmmu alpha-softmmu - MAKE_CHECK_ARGS: check-tcg - -# Run check-tcg against linux-user (with plugins) -# we skip sparc64-linux-user until it has been fixed somewhat -# we skip cris-linux-user as it doesn't use the common run loop -build-user-plugins: - extends: .native_build_job_template - needs: - job: amd64-debian-user-cross-container - variables: - IMAGE: debian-all-test-cross - CONFIGURE_ARGS: --disable-tools --disable-system --enable-plugins --enable-debug-tcg --target-list-exclude=sparc64-linux-user,cris-linux-user - MAKE_CHECK_ARGS: check-tcg - timeout: 1h 30m - -build-some-softmmu-plugins: - extends: .native_build_job_template - needs: - job: amd64-debian-user-cross-container - variables: - IMAGE: debian-all-test-cross - CONFIGURE_ARGS: --disable-tools --disable-user --enable-plugins --enable-debug-tcg - TARGETS: xtensa-softmmu arm-softmmu aarch64-softmmu alpha-softmmu - MAKE_CHECK_ARGS: check-tcg - -clang-system: - extends: .native_build_job_template - needs: - job: amd64-fedora-container - variables: - IMAGE: fedora - CONFIGURE_ARGS: --cc=clang --cxx=clang++ - --extra-cflags=-fsanitize=undefined --extra-cflags=-fno-sanitize-recover=undefined - TARGETS: alpha-softmmu arm-softmmu m68k-softmmu mips64-softmmu - ppc-softmmu s390x-softmmu - MAKE_CHECK_ARGS: check-qtest check-tcg - -clang-user: - extends: .native_build_job_template - needs: - job: amd64-debian-user-cross-container - variables: - IMAGE: debian-all-test-cross - CONFIGURE_ARGS: --cc=clang --cxx=clang++ --disable-system - --target-list-exclude=microblazeel-linux-user,aarch64_be-linux-user,i386-linux-user,m68k-linux-user,mipsn32el-linux-user,xtensaeb-linux-user - --extra-cflags=-fsanitize=undefined --extra-cflags=-fno-sanitize-recover=undefined - MAKE_CHECK_ARGS: check-unit check-tcg - -# Set LD_JOBS=1 because this requires LTO and ld consumes a large amount of memory. -# On gitlab runners, default value sometimes end up calling 2 lds concurrently and -# triggers an Out-Of-Memory error -# -# Since slirp callbacks are used in QEMU Timers, slirp needs to be compiled together -# with QEMU and linked as a static library to avoid false positives in CFI checks. -# This can be accomplished by using -enable-slirp=git, which avoids the use of -# a system-wide version of the library -# -# Split in three sets of build/check/acceptance to limit the execution time of each -# job -build-cfi-aarch64: - extends: .native_build_job_template - needs: - - job: amd64-fedora-container - variables: - LD_JOBS: 1 - AR: llvm-ar - IMAGE: fedora - CONFIGURE_ARGS: --cc=clang --cxx=clang++ --enable-cfi --enable-cfi-debug - --enable-safe-stack --enable-slirp=git - TARGETS: aarch64-softmmu - MAKE_CHECK_ARGS: check-build - timeout: 70m - artifacts: - expire_in: 2 days - paths: - - build - -check-cfi-aarch64: - extends: .native_test_job_template - needs: - - job: build-cfi-aarch64 - artifacts: true - variables: - IMAGE: fedora - MAKE_CHECK_ARGS: check - -acceptance-cfi-aarch64: - extends: .acceptance_test_job_template - needs: - - job: build-cfi-aarch64 - artifacts: true - variables: - IMAGE: fedora - MAKE_CHECK_ARGS: check-acceptance - -build-cfi-ppc64-s390x: - extends: .native_build_job_template - needs: - - job: amd64-fedora-container - variables: - LD_JOBS: 1 - AR: llvm-ar - IMAGE: fedora - CONFIGURE_ARGS: --cc=clang --cxx=clang++ --enable-cfi --enable-cfi-debug - --enable-safe-stack --enable-slirp=git - TARGETS: ppc64-softmmu s390x-softmmu - MAKE_CHECK_ARGS: check-build - timeout: 70m - artifacts: - expire_in: 2 days - paths: - - build - -check-cfi-ppc64-s390x: - extends: .native_test_job_template - needs: - - job: build-cfi-ppc64-s390x - artifacts: true - variables: - IMAGE: fedora - MAKE_CHECK_ARGS: check - -acceptance-cfi-ppc64-s390x: - extends: .acceptance_test_job_template - needs: - - job: build-cfi-ppc64-s390x - artifacts: true - variables: - IMAGE: fedora - MAKE_CHECK_ARGS: check-acceptance - -build-cfi-x86_64: - extends: .native_build_job_template - needs: - - job: amd64-fedora-container - variables: - LD_JOBS: 1 - AR: llvm-ar - IMAGE: fedora - CONFIGURE_ARGS: --cc=clang --cxx=clang++ --enable-cfi --enable-cfi-debug - --enable-safe-stack --enable-slirp=git - TARGETS: x86_64-softmmu - MAKE_CHECK_ARGS: check-build - timeout: 70m - artifacts: - expire_in: 2 days - paths: - - build - -check-cfi-x86_64: - extends: .native_test_job_template - needs: - - job: build-cfi-x86_64 - artifacts: true - variables: - IMAGE: fedora - MAKE_CHECK_ARGS: check - -acceptance-cfi-x86_64: - extends: .acceptance_test_job_template - needs: - - job: build-cfi-x86_64 - artifacts: true - variables: - IMAGE: fedora - MAKE_CHECK_ARGS: check-acceptance - -tsan-build: - extends: .native_build_job_template - needs: - job: amd64-ubuntu2004-container - variables: - IMAGE: ubuntu2004 - CONFIGURE_ARGS: --enable-tsan --cc=clang-10 --cxx=clang++-10 - --enable-trace-backends=ust --enable-fdt=system --enable-slirp=system - TARGETS: x86_64-softmmu ppc64-softmmu riscv64-softmmu x86_64-linux-user - MAKE_CHECK_ARGS: bench V=1 - -# These targets are on the way out -build-deprecated: - extends: .native_build_job_template - needs: - job: amd64-debian-user-cross-container - variables: - IMAGE: debian-all-test-cross - CONFIGURE_ARGS: --disable-tools - MAKE_CHECK_ARGS: build-tcg - TARGETS: ppc64abi32-linux-user - artifacts: - expire_in: 2 days - paths: - - build - -# We split the check-tcg step as test failures are expected but we still -# want to catch the build breaking. -check-deprecated: - extends: .native_test_job_template - needs: - - job: build-deprecated - artifacts: true - variables: - IMAGE: debian-all-test-cross - MAKE_CHECK_ARGS: check-tcg - allow_failure: true - -# gprof/gcov are GCC features -gprof-gcov: - extends: .native_build_job_template - needs: - job: amd64-ubuntu2004-container - variables: - IMAGE: ubuntu2004 - CONFIGURE_ARGS: --enable-gprof --enable-gcov - MAKE_CHECK_ARGS: check - TARGETS: aarch64-softmmu ppc64-softmmu s390x-softmmu x86_64-softmmu - timeout: 70m - after_script: - - ${CI_PROJECT_DIR}/scripts/ci/coverage-summary.sh - -build-oss-fuzz: - extends: .native_build_job_template - needs: - job: amd64-fedora-container - variables: - IMAGE: fedora - script: - - mkdir build-oss-fuzz - - CC="clang" CXX="clang++" CFLAGS="-fsanitize=address" - ./scripts/oss-fuzz/build.sh - - export ASAN_OPTIONS="fast_unwind_on_malloc=0" - - for fuzzer in $(find ./build-oss-fuzz/DEST_DIR/ -executable -type f - | grep -v slirp); do - grep "LLVMFuzzerTestOneInput" ${fuzzer} > /dev/null 2>&1 || continue ; - echo Testing ${fuzzer} ... ; - "${fuzzer}" -runs=1 -seed=1 || exit 1 ; - done - # Unrelated to fuzzer: run some tests with -fsanitize=address - - cd build-oss-fuzz && make check-qtest-i386 check-unit - -build-tci: - extends: .native_build_job_template - needs: - job: amd64-debian-user-cross-container - variables: - IMAGE: debian-all-test-cross - script: - - TARGETS="aarch64 alpha arm hppa m68k microblaze ppc64 s390x x86_64" - - mkdir build - - cd build - - ../configure --enable-tcg-interpreter - --target-list="$(for tg in $TARGETS; do echo -n ${tg}'-softmmu '; done)" || { cat config.log meson-logs/meson-log.txt && exit 1; } - - make -j"$JOBS" - - make tests/qtest/boot-serial-test tests/qtest/cdrom-test tests/qtest/pxe-test - - for tg in $TARGETS ; do - export QTEST_QEMU_BINARY="./qemu-system-${tg}" ; - ./tests/qtest/boot-serial-test || exit 1 ; - ./tests/qtest/cdrom-test || exit 1 ; - done - - QTEST_QEMU_BINARY="./qemu-system-x86_64" ./tests/qtest/pxe-test - - QTEST_QEMU_BINARY="./qemu-system-s390x" ./tests/qtest/pxe-test -m slow - - make check-tcg - -# Alternate coroutines implementations are only really of interest to KVM users -# However we can't test against KVM on Gitlab-CI so we can only run unit tests -build-coroutine-sigaltstack: - extends: .native_build_job_template - needs: - job: amd64-ubuntu2004-container - variables: - IMAGE: ubuntu2004 - CONFIGURE_ARGS: --with-coroutine=sigaltstack --disable-tcg - --enable-trace-backends=ftrace - MAKE_CHECK_ARGS: check-unit - -# Most jobs test latest gcrypt or nettle builds -# -# These jobs test old gcrypt and nettle from RHEL7 -# which had some API differences. -crypto-old-nettle: - extends: .native_build_job_template - needs: - job: amd64-centos7-container - variables: - IMAGE: centos7 - TARGETS: x86_64-softmmu x86_64-linux-user - CONFIGURE_ARGS: --disable-gcrypt --enable-nettle - MAKE_CHECK_ARGS: check - -crypto-old-gcrypt: - extends: .native_build_job_template - needs: - job: amd64-centos7-container - variables: - IMAGE: centos7 - TARGETS: x86_64-softmmu x86_64-linux-user - CONFIGURE_ARGS: --disable-nettle --enable-gcrypt - MAKE_CHECK_ARGS: check - -crypto-only-gnutls: - extends: .native_build_job_template - needs: - job: amd64-centos7-container - variables: - IMAGE: centos7 - TARGETS: x86_64-softmmu x86_64-linux-user - CONFIGURE_ARGS: --disable-nettle --disable-gcrypt --enable-gnutls - MAKE_CHECK_ARGS: check - - -# Check our reduced build configurations -build-without-default-devices: - extends: .native_build_job_template - needs: - job: amd64-centos8-container - variables: - IMAGE: centos8 - CONFIGURE_ARGS: --without-default-devices --disable-user - -build-without-default-features: - extends: .native_build_job_template - needs: - job: amd64-debian-container - variables: - IMAGE: debian-amd64 - CONFIGURE_ARGS: --without-default-features --disable-user - --target-list-exclude=arm-softmmu,i386-softmmu,mipsel-softmmu,mips64-softmmu,ppc-softmmu - MAKE_CHECK_ARGS: check-unit - -build-libvhost-user: - stage: build - image: $CI_REGISTRY_IMAGE/qemu/fedora:latest - needs: - job: amd64-fedora-container - before_script: - - dnf install -y meson ninja-build - script: - - mkdir subprojects/libvhost-user/build - - cd subprojects/libvhost-user/build - - meson - - ninja - -# No targets are built here, just tools, docs, and unit tests. This -# also feeds into the eventual documentation deployment steps later -build-tools-and-docs-debian: - extends: .native_build_job_template - needs: - job: amd64-debian-container - variables: - IMAGE: debian-amd64 - MAKE_CHECK_ARGS: check-unit check-softfloat ctags TAGS cscope - CONFIGURE_ARGS: --disable-system --disable-user --enable-docs --enable-tools - artifacts: - expire_in: 2 days - paths: - - build - -# Prepare for GitLab pages deployment. Anything copied into the -# "public" directory will be deployed to $USER.gitlab.io/$PROJECT -pages: - image: $CI_REGISTRY_IMAGE/qemu/debian-amd64:latest - stage: test - needs: - - job: build-tools-and-docs-debian - script: - - mkdir -p public - # HTML-ised source tree - - make gtags - - htags -anT --tree-view=filetree -m qemu_init - -t "Welcome to the QEMU sourcecode" - - mv HTML public/src - # Project documentation - - make -C build install DESTDIR=$(pwd)/temp-install - - mv temp-install/usr/local/share/doc/qemu/* public/ - artifacts: - paths: - - public From a151926c80c3c8c279d664ca8182c46763ca0e42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Sat, 10 Apr 2021 12:58:55 +0200 Subject: [PATCH 12/18] gitlab: Move current job set to qemu-project.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To allow forks to easily decide which jobs they want to run, but without disrupting the current default, move the current set of jobs to a new file corresponding to the jobs run by the mainstream project CI: https://gitlab.com/qemu-project/qemu/-/pipelines Reviewed-by: Willian Rampazzo Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Alex Bennée Message-Id: <20210511072952.2813358-11-f4bug@amsat.org> --- .gitlab-ci.d/qemu-project.yml | 11 +++++++++++ .gitlab-ci.yml | 8 +------- 2 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 .gitlab-ci.d/qemu-project.yml diff --git a/.gitlab-ci.d/qemu-project.yml b/.gitlab-ci.d/qemu-project.yml new file mode 100644 index 0000000000..64cb2ba1da --- /dev/null +++ b/.gitlab-ci.d/qemu-project.yml @@ -0,0 +1,11 @@ +# This file contains the set of jobs run by the QEMU project: +# https://gitlab.com/qemu-project/qemu/-/pipelines + +include: + - local: '/.gitlab-ci.d/stages.yml' + - local: '/.gitlab-ci.d/edk2.yml' + - local: '/.gitlab-ci.d/opensbi.yml' + - local: '/.gitlab-ci.d/containers.yml' + - local: '/.gitlab-ci.d/crossbuilds.yml' + - local: '/.gitlab-ci.d/buildtest.yml' + - local: '/.gitlab-ci.d/static_checks.yml' diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aea58813a2..533a7e6133 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,2 @@ include: - - local: '/.gitlab-ci.d/stages.yml' - - local: '/.gitlab-ci.d/edk2.yml' - - local: '/.gitlab-ci.d/opensbi.yml' - - local: '/.gitlab-ci.d/containers.yml' - - local: '/.gitlab-ci.d/crossbuilds.yml' - - local: '/.gitlab-ci.d/buildtest.yml' - - local: '/.gitlab-ci.d/static_checks.yml' + - local: '/.gitlab-ci.d/qemu-project.yml' From 8b185c815ce8c16d7dea20c5ec379f9f30f00c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Tue, 25 May 2021 06:34:01 +0200 Subject: [PATCH 13/18] gitlab: Document how forks can use different set of jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document how forks can use different set of jobs and add a big warning so no new configuration is added to this file. Suggested-by: Daniel Berrangé Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Alex Bennée Message-Id: <20210525082556.4011380-4-f4bug@amsat.org> --- .gitlab-ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 533a7e6133..7e4ffab4d1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,2 +1,15 @@ +# +# This is the GitLab CI configuration file for the mainstream QEMU +# project: https://gitlab.com/qemu-project/qemu/-/pipelines +# +# !!! DO NOT ADD ANY NEW CONFIGURATION TO THIS FILE !!! +# +# Only documentation or comments is accepted. +# +# To use a different set of jobs than the mainstream QEMU project, +# you need to set the location of your custom yml file at "custom CI/CD +# configuration path", on your GitLab CI namespace: +# https://docs.gitlab.com/ee/ci/pipelines/settings.html#custom-cicd-configuration-path +# include: - local: '/.gitlab-ci.d/qemu-project.yml' From 961889f00c958e5500ec13eba71fbc5ccd243fd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Tue, 25 May 2021 09:03:17 +0200 Subject: [PATCH 14/18] gitlab: Extract cross-container jobs to container-cross.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the jobs preparing the cross containers into a new file (container-cross.yml). Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Alex Bennée Reviewed-by: Willian Rampazzo Message-Id: <20210525082556.4011380-5-f4bug@amsat.org> --- .gitlab-ci.d/container-cross.yml | 192 ++++++++++++++++++++++++++++++ .gitlab-ci.d/containers.yml | 194 +------------------------------ 2 files changed, 193 insertions(+), 193 deletions(-) create mode 100644 .gitlab-ci.d/container-cross.yml diff --git a/.gitlab-ci.d/container-cross.yml b/.gitlab-ci.d/container-cross.yml new file mode 100644 index 0000000000..0fcebe363a --- /dev/null +++ b/.gitlab-ci.d/container-cross.yml @@ -0,0 +1,192 @@ +alpha-debian-cross-container: + extends: .container_job_template + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-alpha-cross + +amd64-debian-cross-container: + extends: .container_job_template + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-amd64-cross + +amd64-debian-user-cross-container: + extends: .container_job_template + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-all-test-cross + +arm64-debian-cross-container: + extends: .container_job_template + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-arm64-cross + +arm64-test-debian-cross-container: + extends: .container_job_template + stage: containers-layer2 + needs: ['amd64-debian11-container'] + variables: + NAME: debian-arm64-test-cross + +armel-debian-cross-container: + extends: .container_job_template + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-armel-cross + +armhf-debian-cross-container: + extends: .container_job_template + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-armhf-cross + +# We never want to build hexagon in the CI system and by default we +# always want to refer to the master registry where it lives. +hexagon-cross-container: + image: docker:stable + stage: containers + rules: + - if: '$CI_PROJECT_NAMESPACE == "qemu-project"' + when: never + - when: always + variables: + NAME: debian-hexagon-cross + GIT_DEPTH: 1 + services: + - docker:dind + before_script: + - export TAG="$CI_REGISTRY_IMAGE/qemu/$NAME:latest" + - export COMMON_TAG="$CI_REGISTRY/qemu-project/qemu/qemu/$NAME:latest" + - docker info + - docker login $CI_REGISTRY -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" + script: + - echo "TAG:$TAG" + - echo "COMMON_TAG:$COMMON_TAG" + - docker pull $COMMON_TAG + - docker tag $COMMON_TAG $TAG + - docker push "$TAG" + after_script: + - docker logout + +hppa-debian-cross-container: + extends: .container_job_template + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-hppa-cross + +m68k-debian-cross-container: + extends: .container_job_template + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-m68k-cross + +mips64-debian-cross-container: + extends: .container_job_template + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-mips64-cross + +mips64el-debian-cross-container: + extends: .container_job_template + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-mips64el-cross + +mips-debian-cross-container: + extends: .container_job_template + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-mips-cross + +mipsel-debian-cross-container: + extends: .container_job_template + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-mipsel-cross + +powerpc-test-cross-container: + extends: .container_job_template + stage: containers-layer2 + needs: ['amd64-debian11-container'] + variables: + NAME: debian-powerpc-test-cross + +ppc64el-debian-cross-container: + extends: .container_job_template + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-ppc64el-cross + +riscv64-debian-cross-container: + extends: .container_job_template + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-riscv64-cross + +s390x-debian-cross-container: + extends: .container_job_template + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-s390x-cross + +sh4-debian-cross-container: + extends: .container_job_template + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-sh4-cross + +sparc64-debian-cross-container: + extends: .container_job_template + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-sparc64-cross + +tricore-debian-cross-container: + extends: .container_job_template + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-tricore-cross + +xtensa-debian-cross-container: + extends: .container_job_template + variables: + NAME: debian-xtensa-cross + +cris-fedora-cross-container: + extends: .container_job_template + variables: + NAME: fedora-cris-cross + +i386-fedora-cross-container: + extends: .container_job_template + variables: + NAME: fedora-i386-cross + +win32-fedora-cross-container: + extends: .container_job_template + variables: + NAME: fedora-win32-cross + +win64-fedora-cross-container: + extends: .container_job_template + variables: + NAME: fedora-win64-cross diff --git a/.gitlab-ci.d/containers.yml b/.gitlab-ci.d/containers.yml index 0acc8381ac..bd01ae8f80 100644 --- a/.gitlab-ci.d/containers.yml +++ b/.gitlab-ci.d/containers.yml @@ -1,5 +1,6 @@ include: - local: '/.gitlab-ci.d/container-core.yml' + - local: '/.gitlab-ci.d/container-cross.yml' amd64-alpine-container: extends: .container_job_template @@ -16,27 +17,6 @@ amd64-debian11-container: variables: NAME: debian11 -alpha-debian-cross-container: - extends: .container_job_template - stage: containers-layer2 - needs: ['amd64-debian10-container'] - variables: - NAME: debian-alpha-cross - -amd64-debian-cross-container: - extends: .container_job_template - stage: containers-layer2 - needs: ['amd64-debian10-container'] - variables: - NAME: debian-amd64-cross - -amd64-debian-user-cross-container: - extends: .container_job_template - stage: containers-layer2 - needs: ['amd64-debian10-container'] - variables: - NAME: debian-all-test-cross - amd64-debian-container: extends: .container_job_template stage: containers-layer2 @@ -44,178 +24,6 @@ amd64-debian-container: variables: NAME: debian-amd64 -arm64-debian-cross-container: - extends: .container_job_template - stage: containers-layer2 - needs: ['amd64-debian10-container'] - variables: - NAME: debian-arm64-cross - -arm64-test-debian-cross-container: - extends: .container_job_template - stage: containers-layer2 - needs: ['amd64-debian11-container'] - variables: - NAME: debian-arm64-test-cross - -armel-debian-cross-container: - extends: .container_job_template - stage: containers-layer2 - needs: ['amd64-debian10-container'] - variables: - NAME: debian-armel-cross - -armhf-debian-cross-container: - extends: .container_job_template - stage: containers-layer2 - needs: ['amd64-debian10-container'] - variables: - NAME: debian-armhf-cross - -# We never want to build hexagon in the CI system and by default we -# always want to refer to the master registry where it lives. -hexagon-cross-container: - image: docker:stable - stage: containers - rules: - - if: '$CI_PROJECT_NAMESPACE == "qemu-project"' - when: never - - when: always - variables: - NAME: debian-hexagon-cross - GIT_DEPTH: 1 - services: - - docker:dind - before_script: - - export TAG="$CI_REGISTRY_IMAGE/qemu/$NAME:latest" - - export COMMON_TAG="$CI_REGISTRY/qemu-project/qemu/qemu/$NAME:latest" - - docker info - - docker login $CI_REGISTRY -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" - script: - - echo "TAG:$TAG" - - echo "COMMON_TAG:$COMMON_TAG" - - docker pull $COMMON_TAG - - docker tag $COMMON_TAG $TAG - - docker push "$TAG" - after_script: - - docker logout - -hppa-debian-cross-container: - extends: .container_job_template - stage: containers-layer2 - needs: ['amd64-debian10-container'] - variables: - NAME: debian-hppa-cross - -m68k-debian-cross-container: - extends: .container_job_template - stage: containers-layer2 - needs: ['amd64-debian10-container'] - variables: - NAME: debian-m68k-cross - -mips64-debian-cross-container: - extends: .container_job_template - stage: containers-layer2 - needs: ['amd64-debian10-container'] - variables: - NAME: debian-mips64-cross - -mips64el-debian-cross-container: - extends: .container_job_template - stage: containers-layer2 - needs: ['amd64-debian10-container'] - variables: - NAME: debian-mips64el-cross - -mips-debian-cross-container: - extends: .container_job_template - stage: containers-layer2 - needs: ['amd64-debian10-container'] - variables: - NAME: debian-mips-cross - -mipsel-debian-cross-container: - extends: .container_job_template - stage: containers-layer2 - needs: ['amd64-debian10-container'] - variables: - NAME: debian-mipsel-cross - -powerpc-test-cross-container: - extends: .container_job_template - stage: containers-layer2 - needs: ['amd64-debian11-container'] - variables: - NAME: debian-powerpc-test-cross - -ppc64el-debian-cross-container: - extends: .container_job_template - stage: containers-layer2 - needs: ['amd64-debian10-container'] - variables: - NAME: debian-ppc64el-cross - -riscv64-debian-cross-container: - extends: .container_job_template - stage: containers-layer2 - needs: ['amd64-debian10-container'] - variables: - NAME: debian-riscv64-cross - -s390x-debian-cross-container: - extends: .container_job_template - stage: containers-layer2 - needs: ['amd64-debian10-container'] - variables: - NAME: debian-s390x-cross - -sh4-debian-cross-container: - extends: .container_job_template - stage: containers-layer2 - needs: ['amd64-debian10-container'] - variables: - NAME: debian-sh4-cross - -sparc64-debian-cross-container: - extends: .container_job_template - stage: containers-layer2 - needs: ['amd64-debian10-container'] - variables: - NAME: debian-sparc64-cross - -tricore-debian-cross-container: - extends: .container_job_template - stage: containers-layer2 - needs: ['amd64-debian10-container'] - variables: - NAME: debian-tricore-cross - -xtensa-debian-cross-container: - extends: .container_job_template - variables: - NAME: debian-xtensa-cross - -cris-fedora-cross-container: - extends: .container_job_template - variables: - NAME: fedora-cris-cross - -i386-fedora-cross-container: - extends: .container_job_template - variables: - NAME: fedora-i386-cross - -win32-fedora-cross-container: - extends: .container_job_template - variables: - NAME: fedora-win32-cross - -win64-fedora-cross-container: - extends: .container_job_template - variables: - NAME: fedora-win64-cross - amd64-ubuntu1804-container: extends: .container_job_template variables: From f230dba714e7777098ede5758d62b21922e7ec2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Tue, 25 May 2021 06:46:56 +0200 Subject: [PATCH 15/18] gitlab: Keep Avocado reports during 1 week MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sometimes pull requests are merged during the week-end, triggering a CI pipeline. Currently if such pipeline fails, the Avocado reports are available for 2 days. For the reviewers working on the project during office hours, the reports are already discarded when they want to look at them. Increase this time to 1 week, which should give reviewers enough time. Only keep the reports on failure, which is the only case we'll look at them. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Alex Bennée Reviewed-by: Willian Rampazzo Message-Id: <20210525082556.4011380-8-f4bug@amsat.org> --- .gitlab-ci.d/buildtest-template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.d/buildtest-template.yml b/.gitlab-ci.d/buildtest-template.yml index 530d30746c..aabd6c07d2 100644 --- a/.gitlab-ci.d/buildtest-template.yml +++ b/.gitlab-ci.d/buildtest-template.yml @@ -46,8 +46,8 @@ policy: pull-push artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" - when: always - expire_in: 2 days + when: on_failure + expire_in: 7 days paths: - build/tests/results/latest/results.xml - build/tests/results/latest/test-results From f56bf4caf7179c82a50e3f7618f335526fba37d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Tue, 25 May 2021 07:18:39 +0200 Subject: [PATCH 16/18] gitlab: Run Avocado tests manually (except mainstream CI) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Due to a design problem and misunderstanding between the Avocado framework and QEMU, Avocado is fetching many asset artifacts it shouldn't be fetching, exhausting the jobs CI timeout. Since Avocado artifacts are cached, this is not an issue with old forks, which already have populated the cache and do not need to download new artifacts to run the tests. However this is very confusing to new contributors who start to fork the project and keep having failing CI pipelines. As a temporary kludge, add the QEMU_CI_AVOCADO_TESTING variable to allow old forks to keep running the Avocado tests, while still allowing new forks to use the mainstream set of CI tests. Keep the tests enabled by default on the mainstream namespace which is old enough to have a populated cache, hoping we will keep this cache long enough until the Avocado/QEMU design issue is fixed. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Alex Bennée Message-Id: <20210525082556.4011380-9-f4bug@amsat.org> --- .gitlab-ci.d/buildtest-template.yml | 12 ++++++++++++ .gitlab-ci.yml | 24 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/.gitlab-ci.d/buildtest-template.yml b/.gitlab-ci.d/buildtest-template.yml index aabd6c07d2..3e3e19d96b 100644 --- a/.gitlab-ci.d/buildtest-template.yml +++ b/.gitlab-ci.d/buildtest-template.yml @@ -67,3 +67,15 @@ after_script: - cd build - du -chs ${CI_PROJECT_DIR}/avocado-cache + rules: + # Only run these jobs if running on the mainstream namespace, + # or if the user set the QEMU_CI_AVOCADO_TESTING variable (either + # in its namespace setting or via git-push option, see documentation + # in /.gitlab-ci.yml of this repository). + - if: '$CI_PROJECT_NAMESPACE == "qemu-project"' + when: always + - if: '$QEMU_CI_AVOCADO_TESTING' + when: always + # Otherwise, set to manual (the jobs are created but not run). + - when: manual + allow_failure: true diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7e4ffab4d1..6dc5385e69 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,5 +11,29 @@ # configuration path", on your GitLab CI namespace: # https://docs.gitlab.com/ee/ci/pipelines/settings.html#custom-cicd-configuration-path # +# ---------------------------------------------------------------------- +# +# QEMU CI jobs are based on templates. Some templates provide +# user-configurable options, modifiable via configuration variables. +# +# These variables can be set globally in the user's CI namespace +# setting: +# https://docs.gitlab.com/ee/ci/variables/#create-a-custom-variable-in-the-ui +# or set manually each time a branch/tag is pushed, as a git-push +# command line argument: +# https://docs.gitlab.com/ee/user/project/push_options.html#push-options-for-gitlab-cicd +# +# Example setting the QEMU_CI_EXAMPLE_VAR variable: +# +# git push -o ci.variable="QEMU_CI_EXAMPLE_VAR=value" myrepo mybranch +# +# ---------------------------------------------------------------------- +# +# List of environment variables that can be use to modify the set +# of jobs selected: +# +# - QEMU_CI_AVOCADO_TESTING +# If set, tests using the Avocado framework will be run + include: - local: '/.gitlab-ci.d/qemu-project.yml' From 1f88fad50eb7412e27e0db2c4e37cb9e982ffa80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Tue, 25 May 2021 17:30:57 +0200 Subject: [PATCH 17/18] gitlab: Use $CI_DEFAULT_BRANCH instead of hardcoded 'master' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We want to skip the checkpatch and DCO signoff jobs when pushing to the default branch. Currently this branch is called 'master', but we don't need to hardcode this in the CI configuration, because the $CI_DEFAULT_BRANCH env variable exposes it. References: - https://sfconservancy.org/news/2020/jun/23/gitbranchname/ - https://about.gitlab.com/blog/2021/03/10/new-git-default-branch-name/ Suggested-by: Savitoj Singh Signed-off-by: Philippe Mathieu-Daudé Message-Id: <20210525153826.4174157-1-philmd@redhat.com> Reviewed-by: Daniel P. Berrangé Reviewed-by: Willian Rampazzo --- .gitlab-ci.d/static_checks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.d/static_checks.yml b/.gitlab-ci.d/static_checks.yml index f695627b7c..9c9a771b74 100644 --- a/.gitlab-ci.d/static_checks.yml +++ b/.gitlab-ci.d/static_checks.yml @@ -6,7 +6,7 @@ check-patch: script: .gitlab-ci.d/check-patch.py except: variables: - - $CI_PROJECT_NAMESPACE == 'qemu-project' && $CI_COMMIT_BRANCH == 'master' + - $CI_PROJECT_NAMESPACE == 'qemu-project' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH variables: GIT_DEPTH: 1000 allow_failure: true @@ -19,6 +19,6 @@ check-dco: script: .gitlab-ci.d/check-dco.py except: variables: - - $CI_PROJECT_NAMESPACE == 'qemu-project' && $CI_COMMIT_BRANCH == 'master' + - $CI_PROJECT_NAMESPACE == 'qemu-project' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH variables: GIT_DEPTH: 1000 From 2d9e45b9cb0baa7316901238f3795a74206af457 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Tue, 25 May 2021 15:17:10 +0200 Subject: [PATCH 18/18] gitlab: Convert check-dco/check-patch jobs to the 'rules' syntax MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per GitLab documentation [*]: "rules replaces only/except and they can’t be used together in the same job." Since the 'rules' syntax is more powerful and we are already using it, convert the check-dco/check-patch jobs so no job use the 'only/ except' syntax. [*] https://docs.gitlab.com/ee/ci/yaml/#rules Inspired-by: Daniel P. Berrangé Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Willian Rampazzo Message-Id: <20210525132418.4133235-3-f4bug@amsat.org> --- .gitlab-ci.d/static_checks.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.d/static_checks.yml b/.gitlab-ci.d/static_checks.yml index 9c9a771b74..91247a6f67 100644 --- a/.gitlab-ci.d/static_checks.yml +++ b/.gitlab-ci.d/static_checks.yml @@ -4,12 +4,13 @@ check-patch: needs: job: amd64-centos8-container script: .gitlab-ci.d/check-patch.py - except: - variables: - - $CI_PROJECT_NAMESPACE == 'qemu-project' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH variables: GIT_DEPTH: 1000 - allow_failure: true + rules: + - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + when: never + - when: on_success + allow_failure: true check-dco: stage: build @@ -17,8 +18,9 @@ check-dco: needs: job: amd64-centos8-container script: .gitlab-ci.d/check-dco.py - except: - variables: - - $CI_PROJECT_NAMESPACE == 'qemu-project' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH variables: GIT_DEPTH: 1000 + rules: + - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + when: never + - when: on_success