* Improve gitlab-CI with regards to handling of stable staging branches

* Add msys2 gitlab-CI artifacts * Minor qtest fixes -----BEGIN PGP SIGNATURE----- iQJFBAABCAAvFiEEJ7iIR+7gJQEY8+q5LtnXdP5wLbUFAmSZR6gRHHRodXRoQHJl ZGhhdC5jb20ACgkQLtnXdP5wLbWoFQ//VieL2UTOBXvw6TlMCYEpqKURdKYc7Uqp Y/gJRHK+EQ3C4BGzv8l/P07/H3N5da+8Y2Ta37tNritbs+tyrYVIQAY3+bugG6hO lIF5oUGTcbOkC6Z1ajtjHcmxCj+2Z8uumlFW44zMR4HzzcmaRDyVDoU0gUg0Ohkt aNdpjJEA8BRzvQTjx92v31uILk8zpd0yL+40p/2DSx0Dt5eoqTjFN4QCgqk+C9A3 WiiIkJBIIPgfp3XScVGeKS2ZfGSL7/QcJF0wbkkLhWfuF5oBjjkQCJlGYxpAnnbv J7esrNCxsks7T7SC/QnEzyePMXxX1DgV9znwBtEobLTQ38LcDWpdqdr0VYgyQhdo 9NgBLNkI3J1JCmJ5amCLRNcmH75cMnhxXeZYsjZ70VnirgFEQS1C+YELadCY8QWa S3YS/ZvOc5wHFdTrsfIyJG+2AjbefyboiXojzd/sFEY0485A8malTdtn96dhHjkZ KvInxQHV7uoUhok1QC68taMHbRUfA6jU7STYjkgDjnf+L+ywIbbKJE7LpyicvnsU MUR+9H4EsSlmN2koc9bopG0sspLThviIKORqzPEo3WyBj5jCIZ7tkvUEqBUkJwx2 hISZeqdhP+wRVR4Ter0RNywjk2gSbaYcPzlnbaRYZ5OoiRchXr+uh/X0dIdkCNPP YwB1Y0wBpPU= =4Jl2 -----END PGP SIGNATURE----- Merge tag 'pull-request-2023-06-26' of https://gitlab.com/thuth/qemu into staging * Improve gitlab-CI with regards to handling of stable staging branches * Add msys2 gitlab-CI artifacts * Minor qtest fixes # -----BEGIN PGP SIGNATURE----- # # iQJFBAABCAAvFiEEJ7iIR+7gJQEY8+q5LtnXdP5wLbUFAmSZR6gRHHRodXRoQHJl # ZGhhdC5jb20ACgkQLtnXdP5wLbWoFQ//VieL2UTOBXvw6TlMCYEpqKURdKYc7Uqp # Y/gJRHK+EQ3C4BGzv8l/P07/H3N5da+8Y2Ta37tNritbs+tyrYVIQAY3+bugG6hO # lIF5oUGTcbOkC6Z1ajtjHcmxCj+2Z8uumlFW44zMR4HzzcmaRDyVDoU0gUg0Ohkt # aNdpjJEA8BRzvQTjx92v31uILk8zpd0yL+40p/2DSx0Dt5eoqTjFN4QCgqk+C9A3 # WiiIkJBIIPgfp3XScVGeKS2ZfGSL7/QcJF0wbkkLhWfuF5oBjjkQCJlGYxpAnnbv # J7esrNCxsks7T7SC/QnEzyePMXxX1DgV9znwBtEobLTQ38LcDWpdqdr0VYgyQhdo # 9NgBLNkI3J1JCmJ5amCLRNcmH75cMnhxXeZYsjZ70VnirgFEQS1C+YELadCY8QWa # S3YS/ZvOc5wHFdTrsfIyJG+2AjbefyboiXojzd/sFEY0485A8malTdtn96dhHjkZ # KvInxQHV7uoUhok1QC68taMHbRUfA6jU7STYjkgDjnf+L+ywIbbKJE7LpyicvnsU # MUR+9H4EsSlmN2koc9bopG0sspLThviIKORqzPEo3WyBj5jCIZ7tkvUEqBUkJwx2 # hISZeqdhP+wRVR4Ter0RNywjk2gSbaYcPzlnbaRYZ5OoiRchXr+uh/X0dIdkCNPP # YwB1Y0wBpPU= # =4Jl2 # -----END PGP SIGNATURE----- # gpg: Signature made Mon 26 Jun 2023 10:09:12 AM CEST # gpg: using RSA key 27B88847EEE0250118F3EAB92ED9D774FE702DB5 # gpg: issuer "thuth@redhat.com" # gpg: Good signature from "Thomas Huth <th.huth@gmx.de>" [undefined] # gpg: aka "Thomas Huth <thuth@redhat.com>" [undefined] # gpg: aka "Thomas Huth <th.huth@posteo.de>" [unknown] # gpg: aka "Thomas Huth <huth@tuxfamily.org>" [undefined] # gpg: WARNING: This key is not certified with a trusted signature! # gpg: There is no indication that the signature belongs to the owner. # Primary key fingerprint: 27B8 8847 EEE0 2501 18F3 EAB9 2ED9 D774 FE70 2DB5 * tag 'pull-request-2023-06-26' of https://gitlab.com/thuth/qemu: tests/qtest/cxl-test: Clean up temporary directories after testing gitlab-ci: add msys2 meson test to junit report gitlab-ci: grab msys2 meson-logs as artifacts gitlab: support disabling job auto-run in upstream gitlab: avoid extra pipelines for tags and stable branches gitlab: stable staging branches publish containers in a separate tag gitlab: allow overriding name of the upstream repository gitlab: centralize the container tag name tests/qtest: Fix a comment typo in vhost-user-test.c Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
2023-06-26 10:38:41 +02:00 · 2023-06-26 10:38:41 +02:00 · 52ed34cbdd
parent f9925abbda b197ea8636
commit 52ed34cbdd
10 changed files with 92 additions and 22 deletions
--- a/.gitlab-ci.d/base.yml
+++ b/.gitlab-ci.d/base.yml
@ -1,10 +1,24 @@

+variables:
+  # On stable branches this is changed by later rules. Should also
+  # be overridden per pipeline if running pipelines concurrently
+  # for different branches in contributor forks.
+  QEMU_CI_CONTAINER_TAG: latest
+
+  # For purposes of CI rules, upstream is the gitlab.com/qemu-project
+  # namespace. When testing CI, it might be usefult to override this
+  # to point to a fork repo
+  QEMU_CI_UPSTREAM: qemu-project
+
 # The order of rules defined here is critically important.
 # They are evaluated in order and first match wins.
 #
 # Thus we group them into a number of stages, ordered from
 # most restrictive to least restrictive
 #
+# For pipelines running for stable "staging-X.Y" branches
+# we must override QEMU_CI_CONTAINER_TAG
+#
 .base_job_template:
  variables:
    # Each script line from will be in a collapsible section in the job output
@ -19,28 +33,36 @@
    # want jobs to run
    #############################################################

+    # Never run jobs upstream on stable branch, staging branch jobs already ran
+    - if: '$CI_PROJECT_NAMESPACE == $QEMU_CI_UPSTREAM && $CI_COMMIT_BRANCH =~ /^stable-/'
+      when: never
+
+    # Never run jobs upstream on tags, staging branch jobs already ran
+    - if: '$CI_PROJECT_NAMESPACE == $QEMU_CI_UPSTREAM && $CI_COMMIT_TAG'
+      when: never
+
    # Cirrus jobs can't run unless the creds / target repo are set
    - if: '$QEMU_JOB_CIRRUS && ($CIRRUS_GITHUB_REPO == null || $CIRRUS_API_TOKEN == null)'
      when: never

    # Publishing jobs should only run on the default branch in upstream
-    - if: '$QEMU_JOB_PUBLISH == "1" && $CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH'
+    - if: '$QEMU_JOB_PUBLISH == "1" && $CI_PROJECT_NAMESPACE == $QEMU_CI_UPSTREAM && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH'
      when: never

    # Non-publishing jobs should only run on staging branches in upstream
-    - if: '$QEMU_JOB_PUBLISH != "1" && $CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH !~ /staging/'
+    - if: '$QEMU_JOB_PUBLISH != "1" && $CI_PROJECT_NAMESPACE == $QEMU_CI_UPSTREAM && $CI_COMMIT_BRANCH !~ /staging/'
      when: never

    # Jobs only intended for forks should always be skipped on upstream
-    - if: '$QEMU_JOB_ONLY_FORKS == "1" && $CI_PROJECT_NAMESPACE == "qemu-project"'
+    - if: '$QEMU_JOB_ONLY_FORKS == "1" && $CI_PROJECT_NAMESPACE == $QEMU_CI_UPSTREAM'
      when: never

    # Forks don't get pipelines unless QEMU_CI=1 or QEMU_CI=2 is set
-    - if: '$QEMU_CI != "1" && $QEMU_CI != "2" && $CI_PROJECT_NAMESPACE != "qemu-project"'
+    - if: '$QEMU_CI != "1" && $QEMU_CI != "2" && $CI_PROJECT_NAMESPACE != $QEMU_CI_UPSTREAM'
      when: never

    # Avocado jobs don't run in forks unless $QEMU_CI_AVOCADO_TESTING is set
-    - if: '$QEMU_JOB_AVOCADO && $QEMU_CI_AVOCADO_TESTING != "1" && $CI_PROJECT_NAMESPACE != "qemu-project"'
+    - if: '$QEMU_JOB_AVOCADO && $QEMU_CI_AVOCADO_TESTING != "1" && $CI_PROJECT_NAMESPACE != $QEMU_CI_UPSTREAM'
      when: never


@ -50,17 +72,29 @@
    #############################################################

    # Optional jobs should not be run unless manually triggered
+    - if: '$QEMU_JOB_OPTIONAL && $CI_PROJECT_NAMESPACE == $QEMU_CI_UPSTREAM && $CI_COMMIT_BRANCH =~ /staging-[[:digit:]]+\.[[:digit:]]/'
+      when: manual
+      allow_failure: true
+      variables:
+        QEMU_CI_CONTAINER_TAG: $CI_COMMIT_REF_SLUG
+
    - if: '$QEMU_JOB_OPTIONAL'
      when: manual
      allow_failure: true

    # Skipped jobs should not be run unless manually triggered
+    - if: '$QEMU_JOB_SKIPPED && $CI_PROJECT_NAMESPACE == $QEMU_CI_UPSTREAM && $CI_COMMIT_BRANCH =~ /staging-[[:digit:]]+\.[[:digit:]]/'
+      when: manual
+      allow_failure: true
+      variables:
+        QEMU_CI_CONTAINER_TAG: $CI_COMMIT_REF_SLUG
+
    - if: '$QEMU_JOB_SKIPPED'
      when: manual
      allow_failure: true

    # Avocado jobs can be manually start in forks if $QEMU_CI_AVOCADO_TESTING is unset
-    - if: '$QEMU_JOB_AVOCADO && $CI_PROJECT_NAMESPACE != "qemu-project"'
+    - if: '$QEMU_JOB_AVOCADO && $CI_PROJECT_NAMESPACE != $QEMU_CI_UPSTREAM'
      when: manual
      allow_failure: true

@ -72,8 +106,23 @@

    # Forks pipeline jobs don't start automatically unless
    # QEMU_CI=2 is set
-    - if: '$QEMU_CI != "2" && $CI_PROJECT_NAMESPACE != "qemu-project"'
+    - if: '$QEMU_CI != "2" && $CI_PROJECT_NAMESPACE != $QEMU_CI_UPSTREAM'
+      when: manual
+
+    # Upstream pipeline jobs start automatically unless told not to
+    # by setting QEMU_CI=1
+    - if: '$QEMU_CI == "1" && $CI_PROJECT_NAMESPACE == $QEMU_CI_UPSTREAM && $CI_COMMIT_BRANCH =~ /staging-[[:digit:]]+\.[[:digit:]]/'
+      when: manual
+      variables:
+        QEMU_CI_CONTAINER_TAG: $CI_COMMIT_REF_SLUG
+
+    - if: '$QEMU_CI == "1" && $CI_PROJECT_NAMESPACE == $QEMU_CI_UPSTREAM'
      when: manual

    # Jobs can run if any jobs they depend on were successful
+    - if: '$QEMU_JOB_SKIPPED && $CI_PROJECT_NAMESPACE == $QEMU_CI_UPSTREAM && $CI_COMMIT_BRANCH =~ /staging-[[:digit:]]+\.[[:digit:]]/'
+      when: on_success
+      variables:
+        QEMU_CI_CONTAINER_TAG: $CI_COMMIT_REF_SLUG
+
    - when: on_success
--- a/.gitlab-ci.d/buildtest-template.yml
+++ b/.gitlab-ci.d/buildtest-template.yml
@ -1,7 +1,7 @@
 .native_build_job_template:
  extends: .base_job_template
  stage: build
-  image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest
+  image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:$QEMU_CI_CONTAINER_TAG
  before_script:
    - JOBS=$(expr $(nproc) + 1)
  script:
@ -40,7 +40,7 @@
 .common_test_job_template:
  extends: .base_job_template
  stage: test
-  image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest
+  image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:$QEMU_CI_CONTAINER_TAG
  script:
    - scripts/git-submodule.sh update roms/SLOF
    - meson subprojects download $(cd build/subprojects && echo *)
--- a/.gitlab-ci.d/buildtest.yml
+++ b/.gitlab-ci.d/buildtest.yml
@ -532,7 +532,7 @@ build-without-defaults:
 build-libvhost-user:
  extends: .base_job_template
  stage: build
-  image: $CI_REGISTRY_IMAGE/qemu/fedora:latest
+  image: $CI_REGISTRY_IMAGE/qemu/fedora:$QEMU_CI_CONTAINER_TAG
  needs:
    job: amd64-fedora-container
  script:
@ -572,7 +572,7 @@ build-tools-and-docs-debian:
 # of what topic branch they're currently using
 pages:
  extends: .base_job_template
-  image: $CI_REGISTRY_IMAGE/qemu/debian-amd64:latest
+  image: $CI_REGISTRY_IMAGE/qemu/debian-amd64:$QEMU_CI_CONTAINER_TAG
  stage: test
  needs:
    - job: build-tools-and-docs-debian
--- a/.gitlab-ci.d/container-template.yml
+++ b/.gitlab-ci.d/container-template.yml
@ -5,7 +5,8 @@
  services:
    - docker:dind
  before_script:
-    - export TAG="$CI_REGISTRY_IMAGE/qemu/$NAME:latest"
+    - export TAG="$CI_REGISTRY_IMAGE/qemu/$NAME:$QEMU_CI_CONTAINER_TAG"
+    # Always ':latest' because we always use upstream as a common cache source
    - export COMMON_TAG="$CI_REGISTRY/qemu-project/qemu/qemu/$NAME:latest"
    - docker login $CI_REGISTRY -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD"
    - until docker info; do sleep 1; done
--- a/.gitlab-ci.d/crossbuild-template.yml
+++ b/.gitlab-ci.d/crossbuild-template.yml
@ -1,7 +1,7 @@
 .cross_system_build_job:
  extends: .base_job_template
  stage: build
-  image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest
+  image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:$QEMU_CI_CONTAINER_TAG
  timeout: 80m
  script:
    - mkdir build
@ -27,7 +27,7 @@
 .cross_accel_build_job:
  extends: .base_job_template
  stage: build
-  image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest
+  image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:$QEMU_CI_CONTAINER_TAG
  timeout: 30m
  script:
    - mkdir build
@ -39,7 +39,7 @@
 .cross_user_build_job:
  extends: .base_job_template
  stage: build
-  image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest
+  image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:$QEMU_CI_CONTAINER_TAG
  script:
    - mkdir build
    - cd build
--- a/.gitlab-ci.d/static_checks.yml
+++ b/.gitlab-ci.d/static_checks.yml
@ -26,7 +26,7 @@ check-dco:
 check-python-minreqs:
  extends: .base_job_template
  stage: test
-  image: $CI_REGISTRY_IMAGE/qemu/python:latest
+  image: $CI_REGISTRY_IMAGE/qemu/python:$QEMU_CI_CONTAINER_TAG
  script:
    - make -C python check-minreqs
  variables:
@ -37,7 +37,7 @@ check-python-minreqs:
 check-python-tox:
  extends: .base_job_template
  stage: test
-  image: $CI_REGISTRY_IMAGE/qemu/python:latest
+  image: $CI_REGISTRY_IMAGE/qemu/python:$QEMU_CI_CONTAINER_TAG
  script:
    - make -C python check-tox
  variables:
--- a/.gitlab-ci.d/windows.yml
+++ b/.gitlab-ci.d/windows.yml
@ -11,6 +11,13 @@
  needs: []
  stage: build
  timeout: 80m
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
+    expire_in: 7 days
+    paths:
+      - build/meson-logs/testlog.txt
+    reports:
+      junit: "build/meson-logs/testlog.junit.xml"
  before_script:
  - If ( !(Test-Path -Path msys64\var\cache ) ) {
      mkdir msys64\var\cache
@ -65,8 +72,8 @@ msys2-64bit:
  - $env:CHERE_INVOKING = 'yes'  # Preserve the current working directory
  - $env:MSYSTEM = 'MINGW64'     # Start a 64-bit MinGW environment
  - $env:MSYS = 'winsymlinks:native' # Enable native Windows symlink
-  - mkdir output
-  - cd output
+  - mkdir build
+  - cd build
  # Note: do not remove "--without-default-devices"!
  # commit 9f8e6cad65a6 ("gitlab-ci: Speed up the msys2-64bit job by using --without-default-devices"
  # changed to compile QEMU with the --without-default-devices switch
@ -115,8 +122,8 @@ msys2-32bit:
  - $env:CHERE_INVOKING = 'yes'  # Preserve the current working directory
  - $env:MSYSTEM = 'MINGW32'     # Start a 32-bit MinGW environment
  - $env:MSYS = 'winsymlinks:native' # Enable native Windows symlink
-  - mkdir output
-  - cd output
+  - mkdir build
+  - cd build
  - ..\msys64\usr\bin\bash -lc '../configure --target-list=ppc64-softmmu
                                --enable-fdt=system'
  - ..\msys64\usr\bin\bash -lc 'make'
--- a/docs/devel/ci-jobs.rst.inc
+++ b/docs/devel/ci-jobs.rst.inc
@ -70,6 +70,17 @@ in a handful of namespaces
   repository CI settings, or as git push variables, to influence
   which jobs get run in a pipeline

+ * QEMU_CI_CONTAINER_TAG - the tag used to publish containers
+   in stage 1, for use by build jobs in stage 2. Defaults to
+   'latest', but if running pipelines for different branches
+   concurrently, it should be overridden per pipeline.
+
+ * QEMU_CI_UPSTREAM - gitlab namespace that is considered to be
+   the 'upstream'. This defaults to 'qemu-project'. Contributors
+   may choose to override this if they are modifying rules in
+   base.yml and need to validate how they will operate when in
+   an upstream context, as opposed to their fork context.
+
 * nnn - other misc variables not falling into the above
   categories, or using different names for historical reasons
   and not yet converted.
--- a/tests/qtest/cxl-test.c
+++ b/tests/qtest/cxl-test.c
@ -124,6 +124,7 @@ static void cxl_t3d_deprecated(void)

    qtest_start(cmdline->str);
    qtest_end();
+    rmdir(tmpfs);
 }

 static void cxl_t3d_persistent(void)
@ -138,6 +139,7 @@ static void cxl_t3d_persistent(void)

    qtest_start(cmdline->str);
    qtest_end();
+    rmdir(tmpfs);
 }

 static void cxl_t3d_volatile(void)
--- a/tests/qtest/vhost-user-test.c
+++ b/tests/qtest/vhost-user-test.c
@ -281,7 +281,7 @@ static void read_guest_mem_server(QTestState *qts, TestServer *s)
    /* iterate all regions */
    for (i = 0; i < s->fds_num; i++) {

-        /* We'll check only the region statring at 0x0*/
+        /* We'll check only the region starting at 0x0 */
        if (s->memory.regions[i].guest_phys_addr != 0x0) {
            continue;
        }