From 9837697b7f3c5c9e534217e1b691e6b2f40a23ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Tue, 12 Mar 2024 17:00:11 +0000 Subject: [PATCH 1/5] gitlab: aggressively avoid extra GIT data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This avoids fetching blobs and tree references for branches we are not going to worry about. Also skip tag references which are similarly not useful and keep the default --prune. This keeps the .git data to around 100M rather than the ~400M even a shallow clone takes. So we can check the savings we also run a quick du while setting up the build. We also have to have special settings of GIT_FETCH_EXTRA_FLAGS for the Windows build, the migration legacy test and the custom runners. In the case of the custom runners we also move the free floating variable to the runner template. Reviewed-by: Manos Pitsidianakis Signed-off-by: Alex Bennée Message-Id: <20240312170011.1688444-1-alex.bennee@linaro.org> --- .gitlab-ci.d/base.yml | 4 ++++ .gitlab-ci.d/buildtest-template.yml | 1 + .gitlab-ci.d/buildtest.yml | 2 ++ .gitlab-ci.d/custom-runners.yml | 5 +++-- .gitlab-ci.d/windows.yml | 2 ++ 5 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.d/base.yml b/.gitlab-ci.d/base.yml index 2dd8a9b57c..bf3d8efab6 100644 --- a/.gitlab-ci.d/base.yml +++ b/.gitlab-ci.d/base.yml @@ -24,6 +24,10 @@ variables: # Each script line from will be in a collapsible section in the job output # and show the duration of each line. FF_SCRIPT_SECTIONS: 1 + # The project has a fairly fat GIT repo so we try and avoid bringing in things + # we don't need. The --filter options avoid blobs and tree references we aren't going to use + # and we also avoid fetching tags. + GIT_FETCH_EXTRA_FLAGS: --filter=blob:none --filter=tree:0 --no-tags --prune --quiet interruptible: true diff --git a/.gitlab-ci.d/buildtest-template.yml b/.gitlab-ci.d/buildtest-template.yml index 4fbfeb6667..22045add80 100644 --- a/.gitlab-ci.d/buildtest-template.yml +++ b/.gitlab-ci.d/buildtest-template.yml @@ -14,6 +14,7 @@ - export CCACHE_DIR="$CCACHE_BASEDIR/ccache" - export CCACHE_MAXSIZE="500M" - export PATH="$CCACHE_WRAPPERSDIR:$PATH" + - du -sh .git - mkdir build - cd build - ccache --zero-stats diff --git a/.gitlab-ci.d/buildtest.yml b/.gitlab-ci.d/buildtest.yml index c7d92fc301..cfdff175c3 100644 --- a/.gitlab-ci.d/buildtest.yml +++ b/.gitlab-ci.d/buildtest.yml @@ -187,6 +187,8 @@ build-previous-qemu: variables: IMAGE: opensuse-leap TARGETS: x86_64-softmmu aarch64-softmmu + # Override the default flags as we need more to grab the old version + GIT_FETCH_EXTRA_FLAGS: --prune --quiet before_script: - export QEMU_PREV_VERSION="$(sed 's/\([0-9.]*\)\.[0-9]*/v\1.0/' VERSION)" - git remote add upstream https://gitlab.com/qemu-project/qemu diff --git a/.gitlab-ci.d/custom-runners.yml b/.gitlab-ci.d/custom-runners.yml index 8e5b9500f4..a0e79acd39 100644 --- a/.gitlab-ci.d/custom-runners.yml +++ b/.gitlab-ci.d/custom-runners.yml @@ -10,13 +10,14 @@ # gitlab-runner. To avoid problems that gitlab-runner can cause while # reusing the GIT repository, let's enable the clone strategy, which # guarantees a fresh repository on each job run. -variables: - GIT_STRATEGY: clone # All custom runners can extend this template to upload the testlog # data as an artifact and also feed the junit report .custom_runner_template: extends: .base_job_template + variables: + GIT_STRATEGY: clone + GIT_FETCH_EXTRA_FLAGS: --no-tags --prune --quiet artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" expire_in: 7 days diff --git a/.gitlab-ci.d/windows.yml b/.gitlab-ci.d/windows.yml index f116b8012d..94834269ec 100644 --- a/.gitlab-ci.d/windows.yml +++ b/.gitlab-ci.d/windows.yml @@ -28,6 +28,8 @@ msys2-64bit: # qTests don't run successfully with "--without-default-devices", # so let's exclude the qtests from CI for now. TEST_ARGS: --no-suite qtest + # The Windows git is a bit older so override the default + GIT_FETCH_EXTRA_FLAGS: --no-tags --prune --quiet artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" expire_in: 7 days From 06f6bbc3bd3a18c32e6b5ec844f0ea909571df9a Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Mon, 18 Mar 2024 15:08:01 +0100 Subject: [PATCH 2/5] tests/avocado: update sbsa-ref firmware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We now have CI job to build those and publish in space with readable urls. Firmware is built using Debian 'bookworm' cross toolchain (gcc 12.2.0). Used versions: - Trusted Firmware v2.10.2 - Tianocore EDK2 stable202402 - Tianocore EDK2 Platforms code commit 085c2fb Signed-off-by: Marcin Juszkiewicz Tested-by: Philippe Mathieu-Daudé Message-Id: <20240318-sbsa-ref-firmware-update-v3-1-1c33b995a538@linaro.org> Signed-off-by: Alex Bennée --- tests/avocado/machine_aarch64_sbsaref.py | 40 +++++++++++++----------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/tests/avocado/machine_aarch64_sbsaref.py b/tests/avocado/machine_aarch64_sbsaref.py index 528c7d2934..cbab793455 100644 --- a/tests/avocado/machine_aarch64_sbsaref.py +++ b/tests/avocado/machine_aarch64_sbsaref.py @@ -1,6 +1,6 @@ # Functional test that boots a Linux kernel and checks the console # -# SPDX-FileCopyrightText: 2023 Linaro Ltd. +# SPDX-FileCopyrightText: 2023-2024 Linaro Ltd. # SPDX-FileContributor: Philippe Mathieu-Daudé # SPDX-FileContributor: Marcin Juszkiewicz # @@ -32,34 +32,36 @@ class Aarch64SbsarefMachine(QemuSystemTest): """ Flash volumes generated using: - - Fedora GNU Toolchain version 13.2.1 20230728 (Red Hat 13.2.1-1) + Toolchain from Debian: + aarch64-linux-gnu-gcc (Debian 12.2.0-14) 12.2.0 - - Trusted Firmware-A - https://github.com/ARM-software/arm-trusted-firmware/tree/7c3ff62d + Used components: + + - Trusted Firmware 2.10.2 + - Tianocore EDK2 stable202402 + - Tianocore EDK2-platforms commit 085c2fb - - Tianocore EDK II - https://github.com/tianocore/edk2/tree/0f9283429dd4 - https://github.com/tianocore/edk2/tree/ad1c0394b177 - https://github.com/tianocore/edk2-platforms/tree/d03a60523a60 """ # Secure BootRom (TF-A code) fs0_xz_url = ( - "https://fileserver.linaro.org/s/rE43RJyTfxPtBkc/" - "download/SBSA_FLASH0.fd.xz" + "https://artifacts.codelinaro.org/artifactory/linaro-419-sbsa-ref/" + "20240313-116475/edk2/SBSA_FLASH0.fd.xz" ) - fs0_xz_hash = "cdb8e4ffdaaa79292b7b465693f9e5fae6b7062d" - tar_xz_path = self.fetch_asset(fs0_xz_url, asset_hash=fs0_xz_hash) + fs0_xz_hash = "637593749cc307dea7dc13265c32e5d020267552f22b18a31850b8429fc5e159" + tar_xz_path = self.fetch_asset(fs0_xz_url, asset_hash=fs0_xz_hash, + algorithm='sha256') archive.extract(tar_xz_path, self.workdir) fs0_path = os.path.join(self.workdir, "SBSA_FLASH0.fd") # Non-secure rom (UEFI and EFI variables) fs1_xz_url = ( - "https://fileserver.linaro.org/s/AGWPDXbcqJTKS4R/" - "download/SBSA_FLASH1.fd.xz" + "https://artifacts.codelinaro.org/artifactory/linaro-419-sbsa-ref/" + "20240313-116475/edk2/SBSA_FLASH1.fd.xz" ) - fs1_xz_hash = "411155ae6984334714dff08d5d628178e790c875" - tar_xz_path = self.fetch_asset(fs1_xz_url, asset_hash=fs1_xz_hash) + fs1_xz_hash = "cb0a5e8cf5e303c5d3dc106cfd5943ffe9714b86afddee7164c69ee1dd41991c" + tar_xz_path = self.fetch_asset(fs1_xz_url, asset_hash=fs1_xz_hash, + algorithm='sha256') archive.extract(tar_xz_path, self.workdir) fs1_path = os.path.join(self.workdir, "SBSA_FLASH1.fd") @@ -96,15 +98,15 @@ class Aarch64SbsarefMachine(QemuSystemTest): # AP Trusted ROM wait_for_console_pattern(self, "Booting Trusted Firmware") - wait_for_console_pattern(self, "BL1: v2.9(release):v2.9") + wait_for_console_pattern(self, "BL1: v2.10.2(release):") wait_for_console_pattern(self, "BL1: Booting BL2") # Trusted Boot Firmware - wait_for_console_pattern(self, "BL2: v2.9(release)") + wait_for_console_pattern(self, "BL2: v2.10.2(release)") wait_for_console_pattern(self, "Booting BL31") # EL3 Runtime Software - wait_for_console_pattern(self, "BL31: v2.9(release)") + wait_for_console_pattern(self, "BL31: v2.10.2(release)") # Non-trusted Firmware wait_for_console_pattern(self, "UEFI firmware (version 1.0") From 21f123f3c2b0fbb0931d20446f19255e2ee9ba07 Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Mon, 18 Mar 2024 15:08:02 +0100 Subject: [PATCH 3/5] tests/avocado: drop virtio-rng from sbsa-ref tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sbsa-ref is supposed to emulate real hardware so virtio-rng-pci does not fit here Signed-off-by: Marcin Juszkiewicz Message-Id: <20240318-sbsa-ref-firmware-update-v3-2-1c33b995a538@linaro.org> Signed-off-by: Alex Bennée --- tests/avocado/machine_aarch64_sbsaref.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/avocado/machine_aarch64_sbsaref.py b/tests/avocado/machine_aarch64_sbsaref.py index cbab793455..259225f15f 100644 --- a/tests/avocado/machine_aarch64_sbsaref.py +++ b/tests/avocado/machine_aarch64_sbsaref.py @@ -132,10 +132,6 @@ class Aarch64SbsarefMachine(QemuSystemTest): cpu, "-drive", f"file={iso_path},format=raw", - "-device", - "virtio-rng-pci,rng=rng0", - "-object", - "rng-random,id=rng0,filename=/dev/urandom", ) self.vm.launch() @@ -179,10 +175,6 @@ class Aarch64SbsarefMachine(QemuSystemTest): cpu, "-drive", f"file={img_path},format=raw", - "-device", - "virtio-rng-pci,rng=rng0", - "-object", - "rng-random,id=rng0,filename=/dev/urandom", ) self.vm.launch() From 385ac766e8507a96ecbb39a4bf1a03c0259a098c Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Mon, 18 Mar 2024 15:08:03 +0100 Subject: [PATCH 4/5] tests/avocado: sbsa-ref: add Alpine tests for misc 'max' setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PAuth makes run timeout on CI so add tests using 'max' without it and with impdef one. Signed-off-by: Marcin Juszkiewicz Tested-by: Philippe Mathieu-Daudé Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20240318-sbsa-ref-firmware-update-v3-3-1c33b995a538@linaro.org> Signed-off-by: Alex Bennée --- tests/avocado/machine_aarch64_sbsaref.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/avocado/machine_aarch64_sbsaref.py b/tests/avocado/machine_aarch64_sbsaref.py index 259225f15f..cf8954d02e 100644 --- a/tests/avocado/machine_aarch64_sbsaref.py +++ b/tests/avocado/machine_aarch64_sbsaref.py @@ -140,18 +140,36 @@ class Aarch64SbsarefMachine(QemuSystemTest): def test_sbsaref_alpine_linux_cortex_a57(self): """ :avocado: tags=cpu:cortex-a57 + :avocado: tags=os:linux """ self.boot_alpine_linux("cortex-a57") def test_sbsaref_alpine_linux_neoverse_n1(self): """ :avocado: tags=cpu:neoverse-n1 + :avocado: tags=os:linux """ self.boot_alpine_linux("neoverse-n1") + def test_sbsaref_alpine_linux_max_pauth_off(self): + """ + :avocado: tags=cpu:max + :avocado: tags=os:linux + """ + self.boot_alpine_linux("max,pauth=off") + + def test_sbsaref_alpine_linux_max_pauth_impdef(self): + """ + :avocado: tags=cpu:max + :avocado: tags=os:linux + """ + self.boot_alpine_linux("max,pauth-impdef=on") + + @skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') def test_sbsaref_alpine_linux_max(self): """ :avocado: tags=cpu:max + :avocado: tags=os:linux """ self.boot_alpine_linux("max") From 55900f5dcc3205b87609d9be452c6c76c48b863b Mon Sep 17 00:00:00 2001 From: Marcin Juszkiewicz Date: Mon, 18 Mar 2024 15:08:04 +0100 Subject: [PATCH 5/5] tests/avocado: sbsa-ref: add OpenBSD tests for misc 'max' setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PAuth makes run timeout on CI so add tests using 'max' without it and with impdef one. Signed-off-by: Marcin Juszkiewicz Tested-by: Philippe Mathieu-Daudé Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20240318-sbsa-ref-firmware-update-v3-4-1c33b995a538@linaro.org> Signed-off-by: Alex Bennée --- tests/avocado/machine_aarch64_sbsaref.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/avocado/machine_aarch64_sbsaref.py b/tests/avocado/machine_aarch64_sbsaref.py index cf8954d02e..98c76c1ff7 100644 --- a/tests/avocado/machine_aarch64_sbsaref.py +++ b/tests/avocado/machine_aarch64_sbsaref.py @@ -203,18 +203,36 @@ class Aarch64SbsarefMachine(QemuSystemTest): def test_sbsaref_openbsd73_cortex_a57(self): """ :avocado: tags=cpu:cortex-a57 + :avocado: tags=os:openbsd """ self.boot_openbsd73("cortex-a57") def test_sbsaref_openbsd73_neoverse_n1(self): """ :avocado: tags=cpu:neoverse-n1 + :avocado: tags=os:openbsd """ self.boot_openbsd73("neoverse-n1") + def test_sbsaref_openbsd73_max_pauth_off(self): + """ + :avocado: tags=cpu:max + :avocado: tags=os:openbsd + """ + self.boot_openbsd73("max,pauth=off") + + @skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') + def test_sbsaref_openbsd73_max_pauth_impdef(self): + """ + :avocado: tags=cpu:max + :avocado: tags=os:openbsd + """ + self.boot_openbsd73("max,pauth-impdef=on") + + @skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') def test_sbsaref_openbsd73_max(self): """ :avocado: tags=cpu:max + :avocado: tags=os:openbsd """ self.boot_openbsd73("max") -