From c5b5288c3da9c7728b2d10b577bc31beb231fb8a Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Fri, 28 Jul 2023 16:27:45 +0200 Subject: [PATCH 1/9] util/oslib-win32: Fix compiling with Clang from MSYS2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clang complains: ../util/oslib-win32.c:483:56: error: omitting the parameter name in a function definition is a C2x extension [-Werror,-Wc2x-extensions] win32_close_exception_handler(struct _EXCEPTION_RECORD*, ^ Fix it by adding parameter names. Message-Id: <20230728142748.305341-4-thuth@redhat.com> Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Thomas Huth --- util/oslib-win32.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/util/oslib-win32.c b/util/oslib-win32.c index 429542face..19a0ea7fbe 100644 --- a/util/oslib-win32.c +++ b/util/oslib-win32.c @@ -480,8 +480,9 @@ int qemu_bind_wrap(int sockfd, const struct sockaddr *addr, } EXCEPTION_DISPOSITION -win32_close_exception_handler(struct _EXCEPTION_RECORD*, - void*, struct _CONTEXT*, void*) +win32_close_exception_handler(struct _EXCEPTION_RECORD *exception_record, + void *registration, struct _CONTEXT *context, + void *dispatcher) { return EXCEPTION_EXECUTE_HANDLER; } From 11961d08fcbddf37d2393c412271d714900c02cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Tue, 1 Aug 2023 14:03:56 +0100 Subject: [PATCH 2/9] gitlab: remove duplication between msys jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Although they share a common parent, the two msys jobs still have massive duplication in their script definitions that can easily be collapsed. Signed-off-by: Daniel P. Berrangé Reviewed-by: Thomas Huth Message-Id: <20230801130403.164060-2-berrange@redhat.com> Signed-off-by: Thomas Huth --- .gitlab-ci.d/windows.yml | 142 +++++++++++++++------------------------ 1 file changed, 54 insertions(+), 88 deletions(-) diff --git a/.gitlab-ci.d/windows.yml b/.gitlab-ci.d/windows.yml index f889a468b5..f086540e40 100644 --- a/.gitlab-ci.d/windows.yml +++ b/.gitlab-ci.d/windows.yml @@ -35,97 +35,63 @@ - .\msys64\usr\bin\bash -lc 'pacman --noconfirm -Syuu' # Core update - .\msys64\usr\bin\bash -lc 'pacman --noconfirm -Syuu' # Normal update - taskkill /F /FI "MODULES eq msys-2.0.dll" + script: + - .\msys64\usr\bin\bash -lc "pacman -Sy --noconfirm --needed + bison diffutils flex + git grep make sed + $MINGW_TARGET-capstone + $MINGW_TARGET-curl + $MINGW_TARGET-cyrus-sasl + $MINGW_TARGET-dtc + $MINGW_TARGET-gcc + $MINGW_TARGET-glib2 + $MINGW_TARGET-gnutls + $MINGW_TARGET-gtk3 + $MINGW_TARGET-libgcrypt + $MINGW_TARGET-libjpeg-turbo + $MINGW_TARGET-libnfs + $MINGW_TARGET-libpng + $MINGW_TARGET-libssh + $MINGW_TARGET-libtasn1 + $MINGW_TARGET-libusb + $MINGW_TARGET-lzo2 + $MINGW_TARGET-nettle + $MINGW_TARGET-ninja + $MINGW_TARGET-pixman + $MINGW_TARGET-pkgconf + $MINGW_TARGET-python + $MINGW_TARGET-SDL2 + $MINGW_TARGET-SDL2_image + $MINGW_TARGET-snappy + $MINGW_TARGET-spice + $MINGW_TARGET-usbredir + $MINGW_TARGET-zstd " + - $env:CHERE_INVOKING = 'yes' # Preserve the current working directory + - $env:MSYS = 'winsymlinks:native' # Enable native Windows symlink + - mkdir build + - cd build + - ..\msys64\usr\bin\bash -lc "../configure --enable-fdt=system $CONFIGURE_ARGS" + - ..\msys64\usr\bin\bash -lc "make" + - ..\msys64\usr\bin\bash -lc "make check MTESTARGS='$TEST_ARGS' || { cat meson-logs/testlog.txt; exit 1; } ;" msys2-64bit: extends: .shared_msys2_builder - script: - - .\msys64\usr\bin\bash -lc "pacman -Sy --noconfirm --needed - bison diffutils flex - git grep make sed - mingw-w64-x86_64-capstone - mingw-w64-x86_64-curl - mingw-w64-x86_64-cyrus-sasl - mingw-w64-x86_64-dtc - mingw-w64-x86_64-gcc - mingw-w64-x86_64-glib2 - mingw-w64-x86_64-gnutls - mingw-w64-x86_64-gtk3 - mingw-w64-x86_64-libgcrypt - mingw-w64-x86_64-libjpeg-turbo - mingw-w64-x86_64-libnfs - mingw-w64-x86_64-libpng - mingw-w64-x86_64-libssh - mingw-w64-x86_64-libtasn1 - mingw-w64-x86_64-libusb - mingw-w64-x86_64-lzo2 - mingw-w64-x86_64-nettle - mingw-w64-x86_64-ninja - mingw-w64-x86_64-pixman - mingw-w64-x86_64-pkgconf - mingw-w64-x86_64-python - mingw-w64-x86_64-SDL2 - mingw-w64-x86_64-SDL2_image - mingw-w64-x86_64-snappy - mingw-w64-x86_64-spice - mingw-w64-x86_64-usbredir - mingw-w64-x86_64-zstd " - - $env:CHERE_INVOKING = 'yes' # Preserve the current working directory - - $env:MSYSTEM = 'MINGW64' # Start a 64-bit MinGW environment - - $env:MSYS = 'winsymlinks:native' # Enable native Windows symlink - - mkdir build - - cd build - # Note: do not remove "--without-default-devices"! - # commit 9f8e6cad65a6 ("gitlab-ci: Speed up the msys2-64bit job by using --without-default-devices" - # changed to compile QEMU with the --without-default-devices switch - # for the msys2 64-bit job, due to the build could not complete within - # the project timeout. - - ..\msys64\usr\bin\bash -lc '../configure --target-list=x86_64-softmmu - --without-default-devices --enable-fdt=system' - - ..\msys64\usr\bin\bash -lc 'make' - # qTests don't run successfully with "--without-default-devices", - # so let's exclude the qtests from CI for now. - - ..\msys64\usr\bin\bash -lc 'make check MTESTARGS=\"--no-suite qtest\" || { cat meson-logs/testlog.txt; exit 1; } ;' + variables: + MINGW_TARGET: mingw-w64-x86_64 + MSYSTEM: MINGW64 + # do not remove "--without-default-devices"! + # commit 9f8e6cad65a6 ("gitlab-ci: Speed up the msys2-64bit job by using --without-default-devices" + # changed to compile QEMU with the --without-default-devices switch + # for the msys2 64-bit job, due to the build could not complete within + CONFIGURE_ARGS: --target-list=x86_64-softmmu --without-default-devices + # qTests don't run successfully with "--without-default-devices", + # so let's exclude the qtests from CI for now. + TEST_ARGS: --no-suite qtest msys2-32bit: extends: .shared_msys2_builder - script: - - .\msys64\usr\bin\bash -lc "pacman -Sy --noconfirm --needed - bison diffutils flex - git grep make sed - mingw-w64-i686-capstone - mingw-w64-i686-curl - mingw-w64-i686-cyrus-sasl - mingw-w64-i686-dtc - mingw-w64-i686-gcc - mingw-w64-i686-glib2 - mingw-w64-i686-gnutls - mingw-w64-i686-gtk3 - mingw-w64-i686-libgcrypt - mingw-w64-i686-libjpeg-turbo - mingw-w64-i686-libnfs - mingw-w64-i686-libpng - mingw-w64-i686-libssh - mingw-w64-i686-libtasn1 - mingw-w64-i686-libusb - mingw-w64-i686-lzo2 - mingw-w64-i686-nettle - mingw-w64-i686-ninja - mingw-w64-i686-pixman - mingw-w64-i686-pkgconf - mingw-w64-i686-python - mingw-w64-i686-SDL2 - mingw-w64-i686-SDL2_image - mingw-w64-i686-snappy - mingw-w64-i686-spice - mingw-w64-i686-usbredir - mingw-w64-i686-zstd " - - $env:CHERE_INVOKING = 'yes' # Preserve the current working directory - - $env:MSYSTEM = 'MINGW32' # Start a 32-bit MinGW environment - - $env:MSYS = 'winsymlinks:native' # Enable native Windows symlink - - mkdir build - - cd build - - ..\msys64\usr\bin\bash -lc '../configure --target-list=ppc64-softmmu - --enable-fdt=system' - - ..\msys64\usr\bin\bash -lc 'make' - - ..\msys64\usr\bin\bash -lc 'make check MTESTARGS=\"--no-suite qtest\" || - { cat meson-logs/testlog.txt; exit 1; }' + variables: + MINGW_TARGET: mingw-w64-i686 + MSYSTEM: MINGW32 + CONFIGURE_ARGS: --target-list=ppc64-softmmu + TEST_ARGS: --no-suite qtest From b64052cdada197a06fd7ffe1ba239d4de8b89d45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Tue, 1 Aug 2023 14:03:57 +0100 Subject: [PATCH 3/9] gitlab: print timestamps during windows msys jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is hard to get visibility into where time is consumed in our Windows msys jobs. Adding a few log console messages with the timestamp will aid in our debugging. Signed-off-by: Daniel P. Berrangé Reviewed-by: Thomas Huth Message-Id: <20230801130403.164060-3-berrange@redhat.com> Signed-off-by: Thomas Huth --- .gitlab-ci.d/windows.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab-ci.d/windows.yml b/.gitlab-ci.d/windows.yml index f086540e40..831b080d12 100644 --- a/.gitlab-ci.d/windows.yml +++ b/.gitlab-ci.d/windows.yml @@ -19,6 +19,7 @@ reports: junit: "build/meson-logs/testlog.junit.xml" before_script: + - Write-Output "Acquiring msys2.exe installer at $(Get-Date -Format u)" - If ( !(Test-Path -Path msys64\var\cache ) ) { mkdir msys64\var\cache } @@ -27,6 +28,7 @@ "https://github.com/msys2/msys2-installer/releases/download/2022-06-03/msys2-base-x86_64-20220603.sfx.exe" -outfile "msys64\var\cache\msys2.exe" } + - Write-Output "Invoking msys2.exe installer at $(Get-Date -Format u)" - msys64\var\cache\msys2.exe -y - ((Get-Content -path .\msys64\etc\\post-install\\07-pacman-key.post -Raw) -replace '--refresh-keys', '--version') | @@ -36,6 +38,7 @@ - .\msys64\usr\bin\bash -lc 'pacman --noconfirm -Syuu' # Normal update - taskkill /F /FI "MODULES eq msys-2.0.dll" script: + - Write-Output "Installing mingw packages at $(Get-Date -Format u)" - .\msys64\usr\bin\bash -lc "pacman -Sy --noconfirm --needed bison diffutils flex git grep make sed @@ -66,6 +69,7 @@ $MINGW_TARGET-spice $MINGW_TARGET-usbredir $MINGW_TARGET-zstd " + - Write-Output "Running build at $(Get-Date -Format u)" - $env:CHERE_INVOKING = 'yes' # Preserve the current working directory - $env:MSYS = 'winsymlinks:native' # Enable native Windows symlink - mkdir build @@ -73,6 +77,7 @@ - ..\msys64\usr\bin\bash -lc "../configure --enable-fdt=system $CONFIGURE_ARGS" - ..\msys64\usr\bin\bash -lc "make" - ..\msys64\usr\bin\bash -lc "make check MTESTARGS='$TEST_ARGS' || { cat meson-logs/testlog.txt; exit 1; } ;" + - Write-Output "Finished build at $(Get-Date -Format u)" msys2-64bit: extends: .shared_msys2_builder From 46aedd34b459a9e0ff7f61d554368b27910dd56d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Tue, 1 Aug 2023 14:03:58 +0100 Subject: [PATCH 4/9] gitlab: always use updated msys installer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We current reference an msys installer binary from mid-2022, which means after installation, it immediately has to re-download a bunch of newer content. This wastes precious CI time. The msys project publishes an installer binary with a fixed URL that always references the latest content. We cache the downloads in gitlab though and so once downloaded we would never re-fetch the installer leading back to the same problem. To deal with this we also fetch the pgp signature for the installer on every run, and compare that to the previously cached signature. If the signature changes, we re-download the full installer. This ensures we always have the latest installer for msys, while also maximising use of the gitlab cache. Signed-off-by: Daniel P. Berrangé Reviewed-by: Thomas Huth Message-Id: <20230801130403.164060-4-berrange@redhat.com> Signed-off-by: Thomas Huth --- .gitlab-ci.d/windows.yml | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.d/windows.yml b/.gitlab-ci.d/windows.yml index 831b080d12..0bc04ad068 100644 --- a/.gitlab-ci.d/windows.yml +++ b/.gitlab-ci.d/windows.yml @@ -23,10 +23,34 @@ - If ( !(Test-Path -Path msys64\var\cache ) ) { mkdir msys64\var\cache } - - If ( !(Test-Path -Path msys64\var\cache\msys2.exe ) ) { + - Invoke-WebRequest + "https://repo.msys2.org/distrib/msys2-x86_64-latest.sfx.exe.sig" + -outfile "msys2.exe.sig" + - if ( Test-Path -Path msys64\var\cache\msys2.exe.sig ) { + Write-Output "Cached installer sig" ; + if ( ((Get-FileHash msys2.exe.sig).Hash -ne (Get-FileHash msys64\var\cache\msys2.exe.sig).Hash) ) { + Write-Output "Mis-matched installer sig, new installer download required" ; + Remove-Item -Path msys64\var\cache\msys2.exe.sig ; + if ( Test-Path -Path msys64\var\cache\msys2.exe ) { + Remove-Item -Path msys64\var\cache\msys2.exe + } + } else { + Write-Output "Matched installer sig, cached installer still valid" + } + } else { + Write-Output "No cached installer sig, new installer download required" ; + if ( Test-Path -Path msys64\var\cache\msys2.exe ) { + Remove-Item -Path msys64\var\cache\msys2.exe + } + } + - if ( !(Test-Path -Path msys64\var\cache\msys2.exe ) ) { + Write-Output "Fetching latest installer" ; Invoke-WebRequest - "https://github.com/msys2/msys2-installer/releases/download/2022-06-03/msys2-base-x86_64-20220603.sfx.exe" - -outfile "msys64\var\cache\msys2.exe" + "https://repo.msys2.org/distrib/msys2-x86_64-latest.sfx.exe" + -outfile "msys64\var\cache\msys2.exe" ; + Copy-Item -Path msys2.exe.sig -Destination msys64\var\cache\msys2.exe.sig + } else { + Write-Output "Using cached installer" } - Write-Output "Invoking msys2.exe installer at $(Get-Date -Format u)" - msys64\var\cache\msys2.exe -y From 5ef56e3b18b185df12fea0901efa0843058572d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Tue, 1 Aug 2023 14:03:59 +0100 Subject: [PATCH 5/9] gitlab: drop $CI_PROJECT_DIR from cache path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gitlab cache is limited to only handle content within the $CI_PROJECT_DIR hierarchy, and as such relative paths are always implicitly relative to $CI_PROJECT_DIR. Signed-off-by: Daniel P. Berrangé Reviewed-by: Thomas Huth Message-Id: <20230801130403.164060-5-berrange@redhat.com> Signed-off-by: Thomas Huth --- .gitlab-ci.d/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.d/windows.yml b/.gitlab-ci.d/windows.yml index 0bc04ad068..6454880cb7 100644 --- a/.gitlab-ci.d/windows.yml +++ b/.gitlab-ci.d/windows.yml @@ -7,7 +7,7 @@ cache: key: "${CI_JOB_NAME}-cache" paths: - - ${CI_PROJECT_DIR}/msys64/var/cache + - msys64/var/cache needs: [] stage: build timeout: 80m From ef4fe31f33648ece16bf86b10262c7424c8470f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Tue, 1 Aug 2023 14:04:00 +0100 Subject: [PATCH 6/9] gitlab: always populate cache for windows msys jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cache is used to hold the msys installer. Even if the build phase fails, we should still populate the cache as the installer will be valid for next time. Signed-off-by: Daniel P. Berrangé Reviewed-by: Thomas Huth Message-Id: <20230801130403.164060-6-berrange@redhat.com> Signed-off-by: Thomas Huth --- .gitlab-ci.d/windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.d/windows.yml b/.gitlab-ci.d/windows.yml index 6454880cb7..34109a80f2 100644 --- a/.gitlab-ci.d/windows.yml +++ b/.gitlab-ci.d/windows.yml @@ -8,6 +8,7 @@ key: "${CI_JOB_NAME}-cache" paths: - msys64/var/cache + when: always needs: [] stage: build timeout: 80m From ff136d2a99253483f263371bb7b479d2278d9401 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Tue, 1 Aug 2023 14:04:01 +0100 Subject: [PATCH 7/9] configure: support passthrough of -Dxxx args to meson MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This can be useful for setting some meson global options, such as the optimization level or debug state.xs Signed-off-by: Daniel P. Berrangé Message-Id: <20230801130403.164060-7-berrange@redhat.com> [thuth: Move the help text into the section with the other --... options] Signed-off-by: Thomas Huth --- configure | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/configure b/configure index 26ec5e4f54..afb25fd558 100755 --- a/configure +++ b/configure @@ -757,6 +757,9 @@ for opt do # everything else has the same name in configure and meson --*) meson_option_parse "$opt" "$optarg" ;; + # Pass through -Dxxxx options to meson + -D*) meson_options="$meson_options $opt" + ;; esac done @@ -846,6 +849,7 @@ $(echo Available targets: $default_target_list | \ --target-list-exclude=LIST exclude a set of targets from the default target-list Advanced options (experts only): + -Dmesonoptname=val passthrough option to meson unmodified --cross-prefix=PREFIX use PREFIX for compile tools, PREFIX can be blank [$cross_prefix] --cc=CC use C compiler CC [$cc] --host-cc=CC use C compiler CC [$host_cc] for code run at From 63f5365cd4a86b3e1311093c01e9ab01b37acee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Tue, 1 Aug 2023 14:04:02 +0100 Subject: [PATCH 8/9] gitlab: disable optimization and debug symbols in msys build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Building at -O2, adds 33% to the build time, over -O2. IOW a build that takes 45 minutes at -O0, takes 60 minutes at -O2. Turning off debug symbols drops it further, down to 38 minutes. IOW, a "-O2 -g" build is 58% slower than a "-O0" build on msys in the gitlab CI windows shared runners. Signed-off-by: Daniel P. Berrangé Message-Id: <20230801130403.164060-8-berrange@redhat.com> Signed-off-by: Thomas Huth --- .gitlab-ci.d/windows.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.d/windows.yml b/.gitlab-ci.d/windows.yml index 34109a80f2..552e3b751d 100644 --- a/.gitlab-ci.d/windows.yml +++ b/.gitlab-ci.d/windows.yml @@ -113,7 +113,7 @@ msys2-64bit: # commit 9f8e6cad65a6 ("gitlab-ci: Speed up the msys2-64bit job by using --without-default-devices" # changed to compile QEMU with the --without-default-devices switch # for the msys2 64-bit job, due to the build could not complete within - CONFIGURE_ARGS: --target-list=x86_64-softmmu --without-default-devices + CONFIGURE_ARGS: --target-list=x86_64-softmmu --without-default-devices -Ddebug=false -Doptimization=0 # qTests don't run successfully with "--without-default-devices", # so let's exclude the qtests from CI for now. TEST_ARGS: --no-suite qtest @@ -123,5 +123,5 @@ msys2-32bit: variables: MINGW_TARGET: mingw-w64-i686 MSYSTEM: MINGW32 - CONFIGURE_ARGS: --target-list=ppc64-softmmu + CONFIGURE_ARGS: --target-list=ppc64-softmmu -Ddebug=false -Doptimization=0 TEST_ARGS: --no-suite qtest From f54ba56dad0e9cea275e9802915a293f1a8c7d22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Tue, 1 Aug 2023 14:04:03 +0100 Subject: [PATCH 9/9] gitlab: disable FF_SCRIPT_SECTIONS on msys jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The FF_SCRIPT_SECTIONS=1 variable should ordinarily cause output from each line of the job script to be presented in a collapsible section with execution time listed. While it works on Linux shared runners, when used with Windows runners with PowerShell, this option does not create any sections, and actually causes echo'ing of commands to be disabled, making it even worse to debug the jobs. Signed-off-by: Daniel P. Berrangé Acked-by: Thomas Huth Message-Id: <20230801130403.164060-9-berrange@redhat.com> Signed-off-by: Thomas Huth --- .gitlab-ci.d/windows.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.d/windows.yml b/.gitlab-ci.d/windows.yml index 552e3b751d..cd7622a761 100644 --- a/.gitlab-ci.d/windows.yml +++ b/.gitlab-ci.d/windows.yml @@ -12,6 +12,10 @@ needs: [] stage: build timeout: 80m + variables: + # This feature doesn't (currently) work with PowerShell, it stops + # the echo'ing of commands being run and doesn't show any timing + FF_SCRIPT_SECTIONS: 0 artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" expire_in: 7 days