From c831f5759f94d7545da8a8afdf3ec2afdfe3e803 Mon Sep 17 00:00:00 2001 From: Stenzek Date: Thu, 25 Apr 2024 16:55:51 +1000 Subject: [PATCH] CI: Simplify Linux build --- .github/workflows/linux_build_qt.yml | 40 +++++++-- .../scripts/common/update_base_translation.sh | 3 +- .../workflows/scripts/linux/appimage-qt.sh | 8 +- .../scripts/linux/build-dependencies-qt.sh | 44 +--------- .github/workflows/scripts/linux/compile.sh | 15 ---- .../linux/flatpak/net.pcsx2.PCSX2.json | 43 +++++----- .github/workflows/scripts/linux/functions.sh | 16 ---- .../scripts/linux/generate-cmake-qt.sh | 46 ---------- .../scripts/linux/install-packages-qt.sh | 85 ------------------- cmake/SearchForStuff.cmake | 2 +- tools/retry.sh | 13 +++ 11 files changed, 76 insertions(+), 239 deletions(-) delete mode 100755 .github/workflows/scripts/linux/compile.sh delete mode 100755 .github/workflows/scripts/linux/functions.sh delete mode 100755 .github/workflows/scripts/linux/generate-cmake-qt.sh delete mode 100755 .github/workflows/scripts/linux/install-packages-qt.sh create mode 100755 tools/retry.sh diff --git a/.github/workflows/linux_build_qt.yml b/.github/workflows/linux_build_qt.yml index a20a55afb4..fc1540e641 100644 --- a/.github/workflows/linux_build_qt.yml +++ b/.github/workflows/linux_build_qt.yml @@ -107,9 +107,18 @@ jobs: restore-keys: ${{ inputs.os }} ${{ inputs.platform }} ${{ inputs.compiler }} ${{ inputs.detail }} ccache - name: Install Packages - env: - COMPILER: ${{ inputs.compiler }} - run: .github/workflows/scripts/linux/install-packages-qt.sh + run: | + tools/retry.sh wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc + sudo tools/retry.sh apt-add-repository -n 'deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main' + sudo tools/retry.sh apt-get update + sudo tools/retry.sh apt-get -y install \ + build-essential ccache clang-17 cmake curl extra-cmake-modules git libasound2-dev libaio-dev libavcodec-dev libavformat-dev libavutil-dev \ + libcurl4-openssl-dev libdbus-1-dev libdecor-0-dev libegl-dev libevdev-dev libfontconfig-dev libfreetype-dev libfuse2 libgtk-3-dev libgudev-1.0-dev \ + libharfbuzz-dev libinput-dev libopengl-dev libpcap-dev libpipewire-0.3-dev libpulse-dev libssl-dev libswresample-dev libswscale-dev libudev-dev \ + libwayland-dev libx11-dev libx11-xcb-dev libxcb1-dev libxcb-composite0-dev libxcb-cursor-dev libxcb-damage0-dev libxcb-glx0-dev libxcb-icccm4-dev \ + libxcb-image0-dev libxcb-keysyms1-dev libxcb-present-dev libxcb-randr0-dev libxcb-render0-dev libxcb-render-util0-dev libxcb-shape0-dev \ + libxcb-shm0-dev libxcb-sync-dev libxcb-util-dev libxcb-xfixes0-dev libxcb-xinput-dev libxcb-xkb-dev libxext-dev libxkbcommon-x11-dev libxrandr-dev \ + lld-17 llvm-17 ninja-build patchelf pkg-config zlib1g-dev - name: Cache Dependencies id: cache-deps @@ -129,16 +138,31 @@ jobs: - name: Generate CMake env: - COMPILER: ${{ inputs.compiler }} ADDITIONAL_CMAKE_ARGS: ${{ inputs.cmakeflags }} - CLANG_PATH: /usr/bin/clang-17 - CLANGXX_PATH: /usr/bin/clang++-17 run: | - DEPS_PREFIX="$HOME/deps" .github/workflows/scripts/linux/generate-cmake-qt.sh + cmake -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON \ + -DCMAKE_PREFIX_PATH="$HOME/deps" \ + -DCMAKE_C_COMPILER=clang-17 \ + -DCMAKE_CXX_COMPILER=clang++-17 \ + -DCMAKE_EXE_LINKER_FLAGS_INIT="-fuse-ld=lld" \ + -DCMAKE_MODULE_LINKER_FLAGS_INIT="-fuse-ld=lld" \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DENABLE_SETCAP=OFF \ + -DDISABLE_ADVANCE_SIMD=TRUE \ + $ADDITIONAL_CMAKE_ARGS - name: Build PCSX2 working-directory: build - run: ../.github/workflows/scripts/linux/compile.sh + run: | + # Prepare the Cache + ccache -p + ccache -z + # Build + ninja + # Save the Cache + ccache -s - name: Run Tests working-directory: ./build diff --git a/.github/workflows/scripts/common/update_base_translation.sh b/.github/workflows/scripts/common/update_base_translation.sh index 71f8f8eb44..544b915492 100755 --- a/.github/workflows/scripts/common/update_base_translation.sh +++ b/.github/workflows/scripts/common/update_base_translation.sh @@ -1,12 +1,11 @@ #!/bin/bash SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") -source "$SCRIPTDIR/../linux/functions.sh" set -e # While we use custom Qt builds for our releases, the Qt6 package will be good enough # for just updating translations. Saves building it for this action alone. -retry_command sudo apt-get -y install qt6-l10n-tools +"$SCRIPTDIR/../../../../tools/retry.sh" sudo apt-get -y install qt6-l10n-tools PATH=/usr/lib/qt6/bin:$PATH "$SCRIPTDIR/../../../../pcsx2-qt/Translations/update_en_translation.sh" diff --git a/.github/workflows/scripts/linux/appimage-qt.sh b/.github/workflows/scripts/linux/appimage-qt.sh index 76dc35f235..8fc4c14340 100755 --- a/.github/workflows/scripts/linux/appimage-qt.sh +++ b/.github/workflows/scripts/linux/appimage-qt.sh @@ -26,7 +26,6 @@ # For more information, please refer to SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") -source "$SCRIPTDIR/functions.sh" if [ "$#" -ne 4 ]; then echo "Syntax: $0 " @@ -60,17 +59,17 @@ APPIMAGETOOL=./appimagetool-x86_64.AppImage PATCHELF=patchelf if [ ! -f "$LINUXDEPLOY" ]; then - retry_command wget -O "$LINUXDEPLOY" https://github.com/linuxdeploy/linuxdeploy/releases/download/continuous/linuxdeploy-x86_64.AppImage + "$PCSX2DIR/tools/retry.sh" wget -O "$LINUXDEPLOY" https://github.com/linuxdeploy/linuxdeploy/releases/download/continuous/linuxdeploy-x86_64.AppImage chmod +x "$LINUXDEPLOY" fi if [ ! -f "$LINUXDEPLOY_PLUGIN_QT" ]; then - retry_command wget -O "$LINUXDEPLOY_PLUGIN_QT" https://github.com/linuxdeploy/linuxdeploy-plugin-qt/releases/download/continuous/linuxdeploy-plugin-qt-x86_64.AppImage + "$PCSX2DIR/tools/retry.sh" wget -O "$LINUXDEPLOY_PLUGIN_QT" https://github.com/linuxdeploy/linuxdeploy-plugin-qt/releases/download/continuous/linuxdeploy-plugin-qt-x86_64.AppImage chmod +x "$LINUXDEPLOY_PLUGIN_QT" fi if [ ! -f "$APPIMAGETOOL" ]; then - retry_command wget -O "$APPIMAGETOOL" https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-x86_64.AppImage + "$PCSX2DIR/tools/retry.sh" wget -O "$APPIMAGETOOL" https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-x86_64.AppImage chmod +x "$APPIMAGETOOL" fi @@ -100,6 +99,7 @@ cp "$PCSX2DIR/bin/resources/icons/AppIconLarge.png" "PCSX2.png" echo "Running linuxdeploy to create AppDir..." EXTRA_QT_PLUGINS="core;gui;network;svg;waylandclient;widgets;xcbqpa" \ EXTRA_PLATFORM_PLUGINS="libqwayland-egl.so;libqwayland-generic.so" \ +DEPLOY_PLATFORM_THEMES="1" \ QMAKE="$DEPSDIR/bin/qmake" \ NO_STRIP="1" \ $LINUXDEPLOY --plugin qt --appdir="$OUTDIR" --executable="$BUILDDIR/bin/pcsx2-qt" \ diff --git a/.github/workflows/scripts/linux/build-dependencies-qt.sh b/.github/workflows/scripts/linux/build-dependencies-qt.sh index 3240628880..31bc6b8ecf 100755 --- a/.github/workflows/scripts/linux/build-dependencies-qt.sh +++ b/.github/workflows/scripts/linux/build-dependencies-qt.sh @@ -14,8 +14,6 @@ if [ "${INSTALLDIR:0:1}" != "/" ]; then INSTALLDIR="$PWD/$INSTALLDIR" fi -FREETYPE=2.13.2 -HARFBUZZ=8.3.1 LIBBACKTRACE=ad106d5fdd5d960bd33fae1c48a351af567fd075 LIBJPEG=9f LIBPNG=1.6.43 @@ -23,7 +21,6 @@ LIBWEBP=1.3.2 LZ4=b8fd2d15309dd4e605070bd4486e26b6ef814e29 SDL=SDL2-2.30.2 QT=6.7.0 -ZLIB=1.3.1 ZSTD=1.5.5 SHADERC=2024.0 @@ -35,15 +32,12 @@ mkdir -p deps-build cd deps-build cat > SHASUMS < "clang-toolchain.cmake" << EOF -set(CMAKE_C_COMPILER "${CLANG_PATH}") -set(CMAKE_CXX_COMPILER "${CLANGXX_PATH}") -set(CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld") -set(CMAKE_MODULE_LINKER_FLAGS_INIT "-fuse-ld=lld") -set(CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld") -EOF - ADDITIONAL_CMAKE_ARGS="$ADDITIONAL_CMAKE_ARGS -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON -DCMAKE_TOOLCHAIN_FILE=clang-toolchain.cmake" -fi - -echo "Additional CMake Args - ${ADDITIONAL_CMAKE_ARGS}" - -# Generate CMake into ./build -# DISABLE_ADVANCE_SIMD is needed otherwise we end up doing -march=native. - -# shellcheck disable=SC2086 -cmake \ - -B build \ - -G Ninja \ - $ADDITIONAL_CMAKE_ARGS \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_BUILD_TYPE=Release \ - -DX11_API=ON \ - -DWAYLAND_API=ON \ - -DENABLE_SETCAP=OFF \ - -DCMAKE_PREFIX_PATH="${DEPS_PREFIX}" \ - -DDISABLE_ADVANCE_SIMD=TRUE - diff --git a/.github/workflows/scripts/linux/install-packages-qt.sh b/.github/workflows/scripts/linux/install-packages-qt.sh deleted file mode 100755 index c7cecf28d3..0000000000 --- a/.github/workflows/scripts/linux/install-packages-qt.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash - -SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") -source "$SCRIPTDIR/functions.sh" - -set -e - -# Packages - Build and Qt -declare -a BUILD_PACKAGES=( - "build-essential" - "g++" - "git" - "cmake" - "ccache" - "ninja-build" - "patchelf" - "libfuse2" - "libglib2.0-dev" - "libfontconfig1-dev" - "libharfbuzz-dev" - "libjpeg-dev" - "liblz4-dev" - "libpng-dev" - "libfreetype-dev" - "libinput-dev" - "libwebp-dev" - "libxcb-*-dev" - "libxkbcommon-dev" - "libxkbcommon-x11-dev" - "libxrender-dev" - "libwayland-dev" - "libgl1-mesa-dev" - "libegl-dev" - "libegl1-mesa-dev" - "libgl1-mesa-dev" - "libssl-dev" -) - -# Packages - PCSX2 -declare -a PCSX2_PACKAGES=( - "extra-cmake-modules" - "libaio-dev" - "libasound2-dev" - "libbz2-dev" - "libcurl4-openssl-dev" - "libdbus-1-dev" - "libegl1-mesa-dev" - "libgl1-mesa-dev" - "libgtk-3-dev" - "libharfbuzz-dev" - "libjpeg-dev" - "libpcap0.8-dev" - "libpng-dev" - "libpulse-dev" - "librsvg2-dev" - "libsamplerate0-dev" - "libudev-dev" - "libx11-xcb-dev" - "libavcodec-dev" - "libavformat-dev" - "libavutil-dev" - "libswresample-dev" - "libswscale-dev" - "pkg-config" - "zlib1g-dev" -) - -if [ "${COMPILER}" = "clang" ]; then - BUILD_PACKAGES+=("llvm-17" "lld-17" "clang-17") - - # Ubuntu 22.04 doesn't ship with LLVM 16, so we need to pull it from the llvm.org repos. - retry_command wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - sudo apt-add-repository -n 'deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main' -fi - -retry_command sudo apt-get -qq update && break - -# Install packages needed for building -echo "Will install the following packages for building - ${BUILD_PACKAGES[*]}" -retry_command sudo apt-get -y install "${BUILD_PACKAGES[@]}" - -# Install packages needed by pcsx2 -PCSX2_PACKAGES=("${PCSX2_PACKAGES[@]}") -echo "Will install the following packages for pcsx2 - ${PCSX2_PACKAGES[*]}" -retry_command sudo apt-get -y install "${PCSX2_PACKAGES[@]}" diff --git a/cmake/SearchForStuff.cmake b/cmake/SearchForStuff.cmake index e760556410..29b7d80a25 100644 --- a/cmake/SearchForStuff.cmake +++ b/cmake/SearchForStuff.cmake @@ -18,7 +18,7 @@ find_package(Zstd 1.5.5 REQUIRED) find_package(LZ4 REQUIRED) find_package(WebP REQUIRED) # v1.3.2, spews an error on Linux because no pkg-config. find_package(SDL2 2.30.2 REQUIRED) -find_package(Freetype 2.13.1 REQUIRED) +find_package(Freetype 2.11.1 REQUIRED) if(USE_VULKAN) find_package(Shaderc REQUIRED) diff --git a/tools/retry.sh b/tools/retry.sh new file mode 100755 index 0000000000..e4de903b6f --- /dev/null +++ b/tools/retry.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +RETRIES=10 + +for i in $(seq 1 "$RETRIES"); do + "$@" && break + if [ "$i" == "$RETRIES" ]; then + echo "Command \"$@\" failed after ${RETRIES} retries." + exit 1 + fi +done + +exit 0