From fea666bd1e10e4a5c21e836eb197c8e0f033a849 Mon Sep 17 00:00:00 2001 From: Tyler Wilding Date: Sat, 14 May 2022 01:56:57 -0400 Subject: [PATCH] ci: make build workflows re-usable, extract common scripts --- .github/workflows/linux-workflow.yml | 233 ------------------ .github/workflows/linux_build.yml | 146 +++++++++++ .github/workflows/linux_build_matrix.yml | 51 ++++ .github/workflows/macos-workflow.yml | 152 ------------ .github/workflows/macos_build.yml | 99 ++++++++ .github/workflows/macos_build_matrix.yml | 17 ++ .../scripts/common/name-artifacts.sh | 46 ++++ .../releases/generate-release-notes/index.js | 14 +- .github/workflows/windows_build_matrix.yml | 57 +++++ .github/workflows/windows_build_qt.yml | 104 ++++++++ .github/workflows/windows_build_wx.yml | 118 +++++++++ 11 files changed, 647 insertions(+), 390 deletions(-) delete mode 100644 .github/workflows/linux-workflow.yml create mode 100644 .github/workflows/linux_build.yml create mode 100644 .github/workflows/linux_build_matrix.yml delete mode 100644 .github/workflows/macos-workflow.yml create mode 100644 .github/workflows/macos_build.yml create mode 100644 .github/workflows/macos_build_matrix.yml create mode 100644 .github/workflows/scripts/common/name-artifacts.sh create mode 100644 .github/workflows/windows_build_matrix.yml create mode 100644 .github/workflows/windows_build_qt.yml create mode 100644 .github/workflows/windows_build_wx.yml diff --git a/.github/workflows/linux-workflow.yml b/.github/workflows/linux-workflow.yml deleted file mode 100644 index 0a581a49f3..0000000000 --- a/.github/workflows/linux-workflow.yml +++ /dev/null @@ -1,233 +0,0 @@ -name: 🐧 Linux Builds - -# Controls when the action will run. Triggers the workflow on push or pull request -# events but only for the master branch -on: - push: - branches: - - master - tags: - - v* - pull_request: - branches: - - master - paths-ignore: - - "**/*.md" - - "**/*.bat" - - "**/*.cmd" - - "**/*.props" - - "**/*.sln" - - ".clang-format" - - ".codacy.yaml" - - ".github/*" - - ".github/workflows/lint-gamedb.yml" - - ".github/workflows/macos-workflow.yml" - - ".github/workflows/pr-triage.yml" - - ".github/workflows/scripts/windows/**" - - ".github/workflows/scripts/validation/**" - - ".github/workflows/windows-workflow.yml" - - ".gitignore" - - "bin/PCSX2_keys.ini.default" - - "build.sh" - - "buildbot.xml" - - "pcsx2/CDVD/Windows/**" - - "pcsx2/DEV9/Win32/**" - - "pcsx2/PAD/Windows/**" - - "pcsx2/SPU2/Windows/**" - - "pcsx2/USB/Win32/**" - - "pcsx2/windows/**" - -jobs: - build: - strategy: - # Prevent one build from failing everything (although maybe those should be included as experimental builds instead) - fail-fast: false - matrix: - include: - - os: ubuntu-18.04 - platform: x64 - compiler: gcc - cmakeflags: -DLTO_PCSX2_CORE=ON - appimage: true - experimental: false - - os: ubuntu-18.04 - platform: x64 - compiler: clang - # Need to disable PCH until cmake 3.17 - # (PCH conflicts with ccache, fixed by https://gitlab.kitware.com/cmake/cmake/-/merge_requests/4400) - cmakeflags: -DCMAKE_DISABLE_PRECOMPILE_HEADERS=ON - detail: " nopch" - appimage: false - experimental: false - - os: ubuntu-18.04 - platform: x64 - compiler: gcc - cmakeflags: -DCMAKE_DISABLE_PRECOMPILE_HEADERS=ON - detail: " nopch" - appimage: false - experimental: false - - os: ubuntu-18.04 - platform: x64 - compiler: gcc - cmakeflags: -DCMAKE_DISABLE_PRECOMPILE_HEADERS=ON -DARCH_FLAG=-march=haswell - detail: " avx2 nopch" - appimage: false - experimental: false - - name: ${{ matrix.compiler }}${{ matrix.detail }} - runs-on: ${{ matrix.os }} - continue-on-error: ${{ matrix.experimental }} - # Set some sort of timeout in the event of run-away builds. We are limited on concurrent jobs so, get rid of them. - timeout-minutes: 60 - - env: - CCACHE_BASEDIR: ${{ github.workspace }} - CCACHE_DIR: ${{ github.workspace }}/.ccache - CCACHE_COMPRESS: true - CCACHE_COMPRESSLEVEL: 9 - CCACHE_MAXSIZE: 100M - SDL: SDL2-2.0.22 - PATCHELF_VERSION: 0.12 - - steps: - # NOTE - useful for debugging - # - name: Dump GitHub context - # env: - # GITHUB_CONTEXT: ${{ toJson(github) }} - # run: | - # echo "$GITHUB_CONTEXT" - # echo ${{ github.event.pull_request.title }} - - - name: Checkout Repository - uses: actions/checkout@v3 - - - name: Checkout Submodules - if: steps.cache-submodules.outputs.cache-hit != 'true' - run: git submodule update --init --recursive -j $(getconf _NPROCESSORS_ONLN) - - # -- SETUP CCACHE - https://cristianadam.eu/20200113/speeding-up-c-plus-plus-github-actions-using-ccache/ - - name: Prepare ccache timestamp - id: ccache_cache_timestamp - shell: cmake -P {0} - run: | - string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC) - message("::set-output name=timestamp::${current_date}") - - - name: ccache cache files - uses: actions/cache@v3 - with: - path: .ccache - key: ${{ matrix.os }} ${{ matrix.platform }} ${{ matrix.compiler }}${{ matrix.detail }} ccache ${{ steps.ccache_cache_timestamp.outputs.timestamp }} - restore-keys: ${{ matrix.os }} ${{ matrix.platform }} ${{ matrix.compiler }}${{ matrix.detail }} ccache - - - name: Prepare Artifact Metadata - id: artifact-metadata - shell: bash - run: | - ARCH=$([ "${{ matrix.platform }}" == x86 ] && echo "32bit" || echo "64bit") - ARTIFACT_NAME="" - if [ ${{ github.event_name }} == "pull_request" ]; then - PR_SHA=$(git rev-parse --short "${{ github.event.pull_request.head.sha }}") - ARTIFACT_NAME="PCSX2-${ARCH}" - if [ ! -z "${{ github.event.pull_request.number }}" ]; then - PR_NUM=${{ github.event.pull_request.number }} - ARTIFACT_NAME="${ARTIFACT_NAME}-pr[${PR_NUM}]" - fi - ARTIFACT_NAME="${ARTIFACT_NAME}-sha[${PR_SHA}]" - if [ ! -z "${{ github.event.pull_request.title }}" ]; then - PR_TITLE=$(echo "${{ github.event.pull_request.title }}" | tr -cd '[a-zA-Z0-9[:space:]]_-') - ARTIFACT_NAME="${ARTIFACT_NAME}-title["${PR_TITLE}"" - fi - else - SHA=$(git rev-parse --short "$GITHUB_SHA") - ARTIFACT_NAME="PCSX2-${ARCH}-sha[${SHA}" - fi - TRIMMED_ARTIFACT_NAME=$(printf "%.199s]" "$ARTIFACT_NAME") - echo "name=$TRIMMED_ARTIFACT_NAME" - echo "##[set-output name=name;]${TRIMMED_ARTIFACT_NAME}" - echo "##[set-output name=arch;]${ARCH}" - - - name: Install Packages - env: - PLATFORM: ${{ matrix.platform }} - COMPILER: ${{ matrix.compiler }} - run: .github/workflows/scripts/linux/install-packages.sh - - - name: Cache Dependencies - uses: actions/cache@v3 - with: - path: | - 3rdparty/${{ env.SDL }} - 3rdparty/patchelf-${{ env.PATCHELF_VERSION }} - key: ${{ matrix.os }} ${{ matrix.platform }} ${{ env.SDL }} patchelf-${{ env.PATCHELF_VERSION }} - - - name: Build Dependencies - run: | - if [[ ! -e 3rdparty/patchelf-${{ env.PATCHELF_VERSION }} ]]; then - curl -sSfL https://github.com/NixOS/patchelf/releases/download/${{ env.PATCHELF_VERSION }}/patchelf-${{ env.PATCHELF_VERSION }}.tar.bz2 | tar -xjC 3rdparty - mv 3rdparty/patchelf-${{ env.PATCHELF_VERSION }}* 3rdparty/patchelf-${{ env.PATCHELF_VERSION }} - cd 3rdparty/patchelf-${{ env.PATCHELF_VERSION }} - ./configure - make && cd ../../ - fi - sudo make -C 3rdparty/patchelf-${{ env.PATCHELF_VERSION }} install - if [[ ! -e 3rdparty/${{ env.SDL }} ]]; then - curl -sL https://libsdl.org/release/${{ env.SDL }}.tar.gz | tar -xzC 3rdparty - cd 3rdparty/${{ env.SDL }} - if [ "${{ matrix.platform }}" == "x86" ]; then - ./configure --build=i686-pc-linux-gnu CC=${{ matrix.compiler }} CFLAGS=-m32 CXXFLAGS=-m32 LDFLAGS=-m32 --prefix=/usr --libdir=/usr/lib/i386-linux-gnu - else - ./configure --prefix=/usr --libdir=/usr/lib/x86_64-linux-gnu - fi - make -j $(getconf _NPROCESSORS_ONLN) && cd ../../ - fi - sudo make -C 3rdparty/${{ env.SDL }} install - - - name: Generate CMake - env: - PLATFORM: ${{ matrix.platform }} - COMPILER: ${{ matrix.compiler }} - ADDITIONAL_CMAKE_ARGS: ${{ matrix.cmakeflags }} - run: .github/workflows/scripts/linux/generate-cmake.sh - - - name: Build PCSX2 - working-directory: build - run: ../.github/workflows/scripts/linux/compile.sh - - - name: Run Tests - working-directory: ./build - run: ninja unittests - - - name: Package AppImage - if: matrix.appimage == true - env: - PLATFORM: ${{ matrix.platform }} - COMPILER: ${{ matrix.compiler }} - name: ${{ steps.artifact-metadata.outputs.name }} - run: .github/workflows/scripts/linux/appimage.sh - - - name: Upload artifact - if: matrix.appimage == true - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.artifact-metadata.outputs.name }} - path: ci-artifacts - - # ---- Release / Tagging related steps ---- - - name: Prepare Build Artifacts - if: github.repository == 'PCSX2/pcsx2' && startsWith(github.ref, 'refs/tags/') && matrix.appimage == true - run: | - mv ./ci-artifacts/*.AppImage ./ci-artifacts/linux-AppImage-${{ steps.artifact-metadata.outputs.arch }}.AppImage - ls ./ci-artifacts/ - - - name: Upload Assets and Potential Publish Release - if: github.repository == 'PCSX2/pcsx2' && startsWith(github.ref, 'refs/tags/') && matrix.appimage == true - env: - GITHUB_TOKEN: ${{ secrets.BOT_PAT }} - ASSET_DIR: ${{ github.WORKSPACE }}/ci-artifacts - ASSET_EXTENSION: AppImage - TAG_TO_SEARCH_FOR: ${{ github.REF }} - run: | - cd ./.github/workflows/scripts/releases/upload-release-artifacts - npm ci - node index.js diff --git a/.github/workflows/linux_build.yml b/.github/workflows/linux_build.yml new file mode 100644 index 0000000000..55f8382257 --- /dev/null +++ b/.github/workflows/linux_build.yml @@ -0,0 +1,146 @@ +name: Linux Build Steps + +on: + workflow_call: + inputs: + jobName: + required: true + type: string + os: + required: false + type: string + default: ubuntu-18.04 + platform: + required: false + type: string + default: x64 + compiler: + required: true + type: string + cmakeflags: + required: true + type: string + buildAppImage: + required: false + type: boolean + default: false + detail: + required: false + type: string + default: "" + +jobs: + build_linux: + name: ${{ inputs.jobName }} + runs-on: ${{ inputs.os }} + # Set some sort of timeout in the event of run-away builds. We are limited on concurrent jobs so, get rid of them. + timeout-minutes: 60 + env: + CCACHE_BASEDIR: ${{ github.workspace }} + CCACHE_DIR: ${{ github.workspace }}/.ccache + CCACHE_COMPRESS: true + CCACHE_COMPRESSLEVEL: 9 + CCACHE_MAXSIZE: 100M + SDL: SDL2-2.0.22 + PATCHELF_VERSION: 0.12 + + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Prepare Artifact Metadata + id: artifact-metadata + shell: bash + env: + OS: linux + GUI_FRAMEWORK: wxWidgets + ARCH: ${{ inputs.platform }} + SIMD: '' + EVENT_NAME: ${{ github.event_name }} + PR_TITLE: ${{ github.event.pull_request.title }} + PR_NUM: ${{ github.event.pull_request.number }} + PR_SHA: ${{ github.event.pull_request.head.sha }} + run: ./.github/workflows/scripts/common/name-artifacts.sh + + # -- SETUP CCACHE - https://cristianadam.eu/20200113/speeding-up-c-plus-plus-github-actions-using-ccache/ + - name: Prepare ccache timestamp + id: ccache_cache_timestamp + shell: cmake -P {0} + run: | + string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC) + message("::set-output name=timestamp::${current_date}") + + - name: ccache cache files + uses: actions/cache@v3 + with: + path: .ccache + key: ${{ inputs.os }} ${{ inputs.platform }} ${{ inputs.compiler }}${{ inputs.detail }} ccache ${{ steps.ccache_cache_timestamp.outputs.timestamp }} + restore-keys: ${{ inputs.os }} ${{ inputs.platform }} ${{ inputs.compiler }}${{ inputs.detail }} ccache + + - name: Install Packages + env: + PLATFORM: ${{ inputs.platform }} + COMPILER: ${{ inputs.compiler }} + run: .github/workflows/scripts/linux/install-packages.sh + + - name: Cache Dependencies + uses: actions/cache@v3 + with: + path: | + 3rdparty/${{ env.SDL }} + 3rdparty/patchelf-${{ env.PATCHELF_VERSION }} + key: ${{ inputs.os }} ${{ inputs.platform }} ${{ env.SDL }} patchelf-${{ env.PATCHELF_VERSION }} + + - name: Build Dependencies + run: | + if [[ ! -e 3rdparty/patchelf-${{ env.PATCHELF_VERSION }} ]]; then + curl -sSfL https://github.com/NixOS/patchelf/releases/download/${{ env.PATCHELF_VERSION }}/patchelf-${{ env.PATCHELF_VERSION }}.tar.bz2 | tar -xjC 3rdparty + mv 3rdparty/patchelf-${{ env.PATCHELF_VERSION }}* 3rdparty/patchelf-${{ env.PATCHELF_VERSION }} + cd 3rdparty/patchelf-${{ env.PATCHELF_VERSION }} + ./configure + make && cd ../../ + fi + sudo make -C 3rdparty/patchelf-${{ env.PATCHELF_VERSION }} install + if [[ ! -e 3rdparty/${{ env.SDL }} ]]; then + curl -sL https://libsdl.org/release/${{ env.SDL }}.tar.gz | tar -xzC 3rdparty + cd 3rdparty/${{ env.SDL }} + if [ "${{ inputs.platform }}" == "x86" ]; then + ./configure --build=i686-pc-linux-gnu CC=${{ inputs.compiler }} CFLAGS=-m32 CXXFLAGS=-m32 LDFLAGS=-m32 --prefix=/usr --libdir=/usr/lib/i386-linux-gnu + else + ./configure --prefix=/usr --libdir=/usr/lib/x86_64-linux-gnu + fi + make -j $(getconf _NPROCESSORS_ONLN) && cd ../../ + fi + sudo make -C 3rdparty/${{ env.SDL }} install + + - name: Generate CMake + env: + PLATFORM: ${{ inputs.platform }} + COMPILER: ${{ inputs.compiler }} + ADDITIONAL_CMAKE_ARGS: ${{ inputs.cmakeflags }} + run: .github/workflows/scripts/linux/generate-cmake.sh + + - name: Build PCSX2 + working-directory: build + run: ../.github/workflows/scripts/linux/compile.sh + + - name: Run Tests + working-directory: ./build + run: ninja unittests + + - name: Package AppImage + if: inputs.buildAppImage == true + env: + PLATFORM: ${{ inputs.platform }} + COMPILER: ${{ inputs.compiler }} + name: ${{ steps.artifact-metadata.outputs.artifact-name }} + run: .github/workflows/scripts/linux/appimage.sh + + - name: Upload artifact + if: inputs.buildAppImage == true + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.artifact-metadata.outputs.artifact-name }} + path: ci-artifacts diff --git a/.github/workflows/linux_build_matrix.yml b/.github/workflows/linux_build_matrix.yml new file mode 100644 index 0000000000..1bd0e4ce3e --- /dev/null +++ b/.github/workflows/linux_build_matrix.yml @@ -0,0 +1,51 @@ +name: 🐧 Linux Builds + +on: + push: + branches: + - '*' + pull_request: + branches: + - master + +jobs: + build_gcc_lto: + name: "GCC" + uses: ./.github/workflows/linux_build.yml + with: + jobName: "with LTO" + compiler: gcc + cmakeflags: "-DLTO_PCSX2_CORE=ON" + buildAppImage: true + secrets: inherit + + # (PCH conflicts with ccache, fixed by https://gitlab.kitware.com/cmake/cmake/-/merge_requests/4400) + build_gcc_nopch: + name: "GCC" + uses: ./.github/workflows/linux_build.yml + with: + jobName: "No PCH" + compiler: gcc + cmakeflags: "-DCMAKE_DISABLE_PRECOMPILE_HEADERS=ON" + detail: " nopch" + secrets: inherit + + build_gcc_nopch_avx2: + name: "GCC" + uses: ./.github/workflows/linux_build.yml + with: + jobName: "AVX2 and No PCH" + compiler: gcc + cmakeflags: "-DCMAKE_DISABLE_PRECOMPILE_HEADERS=ON -DARCH_FLAG=-march=haswell" + detail: " avx2 nopch" + secrets: inherit + + build_clang_nopch: + name: "Clang" + uses: ./.github/workflows/linux_build.yml + with: + jobName: "No PCH" + compiler: clang + cmakeflags: "" + detail: " nopch" + secrets: inherit diff --git a/.github/workflows/macos-workflow.yml b/.github/workflows/macos-workflow.yml deleted file mode 100644 index f2c8ab49b0..0000000000 --- a/.github/workflows/macos-workflow.yml +++ /dev/null @@ -1,152 +0,0 @@ -name: 🍎 MacOS Builds - -# Controls when the action will run. Triggers the workflow on push or pull request -# events but only for the master branch -on: - push: - branches: - - master - tags: - - v* - pull_request: - branches: - - master - paths-ignore: - - '**/*.md' - - '**/*.bat' - - '**/*.cmd' - - '**/*.props' - - '**/*.sln' - - '.clang-format' - - '.codacy.yaml' - - '.github/*' - - '.github/workflows/lint-gamedb.yml' - - '.github/workflows/linux-workflow.yml' - - '.github/workflows/pr-triage.yml' - - '.github/workflows/scripts/windows/**' - - '.github/workflows/scripts/validation/**' - - '.github/workflows/windows-workflow.yml' - - '.gitignore' - - 'bin/PCSX2_keys.ini.default' - - 'build.sh' - - 'buildbot.xml' - - 'pcsx2/CDVD/Windows/**' - - 'pcsx2/DEV9/Win32/**' - - 'pcsx2/PAD/Windows/**' - - 'pcsx2/SPU2/Windows/**' - - 'pcsx2/USB/Win32/**' - - 'pcsx2/windows/**' - -jobs: - build: - strategy: - # Prevent one build from failing everything (although maybe those should be included as experimental builds instead) - fail-fast: false - matrix: - os: [macos-11.0] - platform: [x64] - experimental: [false] - - runs-on: ${{ matrix.os }} - continue-on-error: ${{ matrix.experimental }} - # Set some sort of timeout in the event of run-away builds. We are limited on concurrent jobs so, get rid of them. - timeout-minutes: 60 - - steps: - # NOTE - useful for debugging - # - name: Dump GitHub context - # env: - # GITHUB_CONTEXT: ${{ toJson(github) }} - # run: | - # echo "$GITHUB_CONTEXT" - # echo ${{ github.event.pull_request.title }} - - - name: Checkout Repository - uses: actions/checkout@v3 - - - name: Checkout Submodules - run: git submodule update --init --recursive -j $(getconf _NPROCESSORS_ONLN) - - - name: Install Packages - env: - PLATFORM: ${{ matrix.platform }} - HOMEBREW_NO_INSTALL_CLEANUP: 1 - HOMEBREW_NO_ANALYTICS: 1 - run: | - brew unlink libjpeg libpng # Conflicts with our self-built dependencies - # Unlike other packages, brew's MoltenVK build uses MoltenVK's minimum macOS version of 10.13 so we can use it - if ! brew install molten-vk; then - brew update - brew install molten-vk - fi - - - name: Cache Dependencies - id: cache-deps - uses: actions/cache@v3 - with: - path: ~/deps - key: ${{ matrix.os }} ${{ matrix.platform }} deps ${{ hashFiles('.github/workflows/scripts/macos/build-dependencies.sh') }} - - - name: Build Dependencies - if: steps.cache-deps.outputs.cache-hit != 'true' - run: .github/workflows/scripts/macos/build-dependencies.sh - - - name: Generate CMake Files - run: cmake -DCMAKE_PREFIX_PATH="$HOME/deps" -DCMAKE_BUILD_TYPE=Release -DUSE_OPENGL=OFF -DDISABLE_ADVANCE_SIMD=ON -DLTO_PCSX2_CORE=ON -DUSE_SYSTEM_LIBS=OFF -DUSE_SYSTEM_SDL2=ON -B build . - - - name: Build PCSX2 - working-directory: build - run: make -j$(getconf _NPROCESSORS_ONLN) # macOS doesn't use make install - - - name: Run Tests - working-directory: build - run: make -j$(getconf _NPROCESSORS_ONLN) unittests - - - name: Prepare Artifact Metadata - id: artifact-metadata - run: | - if [ ${{ github.event_name }} == "pull_request" ]; then - PR_SHA=$(git rev-parse --short "${{ github.event.pull_request.head.sha }}") - ARTIFACT_NAME="PCSX2" - if [ -n "${{ github.event.pull_request.number }}" ]; then - PR_NUM=${{ github.event.pull_request.number }} - ARTIFACT_NAME="${ARTIFACT_NAME}-pr[${PR_NUM}]" - fi - ARTIFACT_NAME="${ARTIFACT_NAME}-sha[${PR_SHA}]" - if [ -n "${{ github.event.pull_request.title }}" ]; then - PR_TITLE=$(echo "${{ github.event.pull_request.title }}" | tr -cd '[a-zA-Z0-9[:space:]]_-') - ARTIFACT_NAME="${ARTIFACT_NAME}-title[${PR_TITLE}" - fi - else - SHA=$(git rev-parse --short "$GITHUB_SHA") - ARTIFACT_NAME="PCSX2-sha[${SHA}" - fi - TRIMMED_ARTIFACT_NAME=$(printf "%.199s]" "$ARTIFACT_NAME") - echo "name=$TRIMMED_ARTIFACT_NAME" - echo "##[set-output name=name;]${TRIMMED_ARTIFACT_NAME}" - - - name: Prepare Build Artifacts - run: | - cp /usr/local/lib/libMoltenVK.dylib build/pcsx2/PCSX2.app/Contents/Frameworks/ - mv build/pcsx2/PCSX2.app "${{ steps.artifact-metadata.outputs.name }}.app" - tar cvzf "${{ steps.artifact-metadata.outputs.name }}.tar.gz" "${{ steps.artifact-metadata.outputs.name }}.app" - mkdir ci-artifacts - cp "${{ steps.artifact-metadata.outputs.name }}.tar.gz" ci-artifacts/macOS.tar.gz - - - name: Upload Artifact - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.artifact-metadata.outputs.name }} - path: "*.tar.gz" - - - name: Upload Assets and Potential Publish Release - if: github.repository == 'PCSX2/pcsx2' && startsWith(github.ref, 'refs/tags/') - env: - GITHUB_TOKEN: ${{ secrets.BOT_PAT }} - ASSET_DIR: ${{ github.WORKSPACE }}/ci-artifacts - ASSET_EXTENSION: tar.gz - TAG_TO_SEARCH_FOR: ${{ github.REF }} - run: | - cd .github/workflows/scripts/releases/upload-release-artifacts - npm ci - node index.js diff --git a/.github/workflows/macos_build.yml b/.github/workflows/macos_build.yml new file mode 100644 index 0000000000..3cec5bcdac --- /dev/null +++ b/.github/workflows/macos_build.yml @@ -0,0 +1,99 @@ +name: MacOS Build Steps + +on: + workflow_call: + inputs: + jobName: + required: true + type: string + os: + required: false + type: string + default: macos-11.0 + platform: + required: false + type: string + default: x64 + +jobs: + build_macos: + name: ${{ inputs.jobName }} + runs-on: ${{ inputs.os }} + # Set some sort of timeout in the event of run-away builds. We are limited on concurrent jobs so, get rid of them. + timeout-minutes: 60 + env: + POWERSHELL_TELEMETRY_OPTOUT: 1 + BUILDCACHE_COMPRESS_FORMAT: ZSTD + BUILDCACHE_COMPRESS_LEVEL: 9 + BUILDCACHE_MAX_CACHE_SIZE: 536870912 # 512MB + BUILDCACHE_DIRECT_MODE: true + BUILDCACHE_LOG_FILE: ${{ github.workspace }}\buildcache.log + + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Prepare Artifact Metadata + id: artifact-metadata + shell: bash + env: + OS: macos + GUI_FRAMEWORK: wxWidgets + ARCH: ${{ inputs.platform }} + SIMD: '' + EVENT_NAME: ${{ github.event_name }} + PR_TITLE: ${{ github.event.pull_request.title }} + PR_NUM: ${{ github.event.pull_request.number }} + PR_SHA: ${{ github.event.pull_request.head.sha }} + run: ./.github/workflows/scripts/common/name-artifacts.sh + + - name: Install Packages + env: + PLATFORM: ${{ inputs.platform }} + HOMEBREW_NO_INSTALL_CLEANUP: 1 + HOMEBREW_NO_ANALYTICS: 1 + run: | + brew unlink libjpeg libpng # Conflicts with our self-built dependencies + # Unlike other packages, brew's MoltenVK build uses MoltenVK's minimum macOS version of 10.13 so we can use it + if ! brew install molten-vk; then + brew update + brew install molten-vk + fi + + - name: Cache Dependencies + id: cache-deps + uses: actions/cache@v3 + with: + path: ~/deps + key: ${{ inputs.os }} ${{ inputs.platform }} deps ${{ hashFiles('.github/workflows/scripts/macos/build-dependencies.sh') }} + + - name: Build Dependencies + if: steps.cache-deps.outputs.cache-hit != 'true' + run: .github/workflows/scripts/macos/build-dependencies.sh + + - name: Generate CMake Files + run: cmake -DCMAKE_PREFIX_PATH="$HOME/deps" -DCMAKE_BUILD_TYPE=Release -DUSE_OPENGL=OFF -DDISABLE_ADVANCE_SIMD=ON -DLTO_PCSX2_CORE=ON -DUSE_SYSTEM_LIBS=OFF -DUSE_SYSTEM_SDL2=ON -B build . + + - name: Build PCSX2 + working-directory: build + run: make -j$(getconf _NPROCESSORS_ONLN) # macOS doesn't use make install + + - name: Run Tests + working-directory: build + run: make -j$(getconf _NPROCESSORS_ONLN) unittests + + - name: Prepare Build Artifacts + run: | + cp /usr/local/lib/libMoltenVK.dylib build/pcsx2/PCSX2.app/Contents/Frameworks/ + mv build/pcsx2/PCSX2.app "${{ steps.artifact-metadata.outputs.artifact-name }}.app" + tar cvzf "${{ steps.artifact-metadata.outputs.artifact-name }}.tar.gz" "${{ steps.artifact-metadata.outputs.artifact-name }}.app" + mkdir ci-artifacts + cp "${{ steps.artifact-metadata.outputs.artifact-name }}.tar.gz" ci-artifacts/macOS.tar.gz + + - name: Upload Artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.artifact-metadata.outputs.artifact-name }} + path: "*.tar.gz" diff --git a/.github/workflows/macos_build_matrix.yml b/.github/workflows/macos_build_matrix.yml new file mode 100644 index 0000000000..95ef7ef969 --- /dev/null +++ b/.github/workflows/macos_build_matrix.yml @@ -0,0 +1,17 @@ +name: 🍎 MacOS Builds + +on: + push: + branches: + - '*' + pull_request: + branches: + - master + +jobs: + build_macos_default: + name: "Defaults" + uses: ./.github/workflows/macos_build.yml + with: + jobName: "wxWidgets" + secrets: inherit diff --git a/.github/workflows/scripts/common/name-artifacts.sh b/.github/workflows/scripts/common/name-artifacts.sh new file mode 100644 index 0000000000..6a78b88791 --- /dev/null +++ b/.github/workflows/scripts/common/name-artifacts.sh @@ -0,0 +1,46 @@ +# Artifact Naming Scheme: +# PCSX2---[ARCH]-[SIMD]-[pr\[PR_NUM\]]-[title|sha\[SHA|PR_TITLE\] +# -- limited to 150 chars +# Outputs: +# - artifact-name + +# Inputs as env-vars +# OS +# GUI_FRAMEWORK +# ARCH +# SIMD +# EVENT_NAME +# PR_TITLE +# PR_NUM +# PR_SHA + +NAME="" + +if [ "${OS}" == "macos" ]; then + NAME="PCSX2-${OS}-${GUI_FRAMEWORK}" +elif [ "${OS}" == "windows" ]; then + NAME="PCSX2-${OS}-${GUI_FRAMEWORK}-${ARCH}-${SIMD}" +else + NAME="PCSX2-${OS}-${GUI_FRAMEWORK}-${ARCH}" +fi + +# Add PR / Commit Metadata +if [ $EVENT_NAME == "pull_request" ]; then + PR_SHA=$(git rev-parse --short "${PR_SHA}") + if [ ! -z "${PR_NUM}" ]; then + NAME="${NAME}-pr[${PR_NUM}]" + fi + NAME="${NAME}-sha[${PR_SHA}]" + if [ ! -z "${PR_TITLE}" ]; then + PR_TITLE=$(echo "${PR_TITLE}" | tr -cd '[a-zA-Z0-9[:space:]]_-') + NAME="${NAME}-title["${PR_TITLE}"" + fi +else + SHA=$(git rev-parse --short "$GITHUB_SHA") + NAME="${NAME}-sha[${SHA}" +fi + +# Trim the Name +NAME=$(printf "%.199s]" "$NAME") +echo ${NAME} +echo "##[set-output name=artifact-name;]${NAME}" diff --git a/.github/workflows/scripts/releases/generate-release-notes/index.js b/.github/workflows/scripts/releases/generate-release-notes/index.js index 86d6365028..4cbacfa47b 100644 --- a/.github/workflows/scripts/releases/generate-release-notes/index.js +++ b/.github/workflows/scripts/releases/generate-release-notes/index.js @@ -1,11 +1,15 @@ import { Octokit } from "@octokit/rest"; import { throttling } from "@octokit/plugin-throttling"; import { retry } from "@octokit/plugin-retry"; + +let owner = process.env.OWNER; +let repo = process.env.REPO; + Octokit.plugin(throttling); Octokit.plugin(retry); const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN, - userAgent: 'PCSX2/pcsx2', + userAgent: `${owner}/${repo}`, log: { debug: () => { }, info: () => { }, @@ -39,14 +43,14 @@ let commitSha = process.env.COMMIT_SHA; console.log(`Searching for Commit - ${commitSha}`); const { data: commit } = await octokit.rest.repos.getCommit({ - owner: "PCSX2", - repo: "pcsx2", + owner: owner, + repo: repo, ref: commitSha, }); const { data: associatedPulls } = await octokit.rest.repos.listPullRequestsAssociatedWithCommit({ - owner: "PCSX2", - repo: "pcsx2", + owner: owner, + repo: repo, commit_sha: commit.sha, }); diff --git a/.github/workflows/windows_build_matrix.yml b/.github/workflows/windows_build_matrix.yml new file mode 100644 index 0000000000..6c72805de7 --- /dev/null +++ b/.github/workflows/windows_build_matrix.yml @@ -0,0 +1,57 @@ +name: 🖥️ Windows Builds + +on: + push: + branches: + - '*' + pull_request: + branches: + - master + +jobs: + # MSBUILD + lint_vs_proj_files: + name: Lint VS Project Files + runs-on: windows-2019 + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + - name: Verify VS Project Files + run: .github\workflows\scripts\windows\validate-vs-filters.ps1 + + build_wx: + needs: lint_vs_proj_files + name: "SSE4" + uses: ./.github/workflows/windows_build_wx.yml + with: + jobName: wxWidgets + configuration: Release + simd: "SSE4" + secrets: inherit + + build_wx_avx2: + needs: lint_vs_proj_files + name: "AVX2" + uses: ./.github/workflows/windows_build_wx.yml + with: + jobName: wxWidgets + configuration: Release AVX2 + secrets: inherit + + build_qt_avx2: + needs: lint_vs_proj_files + name: "AVX2" + uses: ./.github/workflows/windows_build_qt.yml + with: + jobName: Qt + configuration: Release AVX2 + secrets: inherit + + # CMAKE + build_wx_avx2_cmake: + name: "CMake AVX2" + uses: ./.github/workflows/windows_build_wx.yml + with: + jobName: wxWidgets + configuration: CMake + secrets: inherit diff --git a/.github/workflows/windows_build_qt.yml b/.github/workflows/windows_build_qt.yml new file mode 100644 index 0000000000..1e3fa4e5b2 --- /dev/null +++ b/.github/workflows/windows_build_qt.yml @@ -0,0 +1,104 @@ +name: Windows Build Steps - Qt + +on: + workflow_call: + inputs: + jobName: + required: true + type: string + os: + required: false + type: string + default: windows-2019 + platform: + required: false + type: string + default: x64 + configuration: + required: true + type: string + simd: + required: false + type: string + default: AVX2 + qt_binary_url: + required: false + type: string + default: https://github.com/PCSX2/pcsx2-windows-dependencies/releases/download/2022-04-13/qt-6.3.0-x64.7z + +jobs: + build_windows_qt: + name: ${{ inputs.jobName }} + runs-on: ${{ inputs.os }} + # Set some sort of timeout in the event of run-away builds. We are limited on concurrent jobs so, get rid of them. + timeout-minutes: 60 + env: + POWERSHELL_TELEMETRY_OPTOUT: 1 + BUILDCACHE_COMPRESS_FORMAT: ZSTD + BUILDCACHE_COMPRESS_LEVEL: 9 + BUILDCACHE_MAX_CACHE_SIZE: 536870912 # 512MB + BUILDCACHE_DIRECT_MODE: true + BUILDCACHE_LOG_FILE: ${{ github.workspace }}\buildcache.log + + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Prepare Artifact Metadata + id: artifact-metadata + shell: bash + env: + OS: windows + GUI_FRAMEWORK: Qt + ARCH: ${{ inputs.platform }} + SIMD: ${{ inputs.simd }} + EVENT_NAME: ${{ github.event_name }} + PR_TITLE: ${{ github.event.pull_request.title }} + PR_NUM: ${{ github.event.pull_request.number }} + PR_SHA: ${{ github.event.pull_request.head.sha }} + run: ./.github/workflows/scripts/common/name-artifacts.sh + + - name: Setup msbuild + if: inputs.configuration != 'CMake' + uses: microsoft/setup-msbuild@v1 + + - name: Download Qt build files + shell: cmd + run: | + cd 3rdparty\qt + aria2c ${{ inputs.qt_binary_url }} + 7z x qt-*-x64.7z + del qt-*-x64.7z + + - name: Build PCSX2 + shell: cmd + run: | + if "${{ inputs.configuration }}"=="CMake" ( + call "%ProgramFiles(x86)%\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\${{ steps.cmake.outputs.vcvars }}" + cmake --build build --config ${{ steps.cmake.outputs.buildtype }} + ) else ( + msbuild "PCSX2_suite.sln" /m /v:m /p:Configuration="${{ inputs.configuration }}" /p:Platform="${{ inputs.platform }}" + ) + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.artifact-metadata.outputs.artifact-name }} + path: | + ./bin + !./bin/**/*.bsc + !./bin/**/*.exp + !./bin/**/*.ilk + !./bin/**/*.iobj + !./bin/**/*.ipdb + !./bin/**/*.pdb + !./bin/**/*.lib + + - name: Upload artifact - with symbols + if: inputs.configuration != 'CMake' + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.artifact-metadata.outputs.artifact-name }}-symbols + path: ./bin/**/*.pdb diff --git a/.github/workflows/windows_build_wx.yml b/.github/workflows/windows_build_wx.yml new file mode 100644 index 0000000000..15e342db1f --- /dev/null +++ b/.github/workflows/windows_build_wx.yml @@ -0,0 +1,118 @@ +name: Windows Build Steps - wxWidgets + +on: + workflow_call: + inputs: + jobName: + required: true + type: string + os: + required: false + type: string + default: windows-2019 + platform: + required: false + type: string + default: x64 + simd: + required: false + type: string + default: AVX2 + configuration: + required: true + type: string + +jobs: + build_windows_wx: + name: ${{ inputs.jobName }} + runs-on: ${{ inputs.os }} + # Set some sort of timeout in the event of run-away builds. We are limited on concurrent jobs so, get rid of them. + timeout-minutes: 60 + env: + POWERSHELL_TELEMETRY_OPTOUT: 1 + BUILDCACHE_COMPRESS_FORMAT: ZSTD + BUILDCACHE_COMPRESS_LEVEL: 9 + BUILDCACHE_MAX_CACHE_SIZE: 536870912 # 512MB + BUILDCACHE_DIRECT_MODE: true + BUILDCACHE_LOG_FILE: ${{ github.workspace }}\buildcache.log + + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Prepare Artifact Metadata + id: artifact-metadata + shell: bash + env: + OS: windows + GUI_FRAMEWORK: wxWidgets + ARCH: ${{ inputs.platform }} + SIMD: ${{ inputs.simd }} + EVENT_NAME: ${{ github.event_name }} + PR_TITLE: ${{ github.event.pull_request.title }} + PR_NUM: ${{ github.event.pull_request.number }} + PR_SHA: ${{ github.event.pull_request.head.sha }} + run: ./.github/workflows/scripts/common/name-artifacts.sh + + - name: Setup Buildcache + if: inputs.configuration == 'CMake' # TODO: buildcache on VS + uses: mikehardy/buildcache-action@v1.2.2 + with: + cache_key: ${{ inputs.os }} ${{ inputs.platform }} ${{ inputs.configuration }} + + - name: Setup msbuild + if: inputs.configuration != 'CMake' + uses: microsoft/setup-msbuild@v1 + + - name: Generate CMake + if: inputs.configuration == 'CMake' + id: cmake + shell: cmd + run: | + if "${{ github.event.inputs.retainDebugArtifacts }}"=="true" (SET type=RelWithDebInfo) else (SET type=Release) + if "${{ inputs.platform }}"=="Win32" (SET vcvars=vcvarsamd64_x86.bat) else (SET vcvars=vcvars64.bat) + call "%ProgramFiles(x86)%\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\%vcvars%" + echo ::set-output name=buildtype::%type% + echo ::set-output name=vcvars::%vcvars% + cmake . -B build -DCMAKE_BUILD_TYPE=%type% -DLTO_PCSX2_CORE=ON -G Ninja -DCMAKE_C_COMPILER_LAUNCHER=..\buildcache\bin\buildcache.exe -DCMAKE_CXX_COMPILER_LAUNCHER=..\buildcache\bin\buildcache.exe -DCMAKE_DISABLE_PRECOMPILE_HEADERS=ON + + - name: Build PCSX2 + shell: cmd + run: | + if "${{ inputs.configuration }}"=="CMake" ( + call "%ProgramFiles(x86)%\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\${{ steps.cmake.outputs.vcvars }}" + cmake --build build --config ${{ steps.cmake.outputs.buildtype }} + cp build/pcsx2/pcsx2* bin/ + ) else ( + msbuild "PCSX2_suite.sln" /m /v:m /p:Configuration="${{ inputs.configuration }}" /p:Platform="${{ inputs.platform }}" + ) + + - name: Run Tests + if: inputs.configuration == 'CMake' + shell: cmd + run: | + call "%ProgramFiles(x86)%\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\${{ steps.cmake.outputs.vcvars }}" + cmake --build build --config ${{ steps.cmake.outputs.buildtype }} --target unittests + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.artifact-metadata.outputs.artifact-name }} + path: | + ./bin + !./bin/**/*.bsc + !./bin/**/*.exp + !./bin/**/*.ilk + !./bin/**/*.iobj + !./bin/**/*.ipdb + !./bin/**/*.pdb + !./bin/**/*.lib + + - name: Upload artifact - with symbols + if: inputs.configuration != 'CMake' + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.artifact-metadata.outputs.artifact-name }}-symbols + path: ./bin/**/*.pdb