From 291ce2cbb0bc796cf180d48dfed752646d4e9070 Mon Sep 17 00:00:00 2001 From: Stenzek Date: Sat, 6 Apr 2024 16:54:17 +1000 Subject: [PATCH] CI: Build shaderc as part of deps --- .../scripts/common/shaderc-install.patch | 146 ++++++++++++++++++ .../scripts/linux/build-dependencies-qt.sh | 35 ++++- .../linux/flatpak/modules/22-shaderc.json | 50 ++++++ .../linux/flatpak/net.pcsx2.PCSX2.json | 1 + .../scripts/macos/build-dependencies.sh | 33 +++- .../scripts/windows/build-dependencies.bat | 28 ++++ 6 files changed, 290 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/scripts/common/shaderc-install.patch create mode 100644 .github/workflows/scripts/linux/flatpak/modules/22-shaderc.json diff --git a/.github/workflows/scripts/common/shaderc-install.patch b/.github/workflows/scripts/common/shaderc-install.patch new file mode 100644 index 0000000000..a56abd0dce --- /dev/null +++ b/.github/workflows/scripts/common/shaderc-install.patch @@ -0,0 +1,146 @@ +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -117,6 +117,9 @@ if(MSVC) + endif() + endif(MSVC) + ++if(NOT WIN32) ++ add_definitions("-fvisibility=hidden") ++endif() + + # Configure subdirectories. + # We depend on these for later projects, so they should come first. +@@ -124,7 +127,6 @@ add_subdirectory(third_party) + + add_subdirectory(libshaderc_util) + add_subdirectory(libshaderc) +-add_subdirectory(glslc) + if(${SHADERC_ENABLE_EXAMPLES}) + add_subdirectory(examples) + endif() +@@ -158,5 +160,3 @@ function(define_pkg_config_file NAME LIBS) + endfunction() + + define_pkg_config_file(shaderc -lshaderc_shared) +-define_pkg_config_file(shaderc_static "-lshaderc ${EXTRA_STATIC_PKGCONFIG_LIBS} -lshaderc_util") +-define_pkg_config_file(shaderc_combined -lshaderc_combined) +--- a/libshaderc/CMakeLists.txt ++++ b/libshaderc/CMakeLists.txt +@@ -24,13 +24,6 @@ set(SHADERC_SOURCES + src/shaderc_private.h + ) + +-add_library(shaderc STATIC ${SHADERC_SOURCES}) +-shaderc_default_compile_options(shaderc) +-target_include_directories(shaderc +- PUBLIC include +- PRIVATE ${glslang_SOURCE_DIR} +- ${SPIRV-Headers_SOURCE_DIR}/include) +- + add_library(shaderc_shared SHARED ${SHADERC_SOURCES}) + shaderc_default_compile_options(shaderc_shared) + target_include_directories(shaderc_shared +@@ -54,7 +47,7 @@ if(SHADERC_ENABLE_INSTALL) + DESTINATION + ${CMAKE_INSTALL_INCLUDEDIR}/shaderc) + +- install(TARGETS shaderc shaderc_shared ++ install(TARGETS shaderc_shared + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + BUNDLE DESTINATION ${CMAKE_INSTALL_BINDIR} +@@ -69,20 +62,8 @@ set(SHADERC_LIBS + SPIRV-Tools + ) + +-target_link_libraries(shaderc PRIVATE ${SHADERC_LIBS}) + target_link_libraries(shaderc_shared PRIVATE ${SHADERC_LIBS}) + +-shaderc_add_tests( +- TEST_PREFIX shaderc +- LINK_LIBS shaderc +- INCLUDE_DIRS include ${shaderc_SOURCE_DIR}/libshaderc_util/include ${glslang_SOURCE_DIR} +- ${spirv-tools_SOURCE_DIR}/include +- ${SPIRV-Headers_SOURCE_DIR}/include +- TEST_NAMES +- shaderc +- shaderc_cpp +- shaderc_private) +- + shaderc_add_tests( + TEST_PREFIX shaderc_shared + LINK_LIBS shaderc_shared SPIRV-Tools +@@ -94,22 +75,6 @@ shaderc_add_tests( + shaderc_cpp + shaderc_private) + +-shaderc_combine_static_lib(shaderc_combined shaderc) +- +-if(SHADERC_ENABLE_INSTALL) +- install(TARGETS shaderc_combined DESTINATION ${CMAKE_INSTALL_LIBDIR}) +-endif(SHADERC_ENABLE_INSTALL) +- +-shaderc_add_tests( +- TEST_PREFIX shaderc_combined +- LINK_LIBS shaderc_combined ${CMAKE_THREAD_LIBS_INIT} +- INCLUDE_DIRS include ${shaderc_SOURCE_DIR}/libshaderc_util/include ${glslang_SOURCE_DIR} +- ${spirv-tools_SOURCE_DIR}/include +- ${SPIRV-Headers_SOURCE_DIR}/include +- TEST_NAMES +- shaderc +- shaderc_cpp) +- + if(${SHADERC_ENABLE_TESTS}) + add_executable(shaderc_c_smoke_test ./src/shaderc_c_smoke_test.c) + shaderc_default_c_compile_options(shaderc_c_smoke_test) +diff --git a/libshaderc_util/src/compiler.cc b/libshaderc_util/src/compiler.cc +index e5f5d10..f4f0fef 100644 +--- a/libshaderc_util/src/compiler.cc ++++ b/libshaderc_util/src/compiler.cc +@@ -341,6 +341,10 @@ std::tuple, size_t> Compiler::Compile( + options.generateDebugInfo = generate_debug_info_; + options.disableOptimizer = true; + options.optimizeSize = false; ++ if (generate_debug_info_) { ++ options.emitNonSemanticShaderDebugInfo = true; ++ options.emitNonSemanticShaderDebugSource = true; ++ } + // Note the call to GlslangToSpv also populates compilation_output_data. + glslang::GlslangToSpv(*program.getIntermediate(used_shader_stage), spirv, + &options); +--- a/third_party/CMakeLists.txt ++++ b/third_party/CMakeLists.txt +@@ -20,9 +20,9 @@ set(SHADERC_TINT_DIR "${SHADERC_THIRD_PARTY_ROOT_DIR}/tint" CACHE STRING + set(SHADERC_ABSL_DIR "${SHADERC_THIRD_PARTY_ROOT_DIR}/abseil_cpp" CACHE STRING + "Location of re2 source") + +-set( SKIP_GLSLANG_INSTALL ${SHADERC_SKIP_INSTALL} ) +-set( SKIP_SPIRV_TOOLS_INSTALL ${SHADERC_SKIP_INSTALL} ) +-set( SKIP_GOOGLETEST_INSTALL ${SHADERC_SKIP_INSTALL} ) ++set( SKIP_GLSLANG_INSTALL ON ) ++set( SKIP_SPIRV_TOOLS_INSTALL ON ) ++set( SKIP_GOOGLETEST_INSTALL ON ) + + # Configure third party projects. + if(${SHADERC_ENABLE_TESTS}) +@@ -64,7 +64,10 @@ if (NOT TARGET SPIRV-Tools) + add_subdirectory(${SHADERC_RE2_DIR} re2) + add_subdirectory(${SHADERC_EFFCEE_DIR} effcee) + endif() +- add_subdirectory(${SHADERC_SPIRV_TOOLS_DIR} spirv-tools) ++ set(SPIRV_SKIP_EXECUTABLES ON CACHE BOOL "Skip building SPIRV-Tools executables") ++ set(SPIRV_TOOLS_BUILD_STATIC OFF CACHE BOOL "Skip building two SPIRV-Tools libs") ++ set(SPIRV_TOOLS_LIBRARY_TYPE STATIC CACHE STRING "Build static SPIRV-Tools libs") ++ add_subdirectory(${SHADERC_SPIRV_TOOLS_DIR} spirv-tools EXCLUDE_FROM_ALL) + if (NOT "${SPIRV_SKIP_TESTS}") + if (MSVC) + if (${MSVC_VERSION} LESS 1920) +@@ -83,7 +86,7 @@ endif() + + if (NOT TARGET glslang) + if (IS_DIRECTORY ${SHADERC_GLSLANG_DIR}) +- add_subdirectory(${SHADERC_GLSLANG_DIR} glslang) ++ add_subdirectory(${SHADERC_GLSLANG_DIR} glslang EXCLUDE_FROM_ALL) + endif() + if (NOT TARGET glslang) + message(FATAL_ERROR "glslang was not found - required for compilation") diff --git a/.github/workflows/scripts/linux/build-dependencies-qt.sh b/.github/workflows/scripts/linux/build-dependencies-qt.sh index 8544e523a4..a9379e560b 100755 --- a/.github/workflows/scripts/linux/build-dependencies-qt.sh +++ b/.github/workflows/scripts/linux/build-dependencies-qt.sh @@ -7,7 +7,7 @@ if [ "$#" -ne 1 ]; then exit 1 fi -#SCRIPTDIR=$(realpath $(dirname "${BASH_SOURCE[0]}")) +SCRIPTDIR=$(realpath $(dirname "${BASH_SOURCE[0]}")) NPROCS="$(getconf _NPROCESSORS_ONLN)" INSTALLDIR="$1" if [ "${INSTALLDIR:0:1}" != "/" ]; then @@ -26,6 +26,11 @@ QT=6.7.0 ZLIB=1.3.1 ZSTD=1.5.5 +SHADERC=2024.0 +SHADERC_GLSLANG=d73712b8f6c9047b09e99614e20d456d5ada2390 +SHADERC_SPIRVHEADERS=8b246ff75c6615ba4532fe4fde20f1be090c3764 +SHADERC_SPIRVTOOLS=04896c462d9f3f504c99a4698605b6524af813c1 + mkdir -p deps-build cd deps-build @@ -46,6 +51,10 @@ fd6f417fe9e3a071cf1424a5152d926a34c4a3c5070745470be6cf12a404ed79 $LIBBACKTRACE. c8da6b239e82fe1e23465cbf0936c0da5a334438d3fb433e19c503cbb1abee7b qttools-everywhere-src-$QT.tar.xz 26fc8047062ca4bacd1bd953be86fd39c6e0a5f5e9920c72ba9d40876cea4b56 qttranslations-everywhere-src-$QT.tar.xz d73470e4217da388d8cd2a517ee8bb373853f33c569306e80f04397845157aea qtwayland-everywhere-src-$QT.tar.xz +c761044e4e204be8e0b9a2d7494f08671ca35b92c4c791c7049594ca7514197f shaderc-$SHADERC.tar.gz +d27f7359156a92749f8fd4681d1d518c736864213c431cf8144ecc2fb6689a2d shaderc-glslang-$SHADERC_GLSLANG.tar.gz +cfeed5f9a97d12a9761a26e7f5bd10fedb1a8ce92033075151ae3bc7206fc229 shaderc-spirv-headers-$SHADERC_SPIRVHEADERS.tar.gz +c0d01e758a543b3a358cb97af02c6817ebd3f5ff13a2edf9fb220646a3d67999 shaderc-spirv-tools-$SHADERC_SPIRVTOOLS.tar.gz EOF curl -C - -L \ @@ -64,7 +73,11 @@ curl -C - -L \ -O "https://download.qt.io/official_releases/qt/${QT%.*}/$QT/submodules/qtsvg-everywhere-src-$QT.tar.xz" \ -O "https://download.qt.io/official_releases/qt/${QT%.*}/$QT/submodules/qttools-everywhere-src-$QT.tar.xz" \ -O "https://download.qt.io/official_releases/qt/${QT%.*}/$QT/submodules/qttranslations-everywhere-src-$QT.tar.xz" \ - -O "https://download.qt.io/official_releases/qt/${QT%.*}/$QT/submodules/qtwayland-everywhere-src-$QT.tar.xz" + -O "https://download.qt.io/official_releases/qt/${QT%.*}/$QT/submodules/qtwayland-everywhere-src-$QT.tar.xz" \ + -o "shaderc-$SHADERC.tar.gz" "https://github.com/google/shaderc/archive/refs/tags/v$SHADERC.tar.gz" \ + -o "shaderc-glslang-$SHADERC_GLSLANG.tar.gz" "https://github.com/KhronosGroup/glslang/archive/$SHADERC_GLSLANG.tar.gz" \ + -o "shaderc-spirv-headers-$SHADERC_SPIRVHEADERS.tar.gz" "https://github.com/KhronosGroup/SPIRV-Headers/archive/$SHADERC_SPIRVHEADERS.tar.gz" \ + -o "shaderc-spirv-tools-$SHADERC_SPIRVTOOLS.tar.gz" "https://github.com/KhronosGroup/SPIRV-Tools/archive/$SHADERC_SPIRVTOOLS.tar.gz" shasum -a 256 --check SHASUMS @@ -281,6 +294,24 @@ cmake --build . --parallel ninja install cd ../../ +echo "Building shaderc..." +rm -fr "shaderc-$SHADERC" +tar xf "shaderc-$SHADERC.tar.gz" +cd "shaderc-$SHADERC" +cd third_party +tar xf "../../shaderc-glslang-$SHADERC_GLSLANG.tar.gz" +mv "glslang-$SHADERC_GLSLANG" "glslang" +tar xf "../../shaderc-spirv-headers-$SHADERC_SPIRVHEADERS.tar.gz" +mv "SPIRV-Headers-$SHADERC_SPIRVHEADERS" "spirv-headers" +tar xf "../../shaderc-spirv-tools-$SHADERC_SPIRVTOOLS.tar.gz" +mv "SPIRV-Tools-$SHADERC_SPIRVTOOLS" "spirv-tools" +cd .. +patch -p1 < "$SCRIPTDIR/../common/shaderc-install.patch" +cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$INSTALLDIR" -DCMAKE_INSTALL_PREFIX="$INSTALLDIR" -DSHADERC_SKIP_TESTS=ON -DSHADERC_SKIP_EXAMPLES=ON -DSHADERC_SKIP_COPYRIGHT_CHECK=ON -B build -G Ninja +cmake --build build --parallel +ninja -C build install +cd .. + echo "Cleaning up..." cd .. rm -r deps-build diff --git a/.github/workflows/scripts/linux/flatpak/modules/22-shaderc.json b/.github/workflows/scripts/linux/flatpak/modules/22-shaderc.json new file mode 100644 index 0000000000..f3c2c8d819 --- /dev/null +++ b/.github/workflows/scripts/linux/flatpak/modules/22-shaderc.json @@ -0,0 +1,50 @@ +{ + "name": "shaderc", + "buildsystem": "cmake-ninja", + "builddir": true, + "config-opts": [ + "-DSHADERC_SKIP_TESTS=ON", + "-DSHADERC_SKIP_EXAMPLES=ON", + "-DSHADERC_SKIP_COPYRIGHT_CHECK=ON" + ], + "build-options": { + "strip": true + }, + "sources": [ + { + "type": "git", + "url": "https://github.com/google/shaderc.git", + "commit": "d792558a8902cb39b1c237243cc4edab226513a5" + }, + { + "type": "archive", + "url": "https://github.com/KhronosGroup/glslang/archive/d73712b8f6c9047b09e99614e20d456d5ada2390.tar.gz", + "sha256": "d27f7359156a92749f8fd4681d1d518c736864213c431cf8144ecc2fb6689a2d", + "dest": "third_party/glslang" + }, + { + "type": "archive", + "url": "https://github.com/KhronosGroup/SPIRV-Headers/archive/8b246ff75c6615ba4532fe4fde20f1be090c3764.tar.gz", + "sha256": "cfeed5f9a97d12a9761a26e7f5bd10fedb1a8ce92033075151ae3bc7206fc229", + "dest": "third_party/spirv-headers" + }, + { + "type": "archive", + "url": "https://github.com/KhronosGroup/SPIRV-Tools/archive/04896c462d9f3f504c99a4698605b6524af813c1.tar.gz", + "sha256": "c0d01e758a543b3a358cb97af02c6817ebd3f5ff13a2edf9fb220646a3d67999", + "dest": "third_party/spirv-tools" + }, + { + "type": "patch", + "path": "../../../common/shaderc-install.patch" + } + ], + "cleanup": [ + "/bin", + "/include", + "/lib/*.a", + "/lib/*.la", + "/lib/cmake", + "/lib/pkgconfig" + ] +} \ No newline at end of file diff --git a/.github/workflows/scripts/linux/flatpak/net.pcsx2.PCSX2.json b/.github/workflows/scripts/linux/flatpak/net.pcsx2.PCSX2.json index a78299f2d8..bdf46849a2 100644 --- a/.github/workflows/scripts/linux/flatpak/net.pcsx2.PCSX2.json +++ b/.github/workflows/scripts/linux/flatpak/net.pcsx2.PCSX2.json @@ -29,6 +29,7 @@ "modules/11-libaio.json", "modules/20-sdl2.json", "modules/21-libbacktrace.json", + "modules/22-shaderc.json", { "name": "pcsx2", "buildsystem": "simple", diff --git a/.github/workflows/scripts/macos/build-dependencies.sh b/.github/workflows/scripts/macos/build-dependencies.sh index a734431811..32fb304f1d 100755 --- a/.github/workflows/scripts/macos/build-dependencies.sh +++ b/.github/workflows/scripts/macos/build-dependencies.sh @@ -10,7 +10,7 @@ fi export MACOSX_DEPLOYMENT_TARGET=11.0 NPROCS="$(getconf _NPROCESSORS_ONLN)" -#SCRIPTDIR=$(realpath $(dirname "${BASH_SOURCE[0]}")) +SCRIPTDIR=$(realpath $(dirname "${BASH_SOURCE[0]}")) INSTALLDIR="$1" if [ "${INSTALLDIR:0:1}" != "/" ]; then INSTALLDIR="$PWD/$INSTALLDIR" @@ -28,6 +28,11 @@ FFMPEG=6.0 MOLTENVK=1.2.8 QT=6.7.0 +SHADERC=2024.0 +SHADERC_GLSLANG=d73712b8f6c9047b09e99614e20d456d5ada2390 +SHADERC_SPIRVHEADERS=8b246ff75c6615ba4532fe4fde20f1be090c3764 +SHADERC_SPIRVTOOLS=04896c462d9f3f504c99a4698605b6524af813c1 + mkdir -p deps-build cd deps-build @@ -59,6 +64,10 @@ cat > SHASUMS <