From 1e6845c940b25a53613c5580c980ebbd364b5a77 Mon Sep 17 00:00:00 2001 From: PatrickvL Date: Wed, 9 Dec 2020 22:55:47 +0100 Subject: [PATCH] Generalized HLSL shader compilation --- CMakeLists.txt | 2 + src/core/hle/D3D8/Direct3D9/Shader.cpp | 142 ++++++++++++++++ src/core/hle/D3D8/Direct3D9/Shader.h | 12 ++ src/core/hle/D3D8/Direct3D9/VertexShader.cpp | 155 ++---------------- src/core/hle/D3D8/Direct3D9/VertexShader.h | 7 +- .../hle/D3D8/Direct3D9/VertexShaderSource.cpp | 2 +- src/core/hle/D3D8/XbVertexShader.cpp | 3 +- src/core/hle/D3D8/XbVertexShader.h | 5 +- 8 files changed, 183 insertions(+), 145 deletions(-) create mode 100644 src/core/hle/D3D8/Direct3D9/Shader.cpp create mode 100644 src/core/hle/D3D8/Direct3D9/Shader.h diff --git a/CMakeLists.txt b/CMakeLists.txt index d764aa57a..bc97c8157 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -135,6 +135,7 @@ file (GLOB CXBXR_HEADER_EMU "${CXBXR_ROOT_DIR}/src/core/hle/D3D8/Direct3D9/Direct3D9.h" "${CXBXR_ROOT_DIR}/src/core/hle/D3D8/Direct3D9/FixedFunctionVertexShader.hlsl" "${CXBXR_ROOT_DIR}/src/core/hle/D3D8/Direct3D9/FixedFunctionVertexShaderState.hlsli" + "${CXBXR_ROOT_DIR}/src/core/hle/D3D8/Direct3D9/Shader.h" "${CXBXR_ROOT_DIR}/src/core/hle/D3D8/Direct3D9/VertexShader.h" "${CXBXR_ROOT_DIR}/src/core/hle/D3D8/Direct3D9/VertexShaderSource.h" "${CXBXR_ROOT_DIR}/src/core/hle/D3D8/Direct3D9/WalkIndexBuffer.h" @@ -295,6 +296,7 @@ file (GLOB CXBXR_SOURCE_EMU "${CXBXR_ROOT_DIR}/src/core/common/video/RenderBase.cpp" "${CXBXR_ROOT_DIR}/src/core/hle/D3D8/Direct3D9/Direct3D9.cpp" "${CXBXR_ROOT_DIR}/src/core/hle/D3D8/Direct3D9/RenderStates.cpp" + "${CXBXR_ROOT_DIR}/src/core/hle/D3D8/Direct3D9/Shader.cpp" "${CXBXR_ROOT_DIR}/src/core/hle/D3D8/Direct3D9/TextureStates.cpp" "${CXBXR_ROOT_DIR}/src/core/hle/D3D8/Direct3D9/VertexShader.cpp" "${CXBXR_ROOT_DIR}/src/core/hle/D3D8/Direct3D9/VertexShaderSource.cpp" diff --git a/src/core/hle/D3D8/Direct3D9/Shader.cpp b/src/core/hle/D3D8/Direct3D9/Shader.cpp new file mode 100644 index 000000000..382fc0aab --- /dev/null +++ b/src/core/hle/D3D8/Direct3D9/Shader.cpp @@ -0,0 +1,142 @@ +// This is an open source non-commercial project. Dear PVS-Studio, please check it. +// PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com +// ****************************************************************** +// * +// * This file is part of the Cxbx project. +// * +// * Cxbx and Cxbe are free software; you can redistribute them +// * and/or modify them under the terms of the GNU General Public +// * License as published by the Free Software Foundation; either +// * version 2 of the license, or (at your option) any later version. +// * +// * This program is distributed in the hope that it will be useful, +// * but WITHOUT ANY WARRANTY; without even the implied warranty of +// * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// * GNU General Public License for more details. +// * +// * You should have recieved a copy of the GNU General Public License +// * along with this program; see the file COPYING. +// * If not, write to the Free Software Foundation, Inc., +// * 59 Temple Place - Suite 330, Bostom, MA 02111-1307, USA. +// * +// * 2020 PatrickvL +// * +// * All rights reserved +// * +// ****************************************************************** + +#define LOG_PREFIX CXBXR_MODULE::VTXSH // TODO : Introduce generic HLSL logging + +#include +#include "Shader.h" +#include "core\kernel\init\CxbxKrnl.h" // LOG_TEST_CASE +#include "core\kernel\support\Emu.h" // EmuLog +//#include + +std::string DebugPrependLineNumbers(std::string shaderString) { + std::stringstream shader(shaderString); + auto debugShader = std::stringstream(); + + int i = 1; + for (std::string line; std::getline(shader, line); ) { + auto lineNumber = std::to_string(i++); + auto paddedLineNumber = lineNumber.insert(0, 3 - lineNumber.size(), ' '); + debugShader << "/* " << paddedLineNumber << " */ " << line << "\n"; + } + + return debugShader.str(); +} + +extern HRESULT EmuCompileShader +( + std::string hlsl_str, + const char* shader_profile, + ID3DBlob** ppHostShader, + const char* pSourceName +) +{ + ID3DBlob* pErrors = nullptr; + ID3DBlob* pErrorsCompatibility = nullptr; + HRESULT hRet = 0; + + EmuLog(LOG_LEVEL::DEBUG, "--- HLSL conversion ---"); + EmuLog(LOG_LEVEL::DEBUG, DebugPrependLineNumbers(hlsl_str).c_str()); + EmuLog(LOG_LEVEL::DEBUG, "-----------------------"); + + + UINT flags1 = D3DCOMPILE_OPTIMIZATION_LEVEL3; + + hRet = D3DCompile( + hlsl_str.c_str(), + hlsl_str.length(), + pSourceName, + nullptr, // pDefines + D3D_COMPILE_STANDARD_FILE_INCLUDE, // pInclude // TODO precompile x_* HLSL functions? + "main", // shader entry poiint + shader_profile, + flags1, // flags1 + 0, // flags2 + ppHostShader, // out + &pErrors // ppErrorMsgs out + ); + if (FAILED(hRet)) { + EmuLog(LOG_LEVEL::WARNING, "Shader compile failed. Recompiling in compatibility mode"); + // Attempt to retry in compatibility mode, this allows some vertex-state shaders to compile + // Test Case: Spy vs Spy + flags1 |= D3DCOMPILE_ENABLE_BACKWARDS_COMPATIBILITY | D3DCOMPILE_AVOID_FLOW_CONTROL; + hRet = D3DCompile( + hlsl_str.c_str(), + hlsl_str.length(), + pSourceName, + nullptr, // pDefines + D3D_COMPILE_STANDARD_FILE_INCLUDE, // pInclude // TODO precompile x_* HLSL functions? + "main", // shader entry poiint + shader_profile, + flags1, // flags1 + 0, // flags2 + ppHostShader, // out + &pErrorsCompatibility // ppErrorMsgs out + ); + + if (FAILED(hRet)) { + LOG_TEST_CASE("Couldn't assemble recompiled shader"); + //EmuLog(LOG_LEVEL::WARNING, "Couldn't assemble recompiled shader"); + } + } + + // Determine the log level + auto hlslErrorLogLevel = FAILED(hRet) ? LOG_LEVEL::ERROR2 : LOG_LEVEL::DEBUG; + if (pErrors) { + // Log errors from the initial compilation + EmuLog(hlslErrorLogLevel, "%s", (char*)(pErrors->GetBufferPointer())); + pErrors->Release(); + pErrors = nullptr; + } + + // Failure to recompile in compatibility mode ignored for now + if (pErrorsCompatibility != nullptr) { + pErrorsCompatibility->Release(); + pErrorsCompatibility = nullptr; + } + + LOG_CHECK_ENABLED(LOG_LEVEL::DEBUG) { + if (g_bPrintfOn) { + if (!FAILED(hRet)) { + // Log disassembly + hRet = D3DDisassemble( + (*ppHostShader)->GetBufferPointer(), + (*ppHostShader)->GetBufferSize(), + D3D_DISASM_ENABLE_DEFAULT_VALUE_PRINTS | D3D_DISASM_ENABLE_INSTRUCTION_NUMBERING, + NULL, + &pErrors + ); + if (pErrors) { + EmuLog(hlslErrorLogLevel, "%s", (char*)(pErrors->GetBufferPointer())); + pErrors->Release(); + } + } + } + } + + return hRet; +} diff --git a/src/core/hle/D3D8/Direct3D9/Shader.h b/src/core/hle/D3D8/Direct3D9/Shader.h new file mode 100644 index 000000000..1a89d14a5 --- /dev/null +++ b/src/core/hle/D3D8/Direct3D9/Shader.h @@ -0,0 +1,12 @@ +#pragma once + +#include // std::string +#include // ID3DBlob (via d3d9.h > d3d11shader.h > d3dcommon.h) + +extern HRESULT EmuCompileShader +( + std::string hlsl_str, + const char* shader_profile, + ID3DBlob** ppHostShader, + const char* pSourceName = nullptr +); diff --git a/src/core/hle/D3D8/Direct3D9/VertexShader.cpp b/src/core/hle/D3D8/Direct3D9/VertexShader.cpp index d6b8d1047..21bb9582f 100644 --- a/src/core/hle/D3D8/Direct3D9/VertexShader.cpp +++ b/src/core/hle/D3D8/Direct3D9/VertexShader.cpp @@ -1,11 +1,12 @@ #define LOG_PREFIX CXBXR_MODULE::VTXSH -#include "VertexShader.h" -#include "core\kernel\init\CxbxKrnl.h" -#include "core\kernel\support\Emu.h" +#include "Shader.h" // EmuCompileShader +#include "VertexShader.h" // EmuCompileVertexShader +#include "core\kernel\init\CxbxKrnl.h" // implicit CxbxKrnl_Xbe used in LOG_TEST_CASE +#include "core\kernel\support\Emu.h" // LOG_TEST_CASE (via Logging.h) #include -#include +#include // std::stringstream extern const char* g_vs_model = vs_model_2_a; @@ -181,133 +182,8 @@ void BuildShader(IntermediateVertexShader* pShader, std::stringstream& hlsl) } } -std::string DebugPrependLineNumbers(std::string shaderString) { - std::stringstream shader(shaderString); - auto debugShader = std::stringstream(); - - int i = 1; - for (std::string line; std::getline(shader, line); ) { - auto lineNumber = std::to_string(i++); - auto paddedLineNumber = lineNumber.insert(0, 3 - lineNumber.size(), ' '); - debugShader << "/* " << paddedLineNumber << " */ " << line << "\n"; - } - - return debugShader.str(); -} - -HRESULT CompileHlsl(const std::string& hlsl, ID3DBlob** ppHostShader, const char* pSourceName) -{ - // TODO include header in vertex shader - //xbox::X_VSH_SHADER_HEADER* pXboxVertexShaderHeader = (xbox::X_VSH_SHADER_HEADER*)pXboxFunction; - ID3DBlob* pErrors = nullptr; - ID3DBlob* pErrorsCompatibility = nullptr; - HRESULT hRet = 0; - auto hlslErrorLogLevel = FAILED(hRet) ? LOG_LEVEL::ERROR2 : LOG_LEVEL::DEBUG; - - UINT flags1 = D3DCOMPILE_OPTIMIZATION_LEVEL3; - hRet = D3DCompile( - hlsl.c_str(), - hlsl.length(), - pSourceName, // pSourceName - nullptr, // pDefines - D3D_COMPILE_STANDARD_FILE_INCLUDE, // pInclude // TODO precompile x_* HLSL functions? - "main", // shader entry poiint - g_vs_model, // shader profile - flags1, // flags1 - 0, // flags2 - ppHostShader, // out - &pErrors // ppErrorMsgs out - ); - - // If the shader failed in the default vertex shader model, retry in vs_model_3_0 - // This allows shaders too large for 2_a to be compiled (Test Case: Shenmue 2) - if (FAILED(hRet)) { - if (pErrors) { - // Log HLSL compiler errors - EmuLog(hlslErrorLogLevel, "%s", (char*)(pErrors->GetBufferPointer())); - pErrors->Release(); - pErrors = nullptr; - } - - EmuLog(LOG_LEVEL::WARNING, "Shader compile failed. Retrying with shader model 3.0"); - hRet = D3DCompile( - hlsl.c_str(), - hlsl.length(), - pSourceName, // pSourceName - nullptr, // pDefines - D3D_COMPILE_STANDARD_FILE_INCLUDE, // pInclude // TODO precompile x_* HLSL functions? - "main", // shader entry poiint - vs_model_3_0, // shader profile - flags1, // flags1 - 0, // flags2 - ppHostShader, // out - &pErrors // ppErrorMsgs out - ); - } - - // If the shader failed again, retry in compatibility mode - if (FAILED(hRet)) { - EmuLog(LOG_LEVEL::WARNING, "Shader compile failed. Recompiling in compatibility mode"); - // Attempt to retry in compatibility mode, this allows some vertex-state shaders to compile - // Test Case: Spy vs Spy - flags1 |= D3DCOMPILE_ENABLE_BACKWARDS_COMPATIBILITY | D3DCOMPILE_AVOID_FLOW_CONTROL; - hRet = D3DCompile( - hlsl.c_str(), - hlsl.length(), - pSourceName, // pSourceName - nullptr, // pDefines - D3D_COMPILE_STANDARD_FILE_INCLUDE, // pInclude // TODO precompile x_* HLSL functions? - "main", // shader entry poiint - g_vs_model, // shader profile - flags1, // flags1 - 0, // flags2 - ppHostShader, // out - &pErrorsCompatibility // ppErrorMsgs out - ); - - if (FAILED(hRet)) { - LOG_TEST_CASE("Couldn't assemble vertex shader"); - } - } - - // Determine the log level - if (pErrors) { - // Log errors from the initial compilation - EmuLog(hlslErrorLogLevel, "%s", (char*)(pErrors->GetBufferPointer())); - pErrors->Release(); - pErrors = nullptr; - } - - // Failure to recompile in compatibility mode ignored for now - if (pErrorsCompatibility != nullptr) { - pErrorsCompatibility->Release(); - pErrorsCompatibility = nullptr; - } - - LOG_CHECK_ENABLED(LOG_LEVEL::DEBUG) { - if (g_bPrintfOn) { - if (!FAILED(hRet)) { - // Log disassembly - hRet = D3DDisassemble( - (*ppHostShader)->GetBufferPointer(), - (*ppHostShader)->GetBufferSize(), - D3D_DISASM_ENABLE_DEFAULT_VALUE_PRINTS | D3D_DISASM_ENABLE_INSTRUCTION_NUMBERING, - NULL, - &pErrors - ); - if (pErrors) { - EmuLog(hlslErrorLogLevel, "%s", (char*)(pErrors->GetBufferPointer())); - pErrors->Release(); - } - } - } - } - - return hRet; -} - // recompile xbox vertex shader function -extern HRESULT EmuCompileShader +extern HRESULT EmuCompileVertexShader ( IntermediateVertexShader* pIntermediateShader, ID3DBlob** ppHostShader @@ -326,11 +202,16 @@ extern HRESULT EmuCompileShader hlsl_stream << hlsl_template[1]; // Finish with the HLSL template footer std::string hlsl_str = hlsl_stream.str(); - EmuLog(LOG_LEVEL::DEBUG, "--- HLSL conversion ---"); - EmuLog(LOG_LEVEL::DEBUG, DebugPrependLineNumbers(hlsl_str).c_str()); - EmuLog(LOG_LEVEL::DEBUG, "-----------------------"); - - return CompileHlsl(hlsl_str, ppHostShader, "CxbxVertexShaderTemplate.hlsl"); + HRESULT hRet = EmuCompileShader(hlsl_str, g_vs_model, ppHostShader, "CxbxVertexShaderTemplate.hlsl"); + + if (FAILED(hRet) && (g_vs_model != vs_model_3_0)) { + // If the shader failed in the default vertex shader model, retry in vs_model_3_0 + // This allows shaders too large for 2_a to be compiled (Test Case: Shenmue 2) + EmuLog(LOG_LEVEL::WARNING, "Shader compile failed. Retrying with shader model 3.0"); + hRet = EmuCompileShader(hlsl_str, vs_model_3_0, ppHostShader, "CxbxVertexShaderTemplate.hlsl"); + } + + return hRet; } extern void EmuCompileFixedFunction(ID3DBlob** ppHostShader) @@ -352,7 +233,7 @@ extern void EmuCompileFixedFunction(ID3DBlob** ppHostShader) hlsl << hlslStream.rdbuf(); // Compile the shader - CompileHlsl(hlsl.str(), &pShader, sourceFile.c_str()); + EmuCompileShader(hlsl.str(), g_vs_model, &pShader, sourceFile.c_str()); } *ppHostShader = pShader; @@ -473,7 +354,7 @@ VS_OUTPUT main(const VS_INPUT xIn) } )"; - CompileHlsl(hlsl, &pPassthroughShader, "passthrough.hlsl"); + EmuCompileShader(hlsl, g_vs_model, &pPassthroughShader, "passthrough.hlsl"); } *ppHostShader = pPassthroughShader; diff --git a/src/core/hle/D3D8/Direct3D9/VertexShader.h b/src/core/hle/D3D8/Direct3D9/VertexShader.h index 29d8cc57c..c2ef913b2 100644 --- a/src/core/hle/D3D8/Direct3D9/VertexShader.h +++ b/src/core/hle/D3D8/Direct3D9/VertexShader.h @@ -1,6 +1,4 @@ - -#ifndef DIRECT3D9VERTEXSHADER_H -#define DIRECT3D9VERTEXSHADER_H +#pragma once #include "core\hle\D3D8\XbVertexShader.h" #include "FixedFunctionVertexShaderState.hlsli" @@ -15,7 +13,7 @@ static const char* vs_model_2_a = "vs_2_a"; static const char* vs_model_3_0 = "vs_3_0"; extern const char* g_vs_model; -extern HRESULT EmuCompileShader +extern HRESULT EmuCompileVertexShader ( IntermediateVertexShader* pIntermediateShader, ID3DBlob** ppHostShader @@ -25,4 +23,3 @@ extern void EmuCompileFixedFunction(ID3DBlob** ppHostShader); extern HRESULT EmuCompileXboxPassthrough(ID3DBlob** ppHostShader); -#endif diff --git a/src/core/hle/D3D8/Direct3D9/VertexShaderSource.cpp b/src/core/hle/D3D8/Direct3D9/VertexShaderSource.cpp index a00fa252f..1748651ea 100644 --- a/src/core/hle/D3D8/Direct3D9/VertexShaderSource.cpp +++ b/src/core/hle/D3D8/Direct3D9/VertexShaderSource.cpp @@ -13,7 +13,7 @@ VertexShaderSource g_VertexShaderSource = VertexShaderSource(); ID3DBlob* AsyncCreateVertexShader(IntermediateVertexShader intermediateShader, ShaderKey key) { ID3DBlob* pCompiledShader; - auto hRet = EmuCompileShader( + auto hRet = EmuCompileVertexShader( &intermediateShader, &pCompiledShader ); diff --git a/src/core/hle/D3D8/XbVertexShader.cpp b/src/core/hle/D3D8/XbVertexShader.cpp index fd47d2cbf..baa682fb1 100644 --- a/src/core/hle/D3D8/XbVertexShader.cpp +++ b/src/core/hle/D3D8/XbVertexShader.cpp @@ -38,6 +38,7 @@ #include "core\hle\D3D8\XbVertexBuffer.h" // For CxbxImpl_SetVertexData4f #include "core\hle\D3D8\XbVertexShader.h" #include "core\hle\D3D8\XbD3D8Logging.h" // For DEBUG_D3DRESULT +#include "devices\xbox.h" #include "core\hle\D3D8\XbConvert.h" // For NV2A_VP_UPLOAD_INST, NV2A_VP_UPLOAD_CONST_ID, NV2A_VP_UPLOAD_CONST #include "devices\video\nv2a.h" // For D3DPUSH_DECODE #include "common\Logging.h" // For LOG_INIT @@ -98,7 +99,7 @@ void CxbxVertexShaderSetFlags() // Note : Temporary, until we reliably locate the Xbox internal state for this // See D3DXDeclaratorFromFVF docs https://docs.microsoft.com/en-us/windows/win32/direct3d9/d3dxdeclaratorfromfvf // and https://github.com/reactos/wine/blob/2e8dfbb1ad71f24c41e8485a39df01bb9304127f/dlls/d3dx9_36/mesh.c#L2041 -static xbox::X_D3DVertexShader* XboxVertexShaderFromFVF(DWORD xboxFvf) +static xbox::X_D3DVertexShader* XboxVertexShaderFromFVF(DWORD xboxFvf) // TODO : Rename CxbxFVFToXboxVertexAttributeFormat? { using namespace xbox; diff --git a/src/core/hle/D3D8/XbVertexShader.h b/src/core/hle/D3D8/XbVertexShader.h index 5b82c28a3..aed15821c 100644 --- a/src/core/hle/D3D8/XbVertexShader.h +++ b/src/core/hle/D3D8/XbVertexShader.h @@ -209,6 +209,10 @@ extern void EmuParseVshFunction extern size_t GetVshFunctionSize(const xbox::dword_xt* pXboxFunction); inline boolean VshHandleIsVertexShader(DWORD Handle) { return (Handle & X_D3DFVF_RESERVED0) ? TRUE : FALSE; } +inline boolean VshHandleIsFVF(DWORD Handle) { return !VshHandleIsVertexShader(Handle); } +inline boolean VshHandleIsPassthrough(DWORD Handle) { + return VshHandleIsFVF(Handle) && ((Handle & X_D3DFVF_POSITION_MASK) == X_D3DFVF_XYZRHW); +} inline xbox::X_D3DVertexShader *VshHandleToXboxVertexShader(DWORD Handle) { return (xbox::X_D3DVertexShader *)(Handle & ~X_D3DFVF_RESERVED0);} // Get the number of components represented by the given xbox vertex data type @@ -228,5 +232,4 @@ extern void CxbxImpl_SetVertexShaderInput(DWORD Handle, UINT StreamCount, xbox:: extern void CxbxImpl_SetVertexShaderConstant(INT Register, PVOID pConstantData, DWORD ConstantCount); extern void CxbxImpl_DeleteVertexShader(DWORD Handle); extern void CxbxVertexShaderSetFlags(); -extern HRESULT SetVertexShader(IDirect3DVertexShader* pShader); #endif