From 3abddb598391bc7b989830cdc49853a32bcc5ee5 Mon Sep 17 00:00:00 2001
From: Luke Usher <luke.usher@outlook.com>
Date: Fri, 29 Jan 2021 18:22:59 +0000
Subject: [PATCH] vs: use model 2.a primarily, with 3.0 fallback

---
 src/core/hle/D3D8/Direct3D9/Direct3D9.cpp    | 11 -------
 src/core/hle/D3D8/Direct3D9/VertexShader.cpp | 32 ++++++++++++++++++--
 2 files changed, 30 insertions(+), 13 deletions(-)

diff --git a/src/core/hle/D3D8/Direct3D9/Direct3D9.cpp b/src/core/hle/D3D8/Direct3D9/Direct3D9.cpp
index facb4944e..0a44a22ed 100644
--- a/src/core/hle/D3D8/Direct3D9/Direct3D9.cpp
+++ b/src/core/hle/D3D8/Direct3D9/Direct3D9.cpp
@@ -1634,17 +1634,6 @@ void EmuD3DInit()
 		std::cout << "Host D3DCaps : " << g_D3DCaps << "\n";
 		std::cout << "----------------------------------------\n";
 	}
-
-	// AMD compatibility workaround since VS model 3.0 doesn't work as intended with Direct3D9.
-	{
-		D3DADAPTER_IDENTIFIER9 adapter_info;
-		HRESULT status = g_pDirect3D->GetAdapterIdentifier(g_EmuCDPD.Adapter, 0, &adapter_info);
-		// 1002 and 1022 are vendor ids of AMD gpus
-		if (status == D3D_OK && (adapter_info.VendorId == 0x1002 || adapter_info.VendorId == 0x1022)) {
-			g_vs_model = vs_model_2_a;
-			EmuLogInit(LOG_LEVEL::WARNING, "AMD GPU Detected, falling back to shader model 2.X to prevent missing polygons");
-		}
-	}
 }
 
 // cleanup Direct3D
diff --git a/src/core/hle/D3D8/Direct3D9/VertexShader.cpp b/src/core/hle/D3D8/Direct3D9/VertexShader.cpp
index b14b4a27b..0f8753914 100644
--- a/src/core/hle/D3D8/Direct3D9/VertexShader.cpp
+++ b/src/core/hle/D3D8/Direct3D9/VertexShader.cpp
@@ -6,7 +6,7 @@
 
 #include <sstream>
 
-extern const char* g_vs_model = vs_model_3_0;
+extern const char* g_vs_model = vs_model_2_a;
 
 // HLSL generation
 void OutputHlsl(std::stringstream& hlsl, VSH_IMD_OUTPUT& dest)
@@ -201,6 +201,7 @@ HRESULT CompileHlsl(const std::string& hlsl, ID3DBlob** ppHostShader, const char
 	ID3DBlob* pErrors = nullptr;
 	ID3DBlob* pErrorsCompatibility = nullptr;
 	HRESULT             hRet = 0;
+	auto hlslErrorLogLevel = FAILED(hRet) ? LOG_LEVEL::ERROR2 : LOG_LEVEL::DEBUG;
 
 	UINT flags1 = D3DCOMPILE_OPTIMIZATION_LEVEL3;
 	hRet = D3DCompile(
@@ -216,6 +217,34 @@ HRESULT CompileHlsl(const std::string& hlsl, ID3DBlob** ppHostShader, const char
 		ppHostShader, // out
 		&pErrors // ppErrorMsgs out
 	);
+
+	// If the shader failed in the default vertex shader model, retry in vs_model_3_0
+	// This allows shaders too large for 2_a to be compiled (Test Case: Shenmue 2)
+	if (FAILED(hRet)) {
+		if (pErrors) {
+			// Log HLSL compiler errors
+			EmuLog(hlslErrorLogLevel, "%s", (char*)(pErrors->GetBufferPointer()));
+			pErrors->Release();
+			pErrors = nullptr;
+		}
+
+		EmuLog(LOG_LEVEL::WARNING, "Shader compile failed. Retrying with shader model 3.0");
+		hRet = D3DCompile(
+			hlsl.c_str(),
+			hlsl.length(),
+			pSourceName, // pSourceName
+			nullptr, // pDefines
+			D3D_COMPILE_STANDARD_FILE_INCLUDE, // pInclude // TODO precompile x_* HLSL functions?
+			"main", // shader entry poiint
+			vs_model_3_0, // shader profile
+			flags1, // flags1
+			0, // flags2
+			ppHostShader, // out
+			&pErrors // ppErrorMsgs out
+		);
+	}
+
+	// If the shader failed again, retry in compatibility mode
 	if (FAILED(hRet)) {
 		EmuLog(LOG_LEVEL::WARNING, "Shader compile failed. Recompiling in compatibility mode");
 		// Attempt to retry in compatibility mode, this allows some vertex-state shaders to compile
@@ -242,7 +271,6 @@ HRESULT CompileHlsl(const std::string& hlsl, ID3DBlob** ppHostShader, const char
 	}
 
 	// Determine the log level
-	auto hlslErrorLogLevel = FAILED(hRet) ? LOG_LEVEL::ERROR2 : LOG_LEVEL::DEBUG;
 	if (pErrors) {
 		// Log HLSL compiler errors
 		EmuLog(hlslErrorLogLevel, "%s", (char*)(pErrors->GetBufferPointer()));