Calculate the fog factor in the vertex shader

(cherry picked from commit 3c3ede07da1f0829697cc1635058b5dc4ec610d2)

src/core/hle/D3D8/Direct3D9/Direct3D9.cpp

@@ -6491,9 +6491,18 @@ void UpdateFixedFunctionVertexShaderState()
 	ffShaderState.PointSprite.RenderUpscaleFactor = g_RenderUpscaleFactor;
 
 	// Fog
+	// Determine how the fog depth is transformed into the fog factor
+	auto fogEnable = XboxRenderStates.GetXboxRenderState(X_D3DRS_FOGENABLE);
+	auto fogTableMode = XboxRenderStates.GetXboxRenderState(X_D3DRS_FOGTABLEMODE);
+	ffShaderState.Fog.Enable = fogEnable;
+	// FIXME remove when fixed function PS is implemented
+	// Note if we are using the fixed function pixel shader
+	// We only want to produce the fog depth value in the VS, not the fog factor
+	auto psIsFixedFunction = g_pXbox_PixelShader == nullptr;
+	ffShaderState.Fog.TableMode = psIsFixedFunction ? D3DFOG_NONE : fogTableMode;
+
 	// Determine how fog depth is calculated
-	if (XboxRenderStates.GetXboxRenderState(X_D3DRS_FOGENABLE) &&
-		XboxRenderStates.GetXboxRenderState(X_D3DRS_FOGTABLEMODE) != D3DFOG_NONE) {
+	if (fogEnable && fogTableMode != D3DFOG_NONE) {
 		auto proj = &ffShaderState.Transforms.Projection;
 
 		if (XboxRenderStates.GetXboxRenderState(X_D3DRS_RANGEFOGENABLE)) {
@@ -6513,6 +6522,13 @@ void UpdateFixedFunctionVertexShaderState()
 			// JSRF (non-compliant projection matrix)
 			ffShaderState.Fog.DepthMode = FixedFunctionVertexShader::FOG_DEPTH_W;
 		}
+
+		auto density = XboxRenderStates.GetXboxRenderState(X_D3DRS_FOGDENSITY);
+		auto fogStart = XboxRenderStates.GetXboxRenderState(X_D3DRS_FOGSTART);
+		auto fogEnd = XboxRenderStates.GetXboxRenderState(X_D3DRS_FOGEND);
+		ffShaderState.Fog.Density = *reinterpret_cast<float*>(&density);
+		ffShaderState.Fog.Start = *reinterpret_cast<float*>(&fogStart);
+		ffShaderState.Fog.End = *reinterpret_cast<float*>(&fogEnd);
 	}
 	else {
 		ffShaderState.Fog.DepthMode = FixedFunctionVertexShader::FOG_DEPTH_NONE;

src/core/hle/D3D8/Direct3D9/FixedFunctionVertexShader.hlsl

@@ -273,11 +273,11 @@ Material DoMaterial(const uint index, const uint diffuseReg, const uint specular
 
 float DoFog(const VS_INPUT xIn)
 {
-    // TODO implement properly
-    // Until we have pixel shader HLSL we are still leaning on D3D renderstates for fogging
-    // So we are not doing any fog density calculations here
+    if (!state.Fog.Enable)
+        return 1; // No fog!
     // http://developer.download.nvidia.com/assets/gamedev/docs/Fog2.pdf
 
+    // Obtain the fog depth value 'd'
     float fogDepth;
 
     if (state.Fog.DepthMode == FixedFunctionVertexShader::FOG_DEPTH_NONE)
@@ -289,7 +289,19 @@ float DoFog(const VS_INPUT xIn)
     if (state.Fog.DepthMode == FixedFunctionVertexShader::FOG_DEPTH_W)
         fogDepth = Projection.Position.w;
 
-    return fogDepth;
+    // Calculate the fog factor
+    // Some of this might be better done in the pixel shader?
+    float fogFactor;
+    if (state.Fog.TableMode == FixedFunctionVertexShader::FOG_TABLE_NONE)
+        fogFactor = fogDepth;
+    if (state.Fog.TableMode == FixedFunctionVertexShader::FOG_TABLE_EXP)
+        fogFactor = 1 / exp(fogDepth * state.Fog.Density); // 1 / e^(d * density)
+    if (state.Fog.TableMode == FixedFunctionVertexShader::FOG_TABLE_EXP2)
+        fogFactor = 1 / exp(pow(fogDepth * state.Fog.Density, 2)); // 1 / e^((d * density)^2)
+    if (state.Fog.TableMode == FixedFunctionVertexShader::FOG_TABLE_LINEAR)
+        fogFactor = (state.Fog.End - fogDepth) / (state.Fog.End - state.Fog.Start); // (end - d) / (end - start)
+
+    return fogFactor;
 }
 
 float4 DoTexCoord(const uint stage, const VS_INPUT xIn)

src/core/hle/D3D8/Direct3D9/FixedFunctionVertexShaderState.hlsli

@@ -29,6 +29,12 @@ namespace FixedFunctionVertexShader {
     const float FOG_DEPTH_W = 2;
 	// Fog depth is based distance of the vertex from the eye position
     const float FOG_DEPTH_RANGE = 3;
+
+    // https://docs.microsoft.com/en-us/windows/win32/direct3d9/fog-formulas
+    const float FOG_TABLE_NONE = 0;
+    const float FOG_TABLE_EXP = 1;
+    const float FOG_TABLE_EXP2 = 2;
+    const float FOG_TABLE_LINEAR = 3;
 }
 
 // Shared HLSL structures
@@ -121,7 +127,12 @@ struct TextureState {
 };
 
 struct Fog {
+    alignas(16) float Enable;
     alignas(16) float DepthMode;
+    alignas(16) float TableMode;
+    alignas(16) float Density; // EXP fog density
+    alignas(16) float Start; // LINEAR fog start
+    alignas(16) float End; // LINEAR fog end
 };
 
 // Vertex lighting