Fix various issues with zfreeze implemntation.

Results are still not correct, but things are getting closer.

 * Don't cull CULLALL primitives so early so they can be used as reference
        planes.
 * Convert CalculateZSlope to screenspace coordinates.
 * Convert Pixelshader to screenspace coordinates (instead of worldspace
        xy coordinates, which is totally wrong)
 * Divide depth by 2^24 instead of clamping to 0.0-1.0 as was done
        before.

Progress:
 * Rouge Squadron 2/3 appear correct in game (videos in rs2 save file
         selection are missing)
 * Shadows draw 100% correctly in NHL 2003.
 * Mario golf menu renders correctly.
 * NFS: HP2, shadows sometimes render on top of car or below the road.
 * Mario Tennis, courts and shadows render correctly, but at wrong depth
 * Blood Omen 2, doesn't work.

Source/Core/VideoBackends/D3D/VertexManager.cpp

@@ -186,6 +186,10 @@ void VertexManager::vFlush(bool useDstAlpha)
 		CalculateZSlope(stride);
 	}
 
+	// if cull mode is CULL_ALL, ignore triangles and quads
+	if (bpmem.genMode.cullmode == GenMode::CULL_ALL && current_primitive_type == PRIMITIVE_TRIANGLES)
+		return;
+
 	VertexLoaderManager::GetCurrentVertexFormat()->SetupVertexPointers();
 	g_renderer->ApplyState(useDstAlpha);
 

Source/Core/VideoBackends/OGL/VertexManager.cpp

@@ -145,6 +145,10 @@ void VertexManager::vFlush(bool useDstAlpha)
 		CalculateZSlope(stride);
 	}
 
+	// if cull mode is CULL_ALL, ignore triangles and quads
+	if (bpmem.genMode.cullmode == GenMode::CULL_ALL && current_primitive_type == PRIMITIVE_TRIANGLES)
+		return;
+
 	// Makes sure we can actually do Dual source blending
 	bool dualSourcePossible = g_ActiveConfig.backend_info.bSupportsDualSourceBlend;
 

Source/Core/VideoCommon/PixelShaderGen.cpp

@@ -546,7 +546,7 @@ static inline void GeneratePixelShader(T& out, DSTALPHA_MODE dstAlphaMode, API_T
 	{
 		if (bpmem.genMode.zfreeze)
 		{
-			out.Write("\tdepth = " I_ZSLOPE".z + " I_ZSLOPE".x * (clipPos.x / clipPos.w) + " I_ZSLOPE".y * (clipPos.y / clipPos.w);\n");
+			out.Write("\tdepth = float(" I_ZSLOPE".z + " I_ZSLOPE".x * rawpos.x + " I_ZSLOPE".y * rawpos.y) / float(0xffffff);\n");
 		}
 		else
 		{
@@ -569,7 +569,7 @@ static inline void GeneratePixelShader(T& out, DSTALPHA_MODE dstAlphaMode, API_T
 	{
 		if (bpmem.genMode.zfreeze)
 		{
-			out.Write("\tdepth = " I_ZSLOPE".z + " I_ZSLOPE".x * (clipPos.x / clipPos.w) + " I_ZSLOPE".y * (clipPos.y / clipPos.w);\n");
+			out.Write("\tdepth = float(" I_ZSLOPE".z + " I_ZSLOPE".x * rawpos.x + " I_ZSLOPE".y * rawpos.y) / float(0xffffff);\n");
 		}
 		else
 		{

Source/Core/VideoCommon/VertexLoaderManager.cpp

@@ -149,11 +149,8 @@ int RunVertices(int vtx_attr_group, int primitive, int count, DataReader src, bo
 	if ((int)src.size() < size)
 		return -1;
 
-	if (skip_drawing || (bpmem.genMode.cullmode == GenMode::CULL_ALL && primitive < 5))
+	if (skip_drawing)
-	{
-		// if cull mode is CULL_ALL, ignore triangles and quads
 		return size;
-	}
 
 	// If the native vertex format changed, force a flush.
 	if (loader->m_native_vertex_format != s_current_vtx_fmt)

Source/Core/VideoCommon/VertexManagerBase.cpp

@@ -259,11 +259,12 @@ void VertexManager::CalculateZSlope(u32 stride)
 
 		VertexShaderManager::TransformToClipSpace(&vtx[i * 3], &out[i * 4]);
 
-		// viewport offset ignored because we only look at coordinate differences.
+		// Transform to Screenspace
-		out[0 + i * 4] = out[0 + i * 4] / out[3 + i * 4] * xfmem.viewport.wd;
+		out[0 + i * 4] = out[0 + i * 4] / out[3 + i * 4] * xfmem.viewport.wd + (xfmem.viewport.xOrig - 342);
-		out[1 + i * 4] = out[1 + i * 4] / out[3 + i * 4] * xfmem.viewport.ht;
+		out[1 + i * 4] = out[1 + i * 4] / out[3 + i * 4] * xfmem.viewport.ht + (xfmem.viewport.yOrig - 342);
 		out[2 + i * 4] = out[2 + i * 4] / out[3 + i * 4] * xfmem.viewport.zRange + xfmem.viewport.farZ;
 	}
+
 	float dx31 = out[8] - out[0];
 	float dx12 = out[0] - out[4];
 	float dy12 = out[1] - out[5];
@@ -277,7 +278,7 @@ void VertexManager::CalculateZSlope(u32 stride)
 
 	float slope_dfdx = -a / c;
 	float slope_dfdy = -b / c;
-	float slope_f0 = out[2];
+	float slope_f0 = out[2] - (out[0] * slope_dfdx + out[1] * slope_dfdy);
 
 	PixelShaderManager::SetZSlope(slope_dfdx, slope_dfdy, slope_f0);
 }