gsdx-hw: Cleanup a bit EmulateZbuffer.

Update the comment to reflect recent changes,

Rename DepthMask to MaxDepth,

use a mask shift to get z format,

Do vs cb padding on d3d11.

plugins/GSdx/Renderers/DX11/GSDevice11.h

@@ -41,15 +41,16 @@ public:
 		GSVector4 VertexScale;
 		GSVector4 VertexOffset;
 		GSVector4 Texture_Scale_Offset;
-
+		GSVector2i MaxDepth;
-		GSVector2i DepthMask;
+		GSVector2i pad_vscb;
 
 		VSConstantBuffer()
 		{
 			VertexScale          = GSVector4::zero();
 			VertexOffset         = GSVector4::zero();
 			Texture_Scale_Offset = GSVector4::zero();
-			DepthMask            = GSVector2i(0);
+			MaxDepth             = GSVector2i(0);
+			pad_vscb             = GSVector2i(0);
 		}
 
 		__forceinline bool Update(const VSConstantBuffer* cb)

plugins/GSdx/Renderers/DX11/GSRendererDX11.cpp

@@ -159,24 +159,16 @@ void GSRendererDX11::EmulateZbuffer()
 		m_om_dssel.ztst = ZTST_ALWAYS;
 	}
 
-	uint32 max_z;
+	// On the real GS we appear to do clamping on the max z value the format allows.
-	if (m_context->ZBUF.PSM == PSM_PSMZ32)
+	// Clamping is done after rasterization.
+	const uint32 max_z = 0xFFFFFFFF >> (GSLocalMemory::m_psm[m_context->ZBUF.PSM].fmt * 8);
+	const bool clamp_z = (uint32)(GSVector4i(m_vt.m_max.p).z) > max_z;
+	vs_cb.MaxDepth = GSVector2i(0xFFFFFFFF);
+	if (clamp_z)
 	{
-		max_z = 0xFFFFFFFF;
+		// FIXME: Do z clamping for sprites on vs, triangles on ps.
+		vs_cb.MaxDepth = GSVector2i(max_z);
 	}
-	else if (m_context->ZBUF.PSM == PSM_PSMZ24)
-	{
-		max_z = 0xFFFFFF;
-	}
-	else
-	{
-		max_z = 0xFFFF;
-	}
-
-	// The real GS appears to do no masking based on the Z buffer format and writing larger Z values
-	// than the buffer supports seems to be an error condition on the real GS, causing it to crash.
-	// We are probably receiving bad coordinates from VU1 in these cases.
-	vs_cb.DepthMask = GSVector2i(max_z, max_z);
 
 	GSVertex* v = &m_vertex.buff[0];
 	// Minor optimization of a corner case (it allow to better emulate some alpha test effects)

plugins/GSdx/Renderers/OpenGL/GSDeviceOGL.h

@@ -117,15 +117,15 @@ public:
 
 		GSVector4 TextureOffset;
 
-		GSVector2i DepthMask;
 		GSVector2 PointSize;
+		GSVector2i MaxDepth;
 
 		VSConstantBuffer()
 		{
 			Vertex_Scale_Offset = GSVector4::zero();
 			TextureOffset       = GSVector4::zero();
-			DepthMask           = GSVector2i(0);
 			PointSize           = GSVector2(0);
+			MaxDepth            = GSVector2i(0);
 		}
 
 		__forceinline bool Update(const VSConstantBuffer* cb)

plugins/GSdx/Renderers/OpenGL/GSRendererOGL.cpp

@@ -171,20 +171,17 @@ void GSRendererOGL::EmulateZbuffer()
 		m_om_dssel.ztst = ZTST_ALWAYS;
 	}
 
-	uint32 max_z;
+	// On the real GS we appear to do clamping on the max z value the format allows.
-	if (m_context->ZBUF.PSM == PSM_PSMZ32) {
+	// Clamping is done after rasterization.
-		max_z     = 0xFFFFFFFF;
+	const uint32 max_z = 0xFFFFFFFF >> (GSLocalMemory::m_psm[m_context->ZBUF.PSM].fmt * 8);
-	} else if (m_context->ZBUF.PSM == PSM_PSMZ24) {
+	const bool clamp_z = (uint32)(GSVector4i(m_vt.m_max.p).z) > max_z;
-		max_z     = 0xFFFFFF;
+	vs_cb.MaxDepth = GSVector2i(0xFFFFFFFF);
-	} else {
+	if (clamp_z) {
-		max_z     = 0xFFFF;
+		// FIXME: Do z clamping for sprites on vs, triangles on ps.
+		if (m_vt.m_primclass == GS_SPRITE_CLASS)
+			vs_cb.MaxDepth = GSVector2i(max_z);
 	}
 
-	// The real GS appears to do no masking based on the Z buffer format and writing larger Z values
-	// than the buffer supports seems to be an error condition on the real GS, causing it to crash.
-	// We are probably receiving bad coordinates from VU1 in these cases.
-	vs_cb.DepthMask = GSVector2i(max_z, max_z);
-
 	GSVertex* v = &m_vertex.buff[0];
 	// Minor optimization of a corner case (it allow to better emulate some alpha test effects)
 	if (m_om_dssel.ztst == ZTST_GEQUAL && m_vt.m_eq.z && v[0].XYZ.Z == max_z) {

plugins/GSdx/res/glsl/common_header.glsl

@@ -65,9 +65,9 @@ layout(std140, binding = 20) uniform cb20
 
     vec4  TextureOffset;
 
-    uint  DepthMask;
-    uint  cb20_pad;
     vec2  PointSize;
+    uint  MaxDepth;
+    uint  pad_cb20;
 };
 #endif
 

plugins/GSdx/res/glsl/tfx_vgs.glsl

@@ -44,8 +44,8 @@ void texture_coord()
 
 void vs_main()
 {
-    // Clamp to depth mask, gs doesn't wrap
+    // Clamp to max depth, gs doesn't wrap
-    highp uint z = min(i_z, DepthMask);
+    highp uint z = min(i_z, MaxDepth);
 
     // pos -= 0.05 (1/320 pixel) helps avoiding rounding problems (integral part of pos is usually 5 digits, 0.05 is about as low as we can go)
     // example: ceil(afterseveralvertextransformations(y = 133)) => 134 => line 133 stays empty

plugins/GSdx/res/tfx.fx

@@ -99,8 +99,8 @@ cbuffer cb0
 	float4 VertexScale;
 	float4 VertexOffset;
 	float4 Texture_Scale_Offset;
-	uint DepthMask;
+	uint MaxDepth;
-	uint3 _pad_cb0;
+	uint3 pad_cb0;
 };
 
 cbuffer cb1
@@ -787,8 +787,8 @@ PS_OUTPUT ps_main(PS_INPUT input)
 
 VS_OUTPUT vs_main(VS_INPUT input)
 {
-	// Clamp to depth mask, gs doesn't wrap
+	// Clamp to max depth, gs doesn't wrap
-	input.z = min(input.z, DepthMask);
+	input.z = min(input.z, MaxDepth);
 
 	VS_OUTPUT output;