GSdx: Added a simple workaround for the 32-bit z problem talked about in the comments of r4956. Since sprites are flat and there was an unused vertex member (t.w), I just decided to pass the raw uint32 value in that to the scanline drawing function. It does not fix triangles and other primitive types, of course. The ideal solution would be to break z into two parts (like 8:24 bits, and only care about the upper part when not zero), interpolate separately and rejoin when needed, it is just too hard to add another variable when the assembly code is already so tightly optimized to use every register.

(HW mode z-test expects a float input, so this trick cannot be done there.)

git-svn-id: http://pcsx2.googlecode.com/svn/trunk@4966 96395faa-99c1-11dd-bbfe-3dabce05a288

plugins/GSdx/GSRendererSW.cpp

@@ -766,6 +766,11 @@ void GSRendererSW::VertexKick(bool skip)
 
 	dst.c = GSVector4::rgba32(m_v.RGBAQ.u32[0], 7);
 
+	if(prim == GS_SPRITE)
+	{
+		dst.t.u32[3] = m_v.XYZ.Z;
+	}
+
 	int count = 0;
 
 	if(GSVertexSW* v = DrawingKick<prim>(skip, count))

plugins/GSdx/GSSetupPrimCodeGenerator.x86.avx.cpp

@@ -123,6 +123,12 @@ void GSSetupPrimCodeGenerator::Depth()
 
 		if(m_en.z)
 		{
+			// uint32 z is bypassed in t.w
+
+			vmovdqa(xmm0, ptr[ecx + offsetof(GSVertexSW, t)]);
+			vpshufd(xmm0, xmm0, _MM_SHUFFLE(3, 3, 3, 3));
+
+/*			
 			// GSVector4 z = p.zzzz();
 
 			vshufps(xmm0, xmm0, _MM_SHUFFLE(2, 2, 2, 2));
@@ -149,6 +155,7 @@ void GSSetupPrimCodeGenerator::Depth()
 
 				vcvttps2dq(xmm0, xmm0);
 			}
+*/			
 
 			vmovdqa(ptr[&m_local.p.z], xmm0);
 		}

plugins/GSdx/GSSetupPrimCodeGenerator.x86.cpp

@@ -128,6 +128,12 @@ void GSSetupPrimCodeGenerator::Depth()
 
 		if(m_en.z)
 		{
+			// uint32 z is bypassed in t.w
+
+			movdqa(xmm0, ptr[ecx + offsetof(GSVertexSW, t)]);
+			pshufd(xmm0, xmm0, _MM_SHUFFLE(3, 3, 3, 3));
+			
+			/*
 			// GSVector4 z = p.zzzz();
 
 			shufps(xmm0, xmm0, _MM_SHUFFLE(2, 2, 2, 2));
@@ -154,6 +160,7 @@ void GSSetupPrimCodeGenerator::Depth()
 
 				cvttps2dq(xmm0, xmm0);
 			}
+			*/
 
 			movdqa(ptr[&m_local.p.z], xmm0);
 		}