gsdx state: post fix depth tracing

The main FindMinMax methods is perf critical so instead I created a separate function
to ensure the constness of the depth

Fix letter regression on Xenosaga3

plugins/GSdx/GSRendererHW.cpp

@@ -1149,7 +1149,7 @@ bool GSRendererHW::OI_FFXII(GSTexture* rt, GSTexture* ds, GSTextureCache::Source
 				m_vertex.head = m_vertex.tail = m_vertex.next = 4;
 				m_index.tail = 6;
 
-				m_vt.Update(m_vertex.buff, m_index.buff, m_index.tail, GS_TRIANGLE_CLASS);
+				m_vt.Update(m_vertex.buff, m_index.buff, m_vertex.tail, m_index.tail, GS_TRIANGLE_CLASS);
 			}
 			else
 			{
@@ -1197,7 +1197,7 @@ bool GSRendererHW::OI_MetalSlug6(GSTexture* rt, GSTexture* ds, GSTextureCache::S
 		}
 	}
 
-	m_vt.Update(m_vertex.buff, m_index.buff, m_index.tail, m_vt.m_primclass);
+	m_vt.Update(m_vertex.buff, m_index.buff, m_vertex.tail, m_index.tail, m_vt.m_primclass);
 
 	return true;
 }

plugins/GSdx/GSState.cpp

@@ -1583,7 +1583,7 @@ void GSState::FlushPrim()
 
 		if(GSLocalMemory::m_psm[m_context->FRAME.PSM].fmt < 3 && GSLocalMemory::m_psm[m_context->ZBUF.PSM].fmt < 3)
 		{
-			m_vt.Update(m_vertex.buff, m_index.buff, m_index.tail, GSUtil::GetPrimClass(PRIM->PRIM));
+			m_vt.Update(m_vertex.buff, m_index.buff, m_vertex.tail, m_index.tail, GSUtil::GetPrimClass(PRIM->PRIM));
 
 			try {
 				Draw();

plugins/GSdx/GSVertexTrace.cpp

@@ -58,7 +58,7 @@ GSVertexTrace::GSVertexTrace(const GSState* state)
 	InitUpdate(GS_SPRITE_CLASS);
 }
 
-void GSVertexTrace::Update(const void* vertex, const uint32* index, int count, GS_PRIM_CLASS primclass)
+void GSVertexTrace::Update(const void* vertex, const uint32* index, int v_count, int i_count, GS_PRIM_CLASS primclass)
 {
 	m_primclass = primclass;
 
@@ -67,12 +67,17 @@ void GSVertexTrace::Update(const void* vertex, const uint32* index, int count, G
 	uint32 fst = m_state->PRIM->FST;
 	uint32 color = !(m_state->PRIM->TME && m_state->m_context->TEX0.TFX == TFX_DECAL && m_state->m_context->TEX0.TCC);
 
-	(this->*m_fmm[color][fst][tme][iip][primclass])(vertex, index, count);
+	(this->*m_fmm[color][fst][tme][iip][primclass])(vertex, index, i_count);
 
 	m_eq.value = (m_min.c == m_max.c).mask() | ((m_min.p == m_max.p).mask() << 16) | ((m_min.t == m_max.t).mask() << 20);
 
 	m_alpha.valid = false;
 
+	// I'm not sure of the cost. In doubt let's do it only when depth is enabled
+	if(m_state->m_context->TEST.ZTE == 1 && m_state->m_context->TEST.ZTST > ZTST_ALWAYS) {
+		CorrectDepthTrace(vertex, v_count);
+	}
+
 	if(m_state->PRIM->TME)
 	{
 		const GIFRegTEX1& TEX1 = m_state->m_context->TEX1;
@@ -495,3 +500,44 @@ void GSVertexTrace::FindMinMax(const void* vertex, const uint32* index, int coun
 		m_max.c = GSVector4i::zero();
 	}
 }
+
+void GSVertexTrace::CorrectDepthTrace(const void* vertex, int count)
+{
+	if (m_eq.z == 0)
+		return;
+
+	// FindMinMax isn't accurate for the depth value. Lsb bit is always 0.
+	// The code below will check that depth value is really constant
+	// and will update m_min/m_max/m_eq accordingly
+	//
+	// Really impact Xenosaga3
+	//
+	// Hopefully function is barely called so AVX/SSE will be useless here
+
+
+	const GSVertex* RESTRICT v = (GSVertex*)vertex;
+	uint32 z = v[0].XYZ.Z;
+
+	// ought to check only 1/2 for sprite
+	if (z & 1) {
+		// Check that first bit is always 1
+		for (int i = 0; i < count; i++) {
+			z &= v[i].XYZ.Z;
+		}
+	} else {
+		// Check that first bit is always 0
+		for (int i = 0; i < count; i++) {
+			z |= v[i].XYZ.Z;
+		}
+	}
+
+	if (z == v[0].XYZ.Z) {
+		m_min.p.z = z;
+		m_max.p.z = z;
+		m_eq.z = 1;
+	} else {
+		m_min.p.z = z & ~1;
+		m_max.p.z = z |  1;
+		m_eq.z = 0;
+	}
+}

plugins/GSdx/GSVertexTrace.h

@@ -74,7 +74,9 @@ public:
 	GSVertexTrace(const GSState* state);
 	virtual ~GSVertexTrace() {}
 
-	void Update(const void* vertex, const uint32* index, int count, GS_PRIM_CLASS primclass);
+	void Update(const void* vertex, const uint32* index, int v_count, int i_count, GS_PRIM_CLASS primclass);
 
 	bool IsLinear() const {return m_filter.linear;}
+
+	void CorrectDepthTrace(const void* vertex, int count);
 };