From 109dd6f373854f7c229e7b7bfe5f6aaf4c5246e5 Mon Sep 17 00:00:00 2001
From: rogerman <rogerman@users.noreply.github.com>
Date: Fri, 10 Nov 2017 09:43:54 -0800
Subject: [PATCH] Render3D: Change the Depth-Equal Test tolerance from +/-512
 to +/-255. - Special thanks to StapleButter for coming to us and informing us
 of this issue.
 https://github.com/TASVideos/desmume/commit/47a71941ace946dcbe5524359be596e4701748b8#commitcomment-25533528
 - This change does imply that GBATEK got this value wrong.
 http://problemkaputt.de/gbatek.htm#ds3dpolygonattributes

---
 desmume/src/OGLRender.cpp     | 4 +++-
 desmume/src/OGLRender_3_2.cpp | 3 ++-
 desmume/src/rasterize.cpp     | 4 ++--
 desmume/src/rasterize.h       | 1 -
 desmume/src/render3D.h        | 1 +
 5 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/desmume/src/OGLRender.cpp b/desmume/src/OGLRender.cpp
index 8d71c8fcc..96ae5a38f 100755
--- a/desmume/src/OGLRender.cpp
+++ b/desmume/src/OGLRender.cpp
@@ -298,6 +298,8 @@ static const char *GeometryVtxShader_100 = {"\
 
 // Fragment Shader GLSL 1.00
 static const char *GeometryFragShader_100 = {"\
+	#define DEPTH_EQUALS_TEST_TOLERANCE 255.0\n\
+	\n\
 	varying vec4 vtxPosition;\n\
 	varying vec2 vtxTexCoord;\n\
 	varying vec4 vtxColor;\n\
@@ -332,7 +334,7 @@ static const char *GeometryFragShader_100 = {"\
 		vec4 newFogAttributes = vec4(0.0, 0.0, 0.0, 0.0);\n\
 		\n\
 		float vertW = (vtxPosition.w == 0.0) ? 0.00000001 : vtxPosition.w;\n\
-		float depthOffset = (polyDepthOffsetMode == 0) ? 0.0 : ((polyDepthOffsetMode == 1) ? -512.0 : 512.0);\n\
+		float depthOffset = (polyDepthOffsetMode == 0) ? 0.0 : ((polyDepthOffsetMode == 1) ? -DEPTH_EQUALS_TEST_TOLERANCE : DEPTH_EQUALS_TEST_TOLERANCE);\n\
 		// hack: when using z-depth, drop some LSBs so that the overworld map in Dragon Quest IV shows up correctly\n\
 		float newFragDepthValue = (stateUseWDepth) ? clamp( ( floor(vtxPosition.w * 4096.0) + depthOffset ) / 16777215.0, 0.0, 1.0 ) : clamp( ( (floor(clamp(((vtxPosition.z/vertW) * 0.5 + 0.5), 0.0, 1.0) * 32767.0) * 512.0) + depthOffset ) / 16777215.0, 0.0, 1.0 );\n\
 		\n\
diff --git a/desmume/src/OGLRender_3_2.cpp b/desmume/src/OGLRender_3_2.cpp
index 335aec200..bb4c92bfd 100644
--- a/desmume/src/OGLRender_3_2.cpp
+++ b/desmume/src/OGLRender_3_2.cpp
@@ -155,6 +155,7 @@ static const char *GeometryVtxShader_150 = {"\
 // Fragment shader for geometry, GLSL 1.50
 static const char *GeometryFragShader_150 = {"\
 	#version 150 \n\
+	#define DEPTH_EQUALS_TEST_TOLERANCE 255.0\n\
 	\n\
 	in vec4 vtxPosition;\n\
 	in vec2 vtxTexCoord;\n\
@@ -204,7 +205,7 @@ static const char *GeometryFragShader_150 = {"\
 		vec4 newFogAttributes = vec4(0.0, 0.0, 0.0, 0.0);\n\
 		\n\
 		float vertW = (vtxPosition.w == 0.0) ? 0.00000001 : vtxPosition.w;\n\
-		float depthOffset = (polyDepthOffsetMode == 0) ? 0.0 : ((polyDepthOffsetMode == 1) ? -512.0 : 512.0);\n\
+		float depthOffset = (polyDepthOffsetMode == 0) ? 0.0 : ((polyDepthOffsetMode == 1) ? -DEPTH_EQUALS_TEST_TOLERANCE : DEPTH_EQUALS_TEST_TOLERANCE);\n\
 		// hack: when using z-depth, drop some LSBs so that the overworld map in Dragon Quest IV shows up correctly\n\
 		float newFragDepthValue = (state.useWDepth) ? clamp( ( floor(vtxPosition.w * 4096.0) + depthOffset ) / 16777215.0, 0.0, 1.0 ) : clamp( ( (floor(clamp(((vtxPosition.z/vertW) * 0.5 + 0.5), 0.0, 1.0) * 32767.0) * 512.0) + depthOffset ) / 16777215.0, 0.0, 1.0 );\n\
 		\n\
diff --git a/desmume/src/rasterize.cpp b/desmume/src/rasterize.cpp
index 8ad4fa712..94a33068f 100644
--- a/desmume/src/rasterize.cpp
+++ b/desmume/src/rasterize.cpp
@@ -532,8 +532,8 @@ public:
 		bool depthFail = false;
 		if (polyAttr.DepthEqualTest_Enable)
 		{
-			const u32 minDepth = max<u32>(0x00000000, dstAttributeDepth - SOFTRASTERIZER_DEPTH_EQUAL_TEST_TOLERANCE);
-			const u32 maxDepth = min<u32>(0x00FFFFFF, dstAttributeDepth + SOFTRASTERIZER_DEPTH_EQUAL_TEST_TOLERANCE);
+			const u32 minDepth = max<u32>(0x00000000, dstAttributeDepth - DEPTH_EQUALS_TEST_TOLERANCE);
+			const u32 maxDepth = min<u32>(0x00FFFFFF, dstAttributeDepth + DEPTH_EQUALS_TEST_TOLERANCE);
 			
 			if (newDepth < minDepth || newDepth > maxDepth)
 			{
diff --git a/desmume/src/rasterize.h b/desmume/src/rasterize.h
index 9a4c62134..0212478ce 100644
--- a/desmume/src/rasterize.h
+++ b/desmume/src/rasterize.h
@@ -21,7 +21,6 @@
 #include "render3D.h"
 #include "gfx3d.h"
 
-#define SOFTRASTERIZER_DEPTH_EQUAL_TEST_TOLERANCE 0x200
 
 extern GPU3DInterface gpu3DRasterize;
 
diff --git a/desmume/src/render3D.h b/desmume/src/render3D.h
index 074e8f2b9..4ebb2a229 100644
--- a/desmume/src/render3D.h
+++ b/desmume/src/render3D.h
@@ -25,6 +25,7 @@
 #include "./filter/filter.h"
 
 #define kUnsetTranslucentPolyID 255
+#define DEPTH_EQUALS_TEST_TOLERANCE 255
 
 class Render3D;