Render3D: Change the Depth-Equal Test tolerance from +/-512 to +/-255.

- Special thanks to StapleButter for coming to us and informing us of this issue. 47a71941ac (commitcomment-25533528)
- This change does imply that GBATEK got this value wrong. http://problemkaputt.de/gbatek.htm#ds3dpolygonattributes

desmume/src/OGLRender.cpp

@@ -298,6 +298,8 @@ static const char *GeometryVtxShader_100 = {"\
 
 // Fragment Shader GLSL 1.00
 static const char *GeometryFragShader_100 = {"\
+	#define DEPTH_EQUALS_TEST_TOLERANCE 255.0\n\
+	\n\
 	varying vec4 vtxPosition;\n\
 	varying vec2 vtxTexCoord;\n\
 	varying vec4 vtxColor;\n\
@@ -332,7 +334,7 @@ static const char *GeometryFragShader_100 = {"\
 		vec4 newFogAttributes = vec4(0.0, 0.0, 0.0, 0.0);\n\
 		\n\
 		float vertW = (vtxPosition.w == 0.0) ? 0.00000001 : vtxPosition.w;\n\
-		float depthOffset = (polyDepthOffsetMode == 0) ? 0.0 : ((polyDepthOffsetMode == 1) ? -512.0 : 512.0);\n\
+		float depthOffset = (polyDepthOffsetMode == 0) ? 0.0 : ((polyDepthOffsetMode == 1) ? -DEPTH_EQUALS_TEST_TOLERANCE : DEPTH_EQUALS_TEST_TOLERANCE);\n\
 		// hack: when using z-depth, drop some LSBs so that the overworld map in Dragon Quest IV shows up correctly\n\
 		float newFragDepthValue = (stateUseWDepth) ? clamp( ( floor(vtxPosition.w * 4096.0) + depthOffset ) / 16777215.0, 0.0, 1.0 ) : clamp( ( (floor(clamp(((vtxPosition.z/vertW) * 0.5 + 0.5), 0.0, 1.0) * 32767.0) * 512.0) + depthOffset ) / 16777215.0, 0.0, 1.0 );\n\
 		\n\

desmume/src/OGLRender_3_2.cpp

@@ -155,6 +155,7 @@ static const char *GeometryVtxShader_150 = {"\
 // Fragment shader for geometry, GLSL 1.50
 static const char *GeometryFragShader_150 = {"\
 	#version 150 \n\
+	#define DEPTH_EQUALS_TEST_TOLERANCE 255.0\n\
 	\n\
 	in vec4 vtxPosition;\n\
 	in vec2 vtxTexCoord;\n\
@@ -204,7 +205,7 @@ static const char *GeometryFragShader_150 = {"\
 		vec4 newFogAttributes = vec4(0.0, 0.0, 0.0, 0.0);\n\
 		\n\
 		float vertW = (vtxPosition.w == 0.0) ? 0.00000001 : vtxPosition.w;\n\
-		float depthOffset = (polyDepthOffsetMode == 0) ? 0.0 : ((polyDepthOffsetMode == 1) ? -512.0 : 512.0);\n\
+		float depthOffset = (polyDepthOffsetMode == 0) ? 0.0 : ((polyDepthOffsetMode == 1) ? -DEPTH_EQUALS_TEST_TOLERANCE : DEPTH_EQUALS_TEST_TOLERANCE);\n\
 		// hack: when using z-depth, drop some LSBs so that the overworld map in Dragon Quest IV shows up correctly\n\
 		float newFragDepthValue = (state.useWDepth) ? clamp( ( floor(vtxPosition.w * 4096.0) + depthOffset ) / 16777215.0, 0.0, 1.0 ) : clamp( ( (floor(clamp(((vtxPosition.z/vertW) * 0.5 + 0.5), 0.0, 1.0) * 32767.0) * 512.0) + depthOffset ) / 16777215.0, 0.0, 1.0 );\n\
 		\n\

desmume/src/rasterize.cpp

@@ -532,8 +532,8 @@ public:
 		bool depthFail = false;
 		if (polyAttr.DepthEqualTest_Enable)
 		{
-			const u32 minDepth = max<u32>(0x00000000, dstAttributeDepth - SOFTRASTERIZER_DEPTH_EQUAL_TEST_TOLERANCE);
-			const u32 maxDepth = min<u32>(0x00FFFFFF, dstAttributeDepth + SOFTRASTERIZER_DEPTH_EQUAL_TEST_TOLERANCE);
+			const u32 minDepth = max<u32>(0x00000000, dstAttributeDepth - DEPTH_EQUALS_TEST_TOLERANCE);
+			const u32 maxDepth = min<u32>(0x00FFFFFF, dstAttributeDepth + DEPTH_EQUALS_TEST_TOLERANCE);
 			
 			if (newDepth < minDepth || newDepth > maxDepth)
 			{

desmume/src/rasterize.h

@@ -21,7 +21,6 @@
 #include "render3D.h"
 #include "gfx3d.h"
 
-#define SOFTRASTERIZER_DEPTH_EQUAL_TEST_TOLERANCE 0x200
 
 extern GPU3DInterface gpu3DRasterize;
 

desmume/src/render3D.h

@@ -25,6 +25,7 @@
 #include "./filter/filter.h"
 
 #define kUnsetTranslucentPolyID 255
+#define DEPTH_EQUALS_TEST_TOLERANCE 255
 
 class Render3D;