gsdx-ogl: rare corner case when both texture shuffle and date are enabled

In texture shuffle mode the texture data is either RG or BA. It means
that DATE must either checks MSB of G or A.

Close #693

plugins/GSdx/GSDeviceOGL.cpp

@@ -673,6 +673,7 @@ GLuint GSDeviceOGL::CompilePS(PSSelector sel)
 		+ format("#define PS_IIP %d\n", sel.iip)
 		+ format("#define PS_SHUFFLE %d\n", sel.shuffle)
 		+ format("#define PS_READ_BA %d\n", sel.read_ba)
+		+ format("#define PS_WRITE_RG %d\n", sel.write_rg)
 		+ format("#define PS_FBMASK %d\n", sel.fbmask)
 		+ format("#define PS_HDR %d\n", sel.hdr)
 		;

plugins/GSdx/GSDeviceOGL.h

@@ -342,9 +342,10 @@ class GSDeviceOGL : public GSDevice
 				// Shuffle and fbmask effect
 				uint32 shuffle:1;
 				uint32 read_ba:1;
+				uint32 write_rg:1;
 				uint32 fbmask:1;
 
-				uint32 _free1:2;
+				uint32 _free1:1;
 
 				// *** Word 2
 				// Blend and Colclip

plugins/GSdx/GSRendererOGL.cpp

@@ -229,6 +229,11 @@ bool GSRendererOGL::EmulateTextureShuffleAndFbmask(GSDeviceOGL::PSSelector& ps_s
 			}
 		}
 
+		// If date is enabled you need to test the green channel instead of the
+		// alpha channel. Only enable this code in DATE mode to reduce the number
+		// of shader.
+		ps_sel.write_rg = !write_ba && m_context->TEST.DATE;
+
 		// Please bang my head against the wall!
 		// 1/ Reduce the frame mask to a 16 bit format
 		const uint32& m = m_context->FRAME.FBMSK;
@@ -598,7 +603,9 @@ void GSRendererOGL::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sour
 	// DATE: selection of the algorithm. Must be done before blending because GL42 is not compatible with blending
 
 	if (DATE && GLLoader::found_GL_ARB_texture_barrier) {
-		if (m_prim_overlap == PRIM_OVERLAP_NO) {
+		if (m_prim_overlap == PRIM_OVERLAP_NO || m_texture_shuffle) {
+			// It is way too complex to emulate texture shuffle with DATE. So just use
+			// the slow but accurate algo
 			require_barrier = true;
 			DATE_GL45 = true;
 			DATE = false;

plugins/GSdx/res/glsl/tfx_fs.glsl

@@ -471,23 +471,24 @@ void ps_blend(inout vec4 Color, float As)
 
 void ps_main()
 {
-#if (PS_DATE & 3) == 1 && !defined(DISABLE_GL42_image)
-	// DATM == 0
-	// Pixel with alpha equal to 1 will failed
+#if ((PS_DATE & 3) == 1 || (PS_DATE & 3) == 2) && !defined(DISABLE_GL42_image)
+
+#if PS_WRITE_RG == 1
+	// Pseudo 16 bits access.
+	float rt_a = texelFetch(RtSampler, ivec2(gl_FragCoord.xy), 0).g;
+#else
 	float rt_a = texelFetch(RtSampler, ivec2(gl_FragCoord.xy), 0).a;
-	if ((127.5f / 255.0f) < rt_a) { // < 0x80 pass (== 0x80 should not pass)
-#if PS_DATE >= 5
-		discard;
-#else
-		imageStore(img_prim_min, ivec2(gl_FragCoord.xy), ivec4(-1));
-		return;
-#endif
-	}
-#elif (PS_DATE & 3) == 2 && !defined(DISABLE_GL42_image)
-	// DATM == 1
-	// Pixel with alpha equal to 0 will failed
-	float rt_a = texelFetch(RtSampler, ivec2(gl_FragCoord.xy), 0).a;
-	if(rt_a < (127.5f / 255.0f)) { // >= 0x80 pass
+#endif
+
+#if (PS_DATE & 3) == 1
+	// DATM == 0: Pixel with alpha equal to 1 will failed
+	bool bad = (127.5f / 255.0f) < rt_a;
+#elif (PS_DATE & 3) == 2
+	// DATM == 1: Pixel with alpha equal to 0 will failed
+	bool bad = rt_a < (127.5f / 255.0f);
+#endif
+
+	if (bad) {
 #if PS_DATE >= 5
 		discard;
 #else
@@ -495,6 +496,7 @@ void ps_main()
 		return;
 #endif
 	}
+
 #endif
 
 #if PS_DATE == 3 && !defined(DISABLE_GL42_image)

plugins/GSdx/res/glsl_source.h

@@ -1346,23 +1346,24 @@ static const char* tfx_fs_all_glsl =
 	"\n"
 	"void ps_main()\n"
 	"{\n"
-	"#if (PS_DATE & 3) == 1 && !defined(DISABLE_GL42_image)\n"
-	"	// DATM == 0\n"
-	"	// Pixel with alpha equal to 1 will failed\n"
+	"#if ((PS_DATE & 3) == 1 || (PS_DATE & 3) == 2) && !defined(DISABLE_GL42_image)\n"
+	"\n"
+	"#if PS_WRITE_RG == 1\n"
+	"	// Pseudo 16 bits access.\n"
+	"	float rt_a = texelFetch(RtSampler, ivec2(gl_FragCoord.xy), 0).g;\n"
+	"#else\n"
 	"	float rt_a = texelFetch(RtSampler, ivec2(gl_FragCoord.xy), 0).a;\n"
-	"	if ((127.5f / 255.0f) < rt_a) { // < 0x80 pass (== 0x80 should not pass)\n"
-	"#if PS_DATE >= 5\n"
-	"		discard;\n"
-	"#else\n"
-	"		imageStore(img_prim_min, ivec2(gl_FragCoord.xy), ivec4(-1));\n"
-	"		return;\n"
-	"#endif\n"
-	"	}\n"
-	"#elif (PS_DATE & 3) == 2 && !defined(DISABLE_GL42_image)\n"
-	"	// DATM == 1\n"
-	"	// Pixel with alpha equal to 0 will failed\n"
-	"	float rt_a = texelFetch(RtSampler, ivec2(gl_FragCoord.xy), 0).a;\n"
-	"	if(rt_a < (127.5f / 255.0f)) { // >= 0x80 pass\n"
+	"#endif\n"
+	"\n"
+	"#if (PS_DATE & 3) == 1\n"
+	"	// DATM == 0: Pixel with alpha equal to 1 will failed\n"
+	"	bool bad = (127.5f / 255.0f) < rt_a;\n"
+	"#elif (PS_DATE & 3) == 2\n"
+	"	// DATM == 1: Pixel with alpha equal to 0 will failed\n"
+	"	bool bad = rt_a < (127.5f / 255.0f);\n"
+	"#endif\n"
+	"\n"
+	"	if (bad) {\n"
 	"#if PS_DATE >= 5\n"
 	"		discard;\n"
 	"#else\n"
@@ -1370,6 +1371,7 @@ static const char* tfx_fs_all_glsl =
 	"		return;\n"
 	"#endif\n"
 	"	}\n"
+	"\n"
 	"#endif\n"
 	"\n"
 	"#if PS_DATE == 3 && !defined(DISABLE_GL42_image)\n"