gsdx-ogl: fix the recently broken advance date feature

Now it is really working with a 2 stages shaders but it is still slow.

plugins/GSdx/GSDeviceOGL.cpp

@@ -562,11 +562,13 @@ GLuint GSDeviceOGL::CreateSampler(bool bilinear, bool tau, bool tav)
 	return sampler;
 }
 
-void GSDeviceOGL::InitPrimDateTexture(int w, int h)
+void GSDeviceOGL::InitPrimDateTexture(GSTexture* rt)
 {
+	const GSVector2i& rtsize = rt->GetSize();
+
 	// Create a texture to avoid the useless clean@0
 	if (m_date.t == NULL)
-		m_date.t = CreateTexture(w, h, GL_R32I);
+		m_date.t = CreateTexture(rtsize.x, rtsize.y, GL_R32I);
 
 	ClearRenderTarget_ui(m_date.t, 0x0FFFFFFF);
 
@@ -574,19 +576,10 @@ void GSDeviceOGL::InitPrimDateTexture(int w, int h)
 	gl_BindTextureUnit(5, static_cast<GSTextureOGL*>(m_date.t)->GetID());
 #endif
 
-	BindDateTexture();
-}
-
-void GSDeviceOGL::BindDateTexture()
-{
-	// TODO: multibind?
-	// GLuint textures[1] = {static_cast<GSTextureOGL*>(m_date.t)->GetID()};
-	// gl_BindImageTextures(2, 1, textures);
-	//gl_BindImageTexture(2, 0, 0, true, 0, GL_READ_WRITE, GL_R32I);
-
 #ifndef ENABLE_GLES
 	gl_BindImageTexture(2, static_cast<GSTextureOGL*>(m_date.t)->GetID(), 0, false, 0, GL_READ_WRITE, GL_R32I);
 #endif
+	gl_BindTextureUnit(3, static_cast<GSTextureOGL*>(rt)->GetID());
 }
 
 void GSDeviceOGL::RecycleDateTexture()

plugins/GSdx/GSDeviceOGL.h

@@ -610,9 +610,8 @@ class GSDeviceOGL : public GSDevice
 	GSTexture* CreateDepthStencil(int w, int h, bool msaa, int format = 0);
 	GSTexture* CreateTexture(int w, int h, int format = 0);
 	GSTexture* CreateOffscreen(int w, int h, int format = 0);
-	void InitPrimDateTexture(int w, int h);
+	void InitPrimDateTexture(GSTexture* rt);
 	void RecycleDateTexture();
-	void BindDateTexture();
 
 	GSTexture* CopyOffscreen(GSTexture* src, const GSVector4& sr, int w, int h, int format = 0);
 

plugins/GSdx/GSRendererOGL.cpp

@@ -225,9 +225,11 @@ void GSRendererOGL::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sour
 		// Reduce the quantity of clean function
 		glScissor( ri.x, ri.y, ri.width(), ri.height() );
 
-		// Note at the moment OGL has always stencil. Rt can be disabled
-		if(dev->HasStencil() && !advance_DATE)
-		{
+		// Must be done here to avoid any GL state pertubation (clear function...)
+		// Create an r32ui image that will containt primitive ID
+		if (advance_DATE) {
+			dev->InitPrimDateTexture(rt);
+		} else {
 			GSVector4 s = GSVector4(rtscale.x / rtsize.x, rtscale.y / rtsize.y);
 
 			GSVector4 src = (b * s.xyxy()).sat(o.zzyy());
@@ -244,11 +246,6 @@ void GSRendererOGL::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sour
 			dev->SetupDATE(rt, ds, vertices, m_context->TEST.DATM);
 		}
 
-		// Must be done here to avoid any GL state pertubation (clear function...)
-		// Create an r32ui image that will containt primitive ID
-		if (advance_DATE)
-			dev->InitPrimDateTexture(rtsize.x, rtsize.y);
-
 		// Restore the scissor state
 		ri = GLState::scissor;
 		glScissor( ri.x, ri.y, ri.width(), ri.height() );
@@ -493,8 +490,6 @@ void GSRendererOGL::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sour
 
 	GSVector4i scissor = GSVector4i(GSVector4(rtscale).xyxy() * context->scissor.in).rintersect(GSVector4i(rtsize).zwxy());
 
-	dev->OMSetRenderTargets(rt, ds, &scissor);
-
 	uint8 afix = context->ALPHA.FIX;
 
 	SetupIA();
@@ -507,6 +502,10 @@ void GSRendererOGL::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sour
 		// Note: do it at the beginning because the clean will dirty the FBO state
 		//dev->InitPrimDateTexture(rtsize.x, rtsize.y);
 
+		// I don't know how much is it legal to mount rt as Texture/RT. No write is done.
+		// In doubt let's detach RT.
+		dev->OMSetRenderTargets(NULL, ds, &scissor);
+
 		// Don't write anything on the color buffer
 		dev->OMSetWriteBuffer(GL_NONE);
 		// Compute primitiveID max that pass the date test
@@ -525,6 +524,8 @@ void GSRendererOGL::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sour
 #endif
 	}
 
+	dev->OMSetRenderTargets(rt, ds, &scissor);
+
 	if(context->TEST.DoFirstPass())
 	{
 		dev->DrawIndexedPrimitive();

plugins/GSdx/res/glsl/tfx_fs.glsl

@@ -73,6 +73,7 @@ layout(bindless_sampler, location = 1) uniform sampler2D PaletteSampler;
 #else
 layout(binding = 0) uniform sampler2D TextureSampler;
 layout(binding = 1) uniform sampler2D PaletteSampler;
+layout(binding = 3) uniform sampler2D RtSampler; // note 2 already use by the image below
 #endif
 
 #ifndef DISABLE_GL42_image
@@ -437,6 +438,20 @@ void ps_main()
 #if !pGL_ES
 void ps_main()
 {
+#if PS_DATE == 1 && !defined(DISABLE_GL42_image)
+    // DATM == 0
+    // Pixel with alpha equal to 1 will failed
+	float rt_a = texelFetch(RtSampler, ivec2(gl_FragCoord.xy), 0).a;
+	if ((127.5f / 255.0f) < rt_a) // < 0x80 pass (== 0x80 should not pass)
+		discard;
+#elif PS_DATE == 2 && !defined(DISABLE_GL42_image)
+    // DATM == 1
+    // Pixel with alpha equal to 0 will failed
+	float rt_a = texelFetch(RtSampler, ivec2(gl_FragCoord.xy), 0).a;
+    if(rt_a < (127.5f / 255.0f)) // >= 0x80 pass
+		discard;
+#endif
+
 #if PS_DATE == 3 && !defined(DISABLE_GL42_image)
     int stencil_ceil = imageLoad(img_prim_min, ivec2(gl_FragCoord.xy));
     // Note gl_PrimitiveID == stencil_ceil will be the primitive that will update
@@ -478,7 +493,6 @@ void ps_main()
     if (c.a > 127.5f / 255.0f) {
         imageAtomicMin(img_prim_min, ivec2(gl_FragCoord.xy), gl_PrimitiveID);
     }
-    //memoryBarrier();
 #elif PS_DATE == 2 && !defined(DISABLE_GL42_image)
     // DATM == 1
     // Pixel with alpha equal to 0 will failed

plugins/GSdx/res/glsl_source.h

@@ -824,6 +824,7 @@ static const char* tfx_fs_all_glsl =
 	"#else\n"
 	"layout(binding = 0) uniform sampler2D TextureSampler;\n"
 	"layout(binding = 1) uniform sampler2D PaletteSampler;\n"
+	"layout(binding = 3) uniform sampler2D RtSampler; // note 2 already use by the image below\n"
 	"#endif\n"
 	"\n"
 	"#ifndef DISABLE_GL42_image\n"
@@ -1188,6 +1189,20 @@ static const char* tfx_fs_all_glsl =
 	"#if !pGL_ES\n"
 	"void ps_main()\n"
 	"{\n"
+	"#if PS_DATE == 1 && !defined(DISABLE_GL42_image)\n"
+	"    // DATM == 0\n"
+	"    // Pixel with alpha equal to 1 will failed\n"
+	"	float rt_a = texelFetch(RtSampler, ivec2(gl_FragCoord.xy), 0).a;\n"
+	"	if ((127.5f / 255.0f) < rt_a) // < 0x80 pass (== 0x80 should not pass)\n"
+	"		discard;\n"
+	"#elif PS_DATE == 2 && !defined(DISABLE_GL42_image)\n"
+	"    // DATM == 1\n"
+	"    // Pixel with alpha equal to 0 will failed\n"
+	"	float rt_a = texelFetch(RtSampler, ivec2(gl_FragCoord.xy), 0).a;\n"
+	"    if(rt_a < (127.5f / 255.0f)) // >= 0x80 pass\n"
+	"		discard;\n"
+	"#endif\n"
+	"\n"
 	"#if PS_DATE == 3 && !defined(DISABLE_GL42_image)\n"
 	"    int stencil_ceil = imageLoad(img_prim_min, ivec2(gl_FragCoord.xy));\n"
 	"    // Note gl_PrimitiveID == stencil_ceil will be the primitive that will update\n"
@@ -1229,7 +1244,6 @@ static const char* tfx_fs_all_glsl =
 	"    if (c.a > 127.5f / 255.0f) {\n"
 	"        imageAtomicMin(img_prim_min, ivec2(gl_FragCoord.xy), gl_PrimitiveID);\n"
 	"    }\n"
-	"    //memoryBarrier();\n"
 	"#elif PS_DATE == 2 && !defined(DISABLE_GL42_image)\n"
 	"    // DATM == 1\n"
 	"    // Pixel with alpha equal to 0 will failed\n"