gsdx ogl: incorporate DX sudonim's changes on ogl

git-svn-id: http://pcsx2.googlecode.com/svn/trunk@5375 96395faa-99c1-11dd-bbfe-3dabce05a288

plugins/GSdx/GSDeviceOGL.h

@@ -380,12 +380,13 @@ class GSDeviceOGL : public GSDevice
 				uint32 colclip:2;
 				uint32 date:2;
 				uint32 spritehack:1;
+				uint32 point_sampler:1;
 			};
 
 			uint32 key;
 		};
 
-		operator uint32() {return key & 0x3ffffff;}
+		operator uint32() {return key & 0xfffffff;}
 
 		PSSelector() : key(0) {}
 	};
@@ -419,12 +420,13 @@ class GSDeviceOGL : public GSDevice
 				uint32 zwe:1;
 				uint32 date:1;
 				uint32 fba:1;
+				uint32 alpha_stencil:1;
 			};
 
 			uint32 key;
 		};
 
-		operator uint32() {return key & 0x1f;}
+		operator uint32() {return key & 0x3f;}
 
 		OMDepthStencilSelector() : key(0) {}
 	};

plugins/GSdx/GSRendererOGL.cpp

@@ -30,6 +30,7 @@ GSRendererOGL::GSRendererOGL()
 	m_fba = !!theApp.GetConfig("fba", 1);
 	UserHacks_AlphaHack = !!theApp.GetConfig("UserHacks_AlphaHack", 0) && !!theApp.GetConfig("UserHacks", 0);
 	UserHacks_WildHack = !!theApp.GetConfig("UserHacks", 0) ? theApp.GetConfig("UserHacks_WildHack", 0) : 0;
+	UserHacks_AlphaStencil = !!theApp.GetConfig("UserHacks_AlphaStencil", 0) && !!theApp.GetConfig("UserHacks", 0);
 	m_pixelcenter = GSVector2(-0.5f, -0.5f);
 }
 
@@ -318,43 +319,51 @@ void GSRendererOGL::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sour
 	}
 
 	if(context->TEST.ATE)
-	{
 		ps_sel.atst = context->TEST.ATST;
-
-		switch(ps_sel.atst)
-		{
-		case ATST_LESS:
-			ps_cb.FogColor_AREF.a = (float)((int)context->TEST.AREF - 1);
-			break;
-		case ATST_GREATER:
-			ps_cb.FogColor_AREF.a = (float)((int)context->TEST.AREF + 1);
-			break;
-		default:
-			ps_cb.FogColor_AREF.a = (float)(int)context->TEST.AREF;
-			break;
-		}
-	}
 	else
-	{
 		ps_sel.atst = ATST_ALWAYS;
+
+	if (context->TEST.ATE && context->TEST.ATST > 1)
+		ps_cb.FogColor_AREF.a = (float)context->TEST.AREF;
+
+	// Destination alpha pseudo stencil hack: use a stencil operation combined with an alpha test
+	// to only draw pixels which would cause the destination alpha test to fail in the future once.
+	// Unfortunately this also means only drawing those pixels at all, which is why this is a hack.
+	// The interaction with FBA in D3D9 is probably less than ideal.
+	if (UserHacks_AlphaStencil && DATE && dev->HasStencil() && om_bsel.wa && (!context->TEST.ATE || context->TEST.ATST == 1))
+	{
+		if (!context->FBA.FBA)
+		{
+			if (context->TEST.DATM == 0)
+				ps_sel.atst = 5; // >=
+			else
+				ps_sel.atst = 2; // <
+			ps_cb.FogColor_AREF.a = (float)0x80;
+		}
+		if (!(context->FBA.FBA && context->TEST.DATM == 1))
+			om_dssel.alpha_stencil = 1;
 	}
 
 	if(tex)
 	{
 		const GSLocalMemory::psm_t &psm = GSLocalMemory::m_psm[context->TEX0.PSM];
+		const GSLocalMemory::psm_t &cpsm = psm.pal > 0 ? GSLocalMemory::m_psm[context->TEX0.CPSM] : psm;
+		bool bilinear = m_filter == 2 ? m_vt.IsLinear() : m_filter != 0;
+		bool simple_sample = !tex->m_palette && cpsm.fmt == 0 && context->CLAMP.WMS < 3 && context->CLAMP.WMT < 3;
 
 		ps_sel.wms = context->CLAMP.WMS;
 		ps_sel.wmt = context->CLAMP.WMT;
+		if (tex->m_palette)
+			ps_sel.fmt = cpsm.fmt | 4;
+		else
+			ps_sel.fmt = cpsm.fmt;
 		ps_sel.aem = env.TEXA.AEM;
 		ps_sel.tfx = context->TEX0.TFX;
 		ps_sel.tcc = context->TEX0.TCC;
-		ps_sel.ltf = m_filter == 2 ? m_vt.IsLinear() : m_filter;
+		ps_sel.ltf = bilinear && !simple_sample;
 		ps_sel.rt = tex->m_target;
-		if (tex->m_palette) {
+		ps_sel.spritehack = tex->m_spritehack_t;
-			const GSLocalMemory::psm_t &cpsm = GSLocalMemory::m_psm[context->TEX0.CPSM];
+		ps_sel.point_sampler = !(bilinear && simple_sample);
-			ps_sel.fmt = cpsm.fmt | 4;
-		} else
-			ps_sel.fmt = psm.fmt;
 
 		int w = tex->m_texture->GetWidth();
 		int h = tex->m_texture->GetHeight();
@@ -384,7 +393,7 @@ void GSRendererOGL::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sour
 
 		ps_ssel.tau = (context->CLAMP.WMS + 3) >> 1;
 		ps_ssel.tav = (context->CLAMP.WMT + 3) >> 1;
-		ps_ssel.ltf = ps_sel.ltf;
+		ps_ssel.ltf = bilinear && simple_sample;
 	}
 	else
 	{
@@ -427,6 +436,7 @@ void GSRendererOGL::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sour
 			dev->SetupPS(ps_selneg, &ps_cb, ps_ssel);
 
 			dev->DrawIndexedPrimitive();
+			dev->SetupOM(om_dssel, om_bsel, afix);
 		}
 	}
 
@@ -438,19 +448,6 @@ void GSRendererOGL::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sour
 
 		ps_sel.atst = iatst[ps_sel.atst];
 
-		switch(ps_sel.atst)
-		{
-		case ATST_LESS:
-			ps_cb.FogColor_AREF.a = (float)((int)context->TEST.AREF - 1);
-			break;
-		case ATST_GREATER:
-			ps_cb.FogColor_AREF.a = (float)((int)context->TEST.AREF + 1);
-			break;
-		default:
-			ps_cb.FogColor_AREF.a = (float)(int)context->TEST.AREF;
-			break;
-		}
-
 		dev->SetupPS(ps_sel, &ps_cb, ps_ssel);
 
 		bool z = om_dssel.zwe;

plugins/GSdx/GSRendererOGL.h

@@ -38,6 +38,7 @@ class GSRendererOGL : public GSRendererHW
 		bool m_logz;
 		bool m_fba;
 		bool UserHacks_AlphaHack;
+		bool UserHacks_AlphaStencil;
 		int UserHacks_WildHack;
 
 	protected:

plugins/GSdx/GSTextureFXOGL.cpp

@@ -254,7 +254,7 @@ void GSDeviceOGL::SetupOM(OMDepthStencilSelector dssel, OMBlendSelector bsel, ui
 		if (dssel.date)
 		{
 			dss->EnableStencil();
-			dss->SetStencil(GL_EQUAL, GL_KEEP);
+			dss->SetStencil(GL_EQUAL, dssel.alpha_stencil ? GL_ZERO : GL_KEEP);
 		}
 
 		if(dssel.ztst != ZTST_ALWAYS || dssel.zwe)

plugins/GSdx/res/tfx.glsl

@@ -378,12 +378,15 @@ vec4 sample_4a(vec4 uv)
 {
     vec4 c;
 
+    // XXX
+    // I think .a is related to 8bit texture in alpha channel
+    // Opengl is only 8 bits on red channel. Not sure exactly of the impact
     c.x = sample_c(uv.xy).a;
     c.y = sample_c(uv.zy).a;
     c.z = sample_c(uv.xw).a;
     c.w = sample_c(uv.zw).a;
 
-    return c;
+	return c * 255./256 + 0.5/256;
 }
 
 mat4 sample_4p(vec4 u)
@@ -406,19 +409,21 @@ vec4 sample_color(vec2 st, float q)
     }
 
     vec4 t;
-    if((PS_FMT <= FMT_16) && (PS_WMS < 3) && (PS_WMT < 3))
+    mat4 c;
+    vec2 dd;
+
+    if (PS_LTF == 0 && PS_FMT <= FMT_16 && PS_WMS < 3 && PS_WMT < 3)
     {
-        t = sample_c(clampuv(st));
+        c[0] = sample_c(clampuv(st));
     }
     else
     {
         vec4 uv;
-        vec2 dd;
 
         if(PS_LTF != 0)
         {
             uv = st.xyxy + HalfTexel;
-            dd = fract(uv.xy * WH.zw); 
+            dd = fract(uv.xy * WH.zw);
         }
         else
         {
@@ -427,45 +432,40 @@ vec4 sample_color(vec2 st, float q)
 
         uv = wrapuv(uv);
 
-        mat4 c;
-
         if((PS_FMT & FMT_PAL) != 0)
+        {
             c = sample_4p(sample_4a(uv));
+        }
         else
+        {
             c = sample_4c(uv);
-
-        // PERF: see the impact of the exansion before/after the interpolation
-        for (int i = 0; i < 4; i++) {
-            if((PS_FMT & ~FMT_PAL) == FMT_16)
-            {
-                // FIXME GLSL any only support bvec so try to mix it with notEqual
-                bvec3 rgb_check = notEqual( t.rgb, vec3(0.0f, 0.0f, 0.0f) );
-                t.a = t.a >= 0.5 ? TA.y : ( (PS_AEM == 0) || any(rgb_check) ) ? TA.x : 0.0f; 
-            }
-        }
-
-        if(PS_LTF != 0)
-        {
-            t = mix(mix(c[0], c[1], dd.x), mix(c[2], c[3], dd.x), dd.y);
-        }
-        else
-        {
-            t = c[0];
         }
     }
 
-    if(PS_FMT == FMT_24)
+    // PERF: see the impact of the exansion before/after the interpolation
+    for (uint i = 0; i < 4; i++)
     {
-        // FIXME GLSL any only support bvec so try to mix it with notEqual
+        if((PS_FMT & ~FMT_PAL) == FMT_24)
-        bvec3 rgb_check = notEqual( t.rgb, vec3(0.0f, 0.0f, 0.0f) );
+        {
-        t.a = ( (PS_AEM == 0) || any(rgb_check)  ) ? TA.x : 0.0f;
+            // FIXME GLSL any only support bvec so try to mix it with notEqual
+            bvec3 rgb_check = notEqual( t.rgb, vec3(0.0f, 0.0f, 0.0f) );
+            t.a = ( (PS_AEM == 0) || any(rgb_check)  ) ? TA.x : 0.0f;
+        }
+        else if((PS_FMT & ~FMT_PAL) == FMT_16)
+        {
+            // FIXME GLSL any only support bvec so try to mix it with notEqual
+            bvec3 rgb_check = notEqual( t.rgb, vec3(0.0f, 0.0f, 0.0f) );
+            t.a = t.a >= 0.5 ? TA.y : ( (PS_AEM == 0) || any(rgb_check) ) ? TA.x : 0.0f;
+        }
     }
-    else if(PS_FMT == FMT_16)
+
+    if(PS_LTF != 0)
     {
-        // a bit incompatible with up-scaling because the 1 bit alpha is interpolated
+        t = mix(mix(c[0], c[1], dd.x), mix(c[2], c[3], dd.x), dd.y);
-        // FIXME GLSL any only support bvec so try to mix it with notEqual
+    }
-        bvec3 rgb_check = notEqual( t.rgb, vec3(0.0f, 0.0f, 0.0f) );
+    else
-        t.a = t.a >= 0.5 ? TA.y : ( (PS_AEM == 0) || any(rgb_check) ) ? TA.x : 0.0f; 
+    {
+        t = c[0];
     }
 
     return t;
@@ -533,7 +533,7 @@ void datst()
 
 void atst(vec4 c)
 {
-    float a = trunc(c.a * 255);
+    float a = trunc(c.a * 255 + 0.01);
 
     if(PS_ATST == 0) // never
     {
@@ -546,12 +546,12 @@ void atst(vec4 c)
     else if(PS_ATST == 2 ) // l
     {
         if (PS_SPRITEHACK == 0)
-            if ((AREF - a) < 0.0f)
+            if ((AREF - a - 0.5f) < 0.0f)
                 discard;
     }
     else if(PS_ATST == 3 ) // le
     {
-        if ((AREF - a) < 0.0f)
+        if ((AREF - a + 0.5f) < 0.0f)
             discard;
     }
     else if(PS_ATST == 4) // e
@@ -559,9 +559,14 @@ void atst(vec4 c)
         if ((0.5f - abs(a - AREF)) < 0.0f)
             discard;
     }
-    else if(PS_ATST == 5 || PS_ATST == 6) // ge, g
+    else if(PS_ATST == 5) // ge
     {
-        if ((a-AREF) < 0.0f)
+        if ((a-AREF + 0.5f) < 0.0f)
+            discard;
+    }
+    else if(PS_ATST == 6) // g
+    {
+        if ((a-AREF - 0.5f) < 0.0f)
             discard;
     }
     else if(PS_ATST == 7) // ne