gsdx-ogl: wipeout AlphaStencil & Alpha hack

Accurate options do a better jobs. Technically it can still
be useful for old gpu/driver that doesn't support the GL4.5 extension.

On Windows, you can still rely on Dx

On linux, free driver support it (except Intel)

plugins/GSdx/GSDeviceOGL.cpp

@@ -653,7 +653,6 @@ GLuint GSDeviceOGL::CompilePS(PSSelector sel)
 		+ format("#define PS_FOG %d\n", sel.fog)
 		+ format("#define PS_CLR1 %d\n", sel.clr1)
 		+ format("#define PS_FBA %d\n", sel.fba)
-		+ format("#define PS_AOUT %d\n", sel.aout)
 		+ format("#define PS_LTF %d\n", sel.ltf)
 		+ format("#define PS_COLCLIP %d\n", sel.colclip)
 		+ format("#define PS_DATE %d\n", sel.date)

plugins/GSdx/GSDeviceOGL.h

@@ -332,14 +332,12 @@ class GSDeviceOGL : public GSDevice
 				uint32 fog:1;
 				uint32 clr1:1;
 				uint32 fba:1;
-				uint32 aout:1;
 				uint32 date:3;
 				uint32 tcoffsethack:1;
 				//uint32 point_sampler:1; Not tested, so keep the bit for blend
 				uint32 iip:1;
 				uint32 colclip:2;
 				uint32 atst:3;
-
 				uint32 tfx:3;
 				uint32 tcc:1;
 				uint32 wms:2;
@@ -348,6 +346,7 @@ class GSDeviceOGL : public GSDevice
 				uint32 ifmt:2;
 				uint32 shuffle:1;
 				uint32 read_ba:1;
+				uint32 fbmask:1;
 
 				//uint32 _free1:0;
 
@@ -357,7 +356,6 @@ class GSDeviceOGL : public GSDevice
 				uint32 blend_c:2;
 				uint32 blend_d:2;
 				uint32 dfmt:2;
-				uint32 fbmask:1;
 
 				uint32 _free2:21;
 			};

plugins/GSdx/GSRendererOGL.cpp

@@ -34,15 +34,11 @@ GSRendererOGL::GSRendererOGL()
 	m_accurate_colclip = theApp.GetConfig("accurate_colclip", 0);
 	m_accurate_fbmask = theApp.GetConfig("accurate_fbmask", 0);
 
-	UserHacks_AlphaHack      = theApp.GetConfig("UserHacks_AlphaHack", 0);
-	UserHacks_AlphaStencil   = theApp.GetConfig("UserHacks_AlphaStencil", 0);
 	UserHacks_TCOffset       = theApp.GetConfig("UserHacks_TCOffset", 0);
 	UserHacks_TCO_x          = (UserHacks_TCOffset & 0xFFFF) / -1000.0f;
 	UserHacks_TCO_y          = ((UserHacks_TCOffset >> 16) & 0xFFFF) / -1000.0f;
 
 	if (!theApp.GetConfig("UserHacks", 0)) {
-		UserHacks_AlphaHack      = false;
-		UserHacks_AlphaStencil   = false;
 		UserHacks_TCOffset       = 0;
 		UserHacks_TCO_x          = 0;
 		UserHacks_TCO_y          = 0;
@@ -457,8 +453,8 @@ void GSRendererOGL::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sour
 		if (GLLoader::found_GL_ARB_texture_barrier && !PrimitiveOverlap()) {
 			DATE_GL45 = true;
 			DATE = false;
-		} else if (m_accurate_date && !UserHacks_AlphaStencil &&
-				om_csel.wa && (!context->TEST.ATE || context->TEST.ATST == ATST_ALWAYS)) {
+		} else if (m_accurate_date && om_csel.wa
+				&& (!context->TEST.ATE || context->TEST.ATST == ATST_ALWAYS)) {
 			// texture barrier will split the draw call into n draw call. It is very efficient for
 			// few primitive draws. Otherwise it sucks.
 			if (GLLoader::found_GL_ARB_texture_barrier && (m_index.tail < 100)) {
@@ -621,13 +617,6 @@ void GSRendererOGL::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sour
 
 	ps_sel.fba = context->FBA.FBA;
 
-	// TODO deprecat this stuff
-	if (ps_sel.shuffle || ps_sel.fbmask) {
-		ps_sel.aout = 0;
-	} else {
-		ps_sel.aout = UserHacks_AlphaHack || ((context->FRAME.FBMSK & 0xff000000) == 0x7f000000) ? 1 : 0;
-	}
-
 	if (PRIM->FGE)
 	{
 		ps_sel.fog = 1;
@@ -643,25 +632,6 @@ void GSRendererOGL::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sour
 	if (context->TEST.ATE && context->TEST.ATST > 1)
 		ps_cb.FogColor_AREF.a = (float)context->TEST.AREF;
 
-	// TODO deprecat this stuff
-	// Destination alpha pseudo stencil hack: use a stencil operation combined with an alpha test
-	// to only draw pixels which would cause the destination alpha test to fail in the future once.
-	// Unfortunately this also means only drawing those pixels at all, which is why this is a hack.
-	// The interaction with FBA in D3D9 is probably less than ideal.
-	if (UserHacks_AlphaStencil && DATE && dev->HasStencil() && om_csel.wa && (!context->TEST.ATE || context->TEST.ATST == ATST_ALWAYS))
-	{
-		if (!context->FBA.FBA)
-		{
-			if (context->TEST.DATM == 0)
-				ps_sel.atst = ATST_GEQUAL; // >=
-			else
-				ps_sel.atst = ATST_LESS; // <
-			ps_cb.FogColor_AREF.a = (float)0x80;
-		}
-		if (!(context->FBA.FBA && context->TEST.DATM == 1))
-			om_dssel.alpha_stencil = 1;
-	}
-
 	// By default don't use texture
 	ps_sel.tfx = 4;
 	bool spritehack = false;

plugins/GSdx/GSRendererOGL.h

@@ -36,8 +36,6 @@ class GSRendererOGL : public GSRendererHW
 		bool m_accurate_colclip;
 		bool m_accurate_fbmask;
 
-		bool UserHacks_AlphaHack;
-		bool UserHacks_AlphaStencil;
 		unsigned int UserHacks_TCOffset;
 		float UserHacks_TCO_x, UserHacks_TCO_y;
 

plugins/GSdx/GSSetting.cpp

@@ -54,7 +54,8 @@ const char* dialog_message(int ID, bool* updateText) {
 				"Use it, for example, to try and get rid of bad post processing effects."
 				" Try values between 1 and 100.";
 		case IDC_ALPHAHACK:
-			return "Alpha Hack\n\nDifferent alpha handling. Can work around some shadow problems.";
+			return "Alpha Hack\n\nDifferent alpha handling. Can work around some shadow problems.\n\n"
+				"DX only, OpenGL emulates those effects correctly with accurate options.";
 		case IDC_OFFSETHACK:
 			return "Halfpixel\n\nMight fix some misaligned fog, bloom, or blend effect.";
 		case IDC_SPRITEHACK:
@@ -79,7 +80,8 @@ const char* dialog_message(int ID, bool* updateText) {
 		case IDC_ALPHASTENCIL:
 			return "Extend stencil based emulation of destination alpha to perform stencil operations while drawing.\n\n"
 				"Improves many shadows which are normally overdrawn in parts, may affect other effects.\n"
-				"Will disable partial transparency in some games or even prevent drawing some elements altogether.";
+				"Will disable partial transparency in some games or even prevent drawing some elements altogether."
+				"DX only, OpenGL emulates those effects correctly with accurate options.";
 		case IDC_CHECK_DISABLE_ALL_HACKS:
 			return "FOR TESTING ONLY!!\n\n"
 				"Disable all CRC hacks - will break many games. Overrides CrcHacksExclusion at gsdx.ini\n"

plugins/GSdx/res/glsl/tfx_fs.glsl

@@ -578,8 +578,7 @@ void ps_main()
 	float alpha_blend = C.a / 128.0f;
 
 	// Correct the ALPHA value based on the output format
-	// FIXME add support of alpha mask to replace properly PS_AOUT
-#if (PS_DFMT == FMT_16) || (PS_AOUT)
+#if (PS_DFMT == FMT_16)
 	float A_one = 128.0f; // alpha output will be 0x80
 	C.a = (PS_FBA != 0) ? A_one : step(128.0f, C.a) * A_one;
 #elif (PS_DFMT == FMT_32) && (PS_FBA != 0)

plugins/GSdx/res/glsl_source.h

@@ -1441,8 +1441,7 @@ static const char* tfx_fs_all_glsl =
 	"	float alpha_blend = C.a / 128.0f;\n"
 	"\n"
 	"	// Correct the ALPHA value based on the output format\n"
-	"	// FIXME add support of alpha mask to replace properly PS_AOUT\n"
-	"#if (PS_DFMT == FMT_16) || (PS_AOUT)\n"
+	"#if (PS_DFMT == FMT_16)\n"
 	"	float A_one = 128.0f; // alpha output will be 0x80\n"
 	"	C.a = (PS_FBA != 0) ? A_one : step(128.0f, C.a) * A_one;\n"
 	"#elif (PS_DFMT == FMT_32) && (PS_FBA != 0)\n"