gsdx: move alpha test optimization in base class

One code for all renderers :)

plugins/GSdx/GSDrawScanlineCodeGenerator.x64.avx.cpp

@@ -1121,21 +1121,6 @@ void GSDrawScanlineCodeGenerator::ReadMask()
 
 void GSDrawScanlineCodeGenerator::TestAlpha()
 {
-	switch(m_sel.afail)
-	{
-	case AFAIL_FB_ONLY:
-		if(!m_sel.zwrite) return;
-		break;
-
-	case AFAIL_ZB_ONLY:
-		if(!m_sel.fwrite) return;
-		break;
-
-	case AFAIL_RGB_ONLY:
-		if(!m_sel.zwrite && m_sel.fpsm == 1) return;
-		break;
-	}
-
 	switch(m_sel.atst)
 	{
 	case ATST_NEVER:

plugins/GSdx/GSDrawScanlineCodeGenerator.x86.avx.cpp

@@ -2115,21 +2115,6 @@ void GSDrawScanlineCodeGenerator::ReadMask()
 
 void GSDrawScanlineCodeGenerator::TestAlpha()
 {
-	switch(m_sel.afail)
-	{
-	case AFAIL_FB_ONLY:
-		if(!m_sel.zwrite) return;
-		break;
-
-	case AFAIL_ZB_ONLY:
-		if(!m_sel.fwrite) return;
-		break;
-
-	case AFAIL_RGB_ONLY:
-		if(!m_sel.zwrite && m_sel.fpsm == 1) return;
-		break;
-	}
-
 	switch(m_sel.atst)
 	{
 	case ATST_NEVER:

plugins/GSdx/GSDrawScanlineCodeGenerator.x86.avx2.cpp

@@ -2069,21 +2069,6 @@ void GSDrawScanlineCodeGenerator::ReadMask()
 
 void GSDrawScanlineCodeGenerator::TestAlpha()
 {
-	switch(m_sel.afail)
-	{
-	case AFAIL_FB_ONLY:
-		if(!m_sel.zwrite) return;
-		break;
-
-	case AFAIL_ZB_ONLY:
-		if(!m_sel.fwrite) return;
-		break;
-
-	case AFAIL_RGB_ONLY:
-		if(!m_sel.zwrite && m_sel.fpsm == 1) return;
-		break;
-	}
-
 	switch(m_sel.atst)
 	{
 	case ATST_NEVER:

plugins/GSdx/GSDrawScanlineCodeGenerator.x86.cpp

@@ -2179,21 +2179,6 @@ void GSDrawScanlineCodeGenerator::ReadMask()
 
 void GSDrawScanlineCodeGenerator::TestAlpha()
 {
-	switch(m_sel.afail)
-	{
-	case AFAIL_FB_ONLY:
-		if(!m_sel.zwrite) return;
-		break;
-
-	case AFAIL_ZB_ONLY:
-		if(!m_sel.fwrite) return;
-		break;
-
-	case AFAIL_RGB_ONLY:
-		if(!m_sel.zwrite && m_sel.fpsm == 1) return;
-		break;
-	}
-
 	switch(m_sel.atst)
 	{
 	case ATST_NEVER:

plugins/GSdx/GSRendererDX.cpp

@@ -440,7 +440,6 @@ void GSRendererDX::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sourc
 	// pass to handle the depth based on the alpha test.
 	bool ate_RGBA_then_Z = false;
 	bool ate_RGB_then_ZA = false;
-	bool ate_skip = false;
 	if (ate_first_pass & ate_second_pass) {
 #ifdef _DEBUG
 		fprintf(stdout, "Complex Alpha Test\n");
@@ -450,17 +449,9 @@ void GSRendererDX::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sourc
 
 		ate_RGBA_then_Z = (m_context->TEST.AFAIL == AFAIL_FB_ONLY) & commutative_depth;
 		ate_RGB_then_ZA = (m_context->TEST.AFAIL == AFAIL_RGB_ONLY) & commutative_depth & commutative_alpha;
-
-		// In FB_ONLY mode, only the z buffer is impacted by the alpha test. No depth => useless alpha test
-		ate_skip = (m_context->TEST.AFAIL == AFAIL_FB_ONLY) & (ds == nullptr);
 	}
 
-	if (ate_skip) {
+	if (ate_RGBA_then_Z) {
-#ifdef _DEBUG
-		fprintf(stdout, "Alternate ATE handling: ate_skip\n");
-#endif
-		ate_second_pass = false;
-	} else if (ate_RGBA_then_Z) {
 #ifdef _DEBUG
 		fprintf(stdout, "Alternate ATE handling: ate_RGBA_then_Z\n");
 #endif

plugins/GSdx/GSRendererOGL.cpp

@@ -1335,7 +1335,6 @@ void GSRendererOGL::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sour
 	// pass to handle the depth based on the alpha test.
 	bool ate_RGBA_then_Z = false;
 	bool ate_RGB_then_ZA = false;
-	bool ate_skip = false;
 	if (ate_first_pass & ate_second_pass) {
 		GL_INS("Complex Alpha Test");
 		bool commutative_depth = (m_om_dssel.ztst == ZTST_GEQUAL && m_vt.m_eq.z) || (m_om_dssel.ztst == ZTST_ALWAYS);
@@ -1343,15 +1342,9 @@ void GSRendererOGL::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sour
 
 		ate_RGBA_then_Z = (m_context->TEST.AFAIL == AFAIL_FB_ONLY) & commutative_depth;
 		ate_RGB_then_ZA = (m_context->TEST.AFAIL == AFAIL_RGB_ONLY) & commutative_depth & commutative_alpha;
-
-		// In FB_ONLY mode, only the z buffer is impacted by the alpha test. No depth => useless alpha test
-		ate_skip = (m_context->TEST.AFAIL == AFAIL_FB_ONLY) & (ds == nullptr);
 	}
 
-	if (ate_skip) {
+	if (ate_RGBA_then_Z) {
-		GL_INS("Alternate ATE handling: ate_skip");
-		ate_second_pass = false;
-	} else if (ate_RGBA_then_Z) {
 		GL_INS("Alternate ATE handling: ate_RGBA_then_Z");
 		// Render all color but don't update depth
 		// ATE is disabled here

plugins/GSdx/GSState.cpp

@@ -2953,6 +2953,21 @@ bool GSState::TryAlphaTest(uint32& fm, uint32& zm)
 	{
 		pass = false;
 	}
+	else if((context->TEST.AFAIL == AFAIL_FB_ONLY) && zm == 0xFFFFFFFF)
+	{
+		// Alpha test controls depth writes but they're masked
+		pass = true;
+	}
+	else if((context->TEST.AFAIL == AFAIL_ZB_ONLY) && fm == 0xFFFFFFFF)
+	{
+		// Alpha test controls color writes but they're masked
+		pass = true;
+	}
+	else if((context->TEST.AFAIL == AFAIL_RGB_ONLY) && zm == 0xFFFFFFFF && ((fm & 0xFF000000) == 0xFF000000 || GSLocalMemory::m_psm[context->FRAME.PSM].fmt == 1))
+	{
+		// Alpha test controls alpha/depth writes but they're both masked
+		pass = true;
+	}
 	else if(context->TEST.ATST != ATST_ALWAYS)
 	{
 		GetAlphaMinMax();