From c8c97312627aee6d88b913d6afff81ee2c0bb9a5 Mon Sep 17 00:00:00 2001
From: lightningterror <lightningterror@yahoo.com>
Date: Mon, 23 Apr 2018 20:09:49 +0200
Subject: [PATCH] GSdx: Update D3D fbmask emulation on texture shuffle.

Improves accuracy on Sly series.

Rendering is still bad and needs a lot more work but
it's a little bit better.
---
 plugins/GSdx/GSRendererDX11.cpp | 19 +++++++++++--------
 plugins/GSdx/GSRendererOGL.cpp  |  4 ++--
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/plugins/GSdx/GSRendererDX11.cpp b/plugins/GSdx/GSRendererDX11.cpp
index 574fff2d26..39b1cb91f7 100644
--- a/plugins/GSdx/GSRendererDX11.cpp
+++ b/plugins/GSdx/GSRendererDX11.cpp
@@ -119,17 +119,20 @@ void GSRendererDX11::EmulateTextureShuffleAndFbmask()
 
 		// Please bang my head against the wall!
 		// 1/ Reduce the frame mask to a 16 bit format
-		// FIXME GSVector will be nice here
 		const uint32& m = m_context->FRAME.FBMSK;
 		uint32 fbmask = ((m >> 3) & 0x1F) | ((m >> 6) & 0x3E0) | ((m >> 9) & 0x7C00) | ((m >> 16) & 0x8000);
+		// FIXME GSVector will be nice here
+		uint8 rg_mask = fbmask & 0xFF;
+		uint8 ba_mask = (fbmask >> 8) & 0xFF;
 		om_bsel.wrgba = 0;
 
 		// 2 Select the new mask (Please someone put SSE here)
-		if ((fbmask & 0xFF) == 0) {
-			if (write_ba)
+		if (rg_mask != 0xFF) {
+			if (write_ba) {
 				om_bsel.wb = 1;
-			else
+			} else {
 				om_bsel.wr = 1;
+			}
 		} else if ((fbmask & 0xFF) != 0xFF) {
 #ifdef _DEBUG
 			fprintf(stderr, "Please fix me! wb %u wr %u\n", om_bsel.wb, om_bsel.wr);
@@ -137,12 +140,12 @@ void GSRendererDX11::EmulateTextureShuffleAndFbmask()
 			//ASSERT(0);
 		}
 
-		fbmask >>= 8;
-		if ((fbmask & 0xFF) == 0) {
-			if (write_ba)
+		if (ba_mask != 0xFF) {
+			if (write_ba) {
 				om_bsel.wa = 1;
-			else
+			} else {
 				om_bsel.wg = 1;
+			}
 		} else if ((fbmask & 0xFF) != 0xFF) {
 #ifdef _DEBUG
 			fprintf(stderr, "Please fix me! wa %u wg %u\n", om_bsel.wa, om_bsel.wg);
diff --git a/plugins/GSdx/GSRendererOGL.cpp b/plugins/GSdx/GSRendererOGL.cpp
index f7ed7c6227..92eadb4c7a 100644
--- a/plugins/GSdx/GSRendererOGL.cpp
+++ b/plugins/GSdx/GSRendererOGL.cpp
@@ -444,7 +444,7 @@ void GSRendererOGL::EmulateChannelShuffle(GSTexture** rt, const GSTextureCache::
 			//
 			// Note: Tales Of Abyss and Tekken5 could hit this path too. Those games are
 			// handled above.
-			GL_INS("maybe not a channel!");
+			GL_INS("Maybe not a channel!");
 			m_channel_shuffle = false;
 		} else if (m_context->CLAMP.WMS == 3 && ((m_context->CLAMP.MAXU & 0x8) == 8)) {
 			// Read either blue or Alpha. Let's go for Blue ;)
@@ -495,7 +495,7 @@ void GSRendererOGL::EmulateChannelShuffle(GSTexture** rt, const GSTextureCache::
 				m_ps_sel.channel = 1;
 			}
 		} else {
-			GL_INS("channel not supported");
+			GL_INS("Channel not supported");
 			m_channel_shuffle = false;
 		}
 	}