From 42e911c78f2aa1802f73bcc0e61f22df120416b7 Mon Sep 17 00:00:00 2001
From: Gregory Hainaut <gregory.hainaut@gmail.com>
Date: Sat, 13 Jun 2015 10:05:33 +0200
Subject: [PATCH] gsdx-ogl: mask alpha channel in depth conversion

Might not work if depth is recasted as a rt (we lost the alpha channel)
---
 plugins/GSdx/GSDeviceOGL.cpp       |  2 +-
 plugins/GSdx/GSDeviceOGL.h         |  2 +-
 plugins/GSdx/GSTextureCache.cpp    |  4 +++-
 plugins/GSdx/res/glsl/convert.glsl | 13 +++++++++++++
 plugins/GSdx/res/glsl_source.h     | 13 +++++++++++++
 5 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/plugins/GSdx/GSDeviceOGL.cpp b/plugins/GSdx/GSDeviceOGL.cpp
index 4bfd25f9d0..756118b1eb 100644
--- a/plugins/GSdx/GSDeviceOGL.cpp
+++ b/plugins/GSdx/GSDeviceOGL.cpp
@@ -740,7 +740,7 @@ void GSDeviceOGL::StretchRect(GSTexture* sTex, const GSVector4& sRect, GSTexture
 		return;
 	}
 
-	bool draw_in_depth = (ps == m_convert.ps[12]);
+	bool draw_in_depth = (ps == m_convert.ps[12] || ps == m_convert.ps[13]);
 
 	// Performance optimization. It might be faster to use a framebuffer blit for standard case
 	// instead to emulate it with shader
diff --git a/plugins/GSdx/GSDeviceOGL.h b/plugins/GSdx/GSDeviceOGL.h
index 00d9d9b367..14c9fb582d 100644
--- a/plugins/GSdx/GSDeviceOGL.h
+++ b/plugins/GSdx/GSDeviceOGL.h
@@ -494,7 +494,7 @@ class GSDeviceOGL : public GSDevice
 
 	struct {
 		GLuint vs;		// program object
-		GLuint ps[13];	// program object
+		GLuint ps[14];	// program object
 		GLuint ln;		// sampler object
 		GLuint pt;		// sampler object
 		GSDepthStencilOGL* dss;
diff --git a/plugins/GSdx/GSTextureCache.cpp b/plugins/GSdx/GSTextureCache.cpp
index 78b37c154f..9b3a4e5dad 100644
--- a/plugins/GSdx/GSTextureCache.cpp
+++ b/plugins/GSdx/GSTextureCache.cpp
@@ -258,7 +258,8 @@ GSTextureCache::Target* GSTextureCache::LookupTarget(const GIFRegTEX0& TEX0, int
 				dst = CreateTarget(TEX0, w, h, type);
 				if (type == DepthStencil) {
 					GL_CACHE("TC: Lookup Target(Depth) %dx%d, hit Color (0x%x, F:0x%x)", w, h, bp, TEX0.PSM);
-					m_renderer->m_dev->StretchRect(t->m_texture, sRect, dst->m_texture, dRect, 12, false);
+					int shader = (TEX0.PSM & 1) ? 13 : 12;
+					m_renderer->m_dev->StretchRect(t->m_texture, sRect, dst->m_texture, dRect, shader, false);
 				} else {
 					GL_CACHE("TC: Lookup Target(Color) %dx%d, hit Depth (0x%x, F:0x%x)", w, h, bp, TEX0.PSM);
 					m_renderer->m_dev->StretchRect(t->m_texture, sRect, dst->m_texture, dRect, 11, false);
@@ -1343,6 +1344,7 @@ void GSTextureCache::Target::Update()
 	{
 		GL_INS("ERROR: Update DepthStencil");
 
+		// FIXME linear or not?
 		m_renderer->m_dev->StretchRect(t, m_texture, GSVector4(r) * GSVector4(m_texture->GetScale()).xyxy(), 12);
 	}
 
diff --git a/plugins/GSdx/res/glsl/convert.glsl b/plugins/GSdx/res/glsl/convert.glsl
index 3a7fa21f17..b8baf5cbe4 100644
--- a/plugins/GSdx/res/glsl/convert.glsl
+++ b/plugins/GSdx/res/glsl/convert.glsl
@@ -183,6 +183,19 @@ void ps_main12()
 }
 #endif
 
+#ifdef ps_main13
+out float gl_FragDepth;
+void ps_main13()
+{
+	// Same as above but without the alpha channel
+
+	// Convert a RRGBA texture into a float depth texture
+	// FIXME: I'm afraid of the accuracy
+	const vec4 bitSh = vec4(1.0/(256.0*256.0*256.0), 1.0/(256.0*256.0), 1.0/256.0, 0.0) * vec4(255.0/256.0);
+	gl_FragDepth = dot(sample_c(), bitSh);
+}
+#endif
+
 #ifdef ps_main7
 void ps_main7()
 {
diff --git a/plugins/GSdx/res/glsl_source.h b/plugins/GSdx/res/glsl_source.h
index 5971a36c7a..3fa091dc87 100644
--- a/plugins/GSdx/res/glsl_source.h
+++ b/plugins/GSdx/res/glsl_source.h
@@ -208,6 +208,19 @@ static const char* convert_glsl =
 	"}\n"
 	"#endif\n"
 	"\n"
+	"#ifdef ps_main13\n"
+	"out float gl_FragDepth;\n"
+	"void ps_main13()\n"
+	"{\n"
+	"	// Same as above but without the alpha channel\n"
+	"\n"
+	"	// Convert a RRGBA texture into a float depth texture\n"
+	"	// FIXME: I'm afraid of the accuracy\n"
+	"	const vec4 bitSh = vec4(1.0/(256.0*256.0*256.0), 1.0/(256.0*256.0), 1.0/256.0, 0.0) * vec4(255.0/256.0);\n"
+	"	gl_FragDepth = dot(sample_c(), bitSh);\n"
+	"}\n"
+	"#endif\n"
+	"\n"
 	"#ifdef ps_main7\n"
 	"void ps_main7()\n"
 	"{\n"