gsdx ogl: handle float integral coordinate for depth sampling

Fix depth effect regression on Nemo (issue #1979)

plugins/GSdx/GSDeviceOGL.cpp

@@ -593,7 +593,8 @@ void GSDeviceOGL::CreateTextureFX()
 	m_gs[2] = CompileGS(GSSelector(2));
 	m_gs[4] = CompileGS(GSSelector(4));
 
-	m_vs[0] = CompileVS(VSSelector(0));
+	for (uint32 key = 0; key < countof(m_vs); key++)
+		m_vs[key] = CompileVS(VSSelector(key));
 
 	// Enable all bits for stencil operations. Technically 1 bit is
 	// enough but buffer is polluted with noise. Clear will be limited
@@ -917,10 +918,12 @@ void GSDeviceOGL::Barrier(GLbitfield b)
 
 GLuint GSDeviceOGL::CompileVS(VSSelector sel)
 {
+	std::string macro = format("#define VS_INT_FST %d\n", sel.int_fst);
+
 	if (GLLoader::buggy_sso_dual_src)
-		return m_shader->CompileShader("tfx_vgs.glsl", "vs_main", GL_VERTEX_SHADER, m_shader_tfx_vgs.data(), "");
+		return m_shader->CompileShader("tfx_vgs.glsl", "vs_main", GL_VERTEX_SHADER, m_shader_tfx_vgs.data(), macro);
 	else
-		return m_shader->Compile("tfx_vgs.glsl", "vs_main", GL_VERTEX_SHADER, m_shader_tfx_vgs.data(), "");
+		return m_shader->Compile("tfx_vgs.glsl", "vs_main", GL_VERTEX_SHADER, m_shader_tfx_vgs.data(), macro);
 }
 
 GLuint GSDeviceOGL::CompileGS(GSSelector sel)

plugins/GSdx/GSDeviceOGL.h

@@ -158,7 +158,8 @@ public:
 		{
 			struct
 			{
-				uint32 _free:32;
+				uint32 int_fst:1;
+				uint32 _free:31;
 			};
 
 			uint32 key;
@@ -473,7 +474,7 @@ public:
 		GLuint timer() { return timer_query[last_query]; }
 	} m_profiler;
 
-	GLuint m_vs[1];
+	GLuint m_vs[1<<1];
 	GLuint m_gs[1<<3];
 	GLuint m_ps_ss[1<<7];
 	GSDepthStencilOGL* m_om_dss[1<<5];

plugins/GSdx/GSRendererOGL.cpp

@@ -733,6 +733,7 @@ void GSRendererOGL::EmulateTextureSampler(const GSTextureCache::Source* tex)
 		// Require a float conversion if the texure is a depth otherwise uses Integral scaling
 		if (psm.depth) {
 			m_ps_sel.depth_fmt = (tex->m_texture->GetType() != GSTexture::DepthStencil) ? 3 : 1;
+			m_vs_sel.int_fst = !PRIM->FST; // select float/int coordinate
 		}
 
 		// Shuffle is a 16 bits format, so aem is always required
@@ -784,12 +785,14 @@ void GSRendererOGL::EmulateTextureSampler(const GSTextureCache::Source* tex)
 		if (tex->m_texture->GetType() == GSTexture::DepthStencil) {
 			// Require a float conversion if the texure is a depth format
 			m_ps_sel.depth_fmt = (psm.bpp == 16) ? 2 : 1;
+			m_vs_sel.int_fst = !PRIM->FST; // select float/int coordinate
 
 			// Don't force interpolation on depth format
 			bilinear &= m_vt.IsLinear();
 		} else if (psm.depth) {
 			// Use Integral scaling
 			m_ps_sel.depth_fmt = 3;
+			m_vs_sel.int_fst = !PRIM->FST; // select float/int coordinate
 
 			// Don't force interpolation on depth format
 			bilinear &= m_vt.IsLinear();

plugins/GSdx/res/glsl/tfx_fs.glsl

@@ -557,9 +557,11 @@ vec4 ps_color()
     //FIXME: maybe we can set gl_Position.w = q in VS
 #if (PS_FST == 0)
     vec2 st = PSin.t_float.xy / vec2(PSin.t_float.w);
+    vec2 st_int = PSin.t_int.zw / vec2(PSin.t_float.w);
 #else
     // Note xy are normalized coordinate
     vec2 st = PSin.t_int.xy;
+    vec2 st_int = PSin.t_int.zw;
 #endif
 
 #if PS_CHANNEL_FETCH == 1
@@ -576,7 +578,7 @@ vec4 ps_color()
     vec4 T = fetch_rgb();
 #elif PS_DEPTH_FMT > 0
     // Integral coordinate
-    vec4 T = sample_depth(PSin.t_int.zw);
+    vec4 T = sample_depth(st_int);
 #else
     vec4 T = sample_color(st);
 #endif

plugins/GSdx/res/glsl/tfx_vgs.glsl

@@ -25,15 +25,21 @@ const float exp_min32 = exp2(-32.0f);
 void texture_coord()
 {
     vec2 uv = vec2(i_uv) - TextureOffset.xy;
+    vec2 st = i_st - TextureOffset.xy;
 
     // Float coordinate
-    VSout.t_float.xy = i_st - TextureOffset.xy;
+    VSout.t_float.xy = st;
     VSout.t_float.w  = i_q;
 
     // Integer coordinate => normalized
     VSout.t_int.xy = uv * TextureScale;
+#if VS_INT_FST == 1
+    // Some games uses float coordinate for post-processing effect
+    VSout.t_int.zw = st / TextureScale;
+#else
     // Integer coordinate => integral
     VSout.t_int.zw = uv;
+#endif
 }
 
 void vs_main()