gsdx-ogl-wnd:

* Don't write color during stencil. Keep the old method to ease debug
* Realign date shader on DX
* keep stencil ref to 1. Reduce stencil management burder
* Fix texture pitch (was in pixels but need bytes). Fix Bleach Blade Battlers 2. Thanks Miseru for your trace

Remember note:
fxaa -> not yet implemented
msaa -> not implemented (I might drop it actually)
8 bit texture -> not yet implemented



git-svn-id: http://pcsx2.googlecode.com/svn/branches/gsdx-ogl-wnd@5657 96395faa-99c1-11dd-bbfe-3dabce05a288

plugins/GSdx/GSDeviceOGL.cpp

@@ -34,6 +34,11 @@
 //#define LOUD_DEBUGGING
 //#define PRINT_FRAME_NUMBER
 //#define ONLY_LINES
+#if 0
+#ifdef _DEBUG
+#define ENABLE_OGL_STENCIL_DEBUG
+#endif
+#endif
 
 static uint32 g_draw_count = 0;
 static uint32 g_frame_count = 1;		
@@ -309,33 +314,18 @@ bool GSDeviceOGL::Create(GSWnd* wnd)
 	//CompileShaderFromSource("fxaa.fx", format("ps_main", i), GL_FRAGMENT_SHADER, &m_fxaa.ps, fxaa_glsl);
 
 	// ****************************************************************
-	// date
+	// DATE
 	// ****************************************************************
 
 	m_date.dss = new GSDepthStencilOGL();
 	m_date.dss->EnableStencil();
 	m_date.dss->SetStencil(GL_ALWAYS, GL_REPLACE);
-	//memset(&dsd, 0, sizeof(dsd));
 
-	//dsd.DepthEnable = false;
-	//dsd.StencilEnable = true;
-	//dsd.StencilReadMask = 1;
-	//dsd.StencilWriteMask = 1;
-
-	//dsd.FrontFace.StencilFunc = D3D11_COMPARISON_ALWAYS;
-	//dsd.FrontFace.StencilPassOp = D3D11_STENCIL_OP_REPLACE;
-	//dsd.FrontFace.StencilFailOp = D3D11_STENCIL_OP_KEEP;
-	//dsd.FrontFace.StencilDepthFailOp = D3D11_STENCIL_OP_KEEP;
-
-	//m_dev->CreateDepthStencilState(&dsd, &m_date.dss);
-
-	// FIXME are the blend state really empty
 	m_date.bs = new GSBlendStateOGL();
-	//D3D11_BLEND_DESC blend;
+#ifndef ENABLE_OGL_STENCIL_DEBUG
-
+	// Only keep stencil data
-	//memset(&blend, 0, sizeof(blend));
+	m_date.bs->SetMask(false, false, false, false);
-
+#endif
-	//m_dev->CreateBlendState(&blend, &m_date.bs);
 
 	// ****************************************************************
 	// HW renderer shader
@@ -944,56 +934,59 @@ void GSDeviceOGL::DoShadeBoost(GSTexture* st, GSTexture* dt)
 
 void GSDeviceOGL::SetupDATE(GSTexture* rt, GSTexture* ds, const GSVertexPT1* vertices, bool datm)
 {
+#ifdef ENABLE_OGL_STENCIL_DEBUG
 	const GSVector2i& size = rt->GetSize();
+	GSTexture* t = CreateRenderTarget(size.x, size.y, rt->IsMSAA());
+#else
+	GSTexture* t = NULL;
+#endif
+	// sfex3 (after the capcom logo), vf4 (first menu fading in), ffxii shadows, rumble roses shadows, persona4 shadows
 
-	if(GSTexture* t = CreateRenderTarget(size.x, size.y, rt->IsMSAA()))
+	BeginScene();
-	{
-		// sfex3 (after the capcom logo), vf4 (first menu fading in), ffxii shadows, rumble roses shadows, persona4 shadows
 
-		BeginScene();
+	ClearStencil(ds, 0);
 
-		ClearStencil(ds, 0);
+	// om
 
-		// om
+	OMSetDepthStencilState(m_date.dss, 1);
+	OMSetBlendState(m_date.bs, 0);
+	OMSetRenderTargets(t, ds);
 
-		OMSetDepthStencilState(m_date.dss, 1);
+	// ia
-		OMSetBlendState(m_date.bs, 0);
-		OMSetRenderTargets(t, ds);
 
-		// ia
+	IASetVertexState(m_vb_sr);
+	IASetVertexBuffer(vertices, 4);
+	IASetPrimitiveTopology(GL_TRIANGLE_STRIP);
 
-		IASetVertexState(m_vb_sr);
+	// vs
-		IASetVertexBuffer(vertices, 4);
-		IASetPrimitiveTopology(GL_TRIANGLE_STRIP);
 
-		// vs
+	VSSetShader(m_convert.vs);
 
-		VSSetShader(m_convert.vs);
+	// gs
 
-		// gs
+	GSSetShader(0);
 
-		GSSetShader(0);
+	// ps
 
-		// ps
+	GSTexture* rt2 = rt->IsMSAA() ? Resolve(rt) : rt;
 
-		GSTexture* rt2 = rt->IsMSAA() ? Resolve(rt) : rt;
+	PSSetShaderResources(rt2, NULL);
+	PSSetSamplerState(m_convert.pt, 0);
+	PSSetShader(m_convert.ps[datm ? 2 : 3]);
 
-		PSSetShaderResources(rt2, NULL);
+	//
-		PSSetSamplerState(m_convert.pt, 0);
-		PSSetShader(m_convert.ps[datm ? 2 : 3]);
 
-		//
+	DrawPrimitive();
 
-		DrawPrimitive();
+	//
 
-		//
+	EndScene();
 
-		EndScene();
+#ifdef ENABLE_OGL_STENCIL_DEBUG
+	Recycle(t);
+#endif
 
-		Recycle(t);
+	if(rt2 != rt) Recycle(rt2);
-
-		if(rt2 != rt) Recycle(rt2);
-	}
 }
 
 // copy a multisample texture to a non-texture multisample. On opengl you need 2 FBO with different level of
@@ -1146,15 +1139,9 @@ void GSDeviceOGL::OMSetDepthStencilState(GSDepthStencilOGL* dss, uint8 sref)
 {
 	if (m_state.dss != dss) {
 		m_state.dss = dss;
-		m_state.sref = sref;
 
 		dss->SetupDepth();
-		dss->SetupStencil(sref);
+		dss->SetupStencil();
-
-	} else if (m_state.sref != sref) {
-		m_state.sref = sref;
-
-		dss->SetupStencil(sref);
 	}
 }
 
@@ -1171,39 +1158,41 @@ void GSDeviceOGL::OMSetBlendState(GSBlendStateOGL* bs, float bf)
 
 void GSDeviceOGL::OMSetRenderTargets(GSTexture* rt, GSTexture* ds, const GSVector4i* scissor)
 {
-	// Hum, need to separate 2 case, Render target fbo and render target backbuffer
+	if (rt == NULL || !static_cast<GSTextureOGL*>(rt)->IsBackbuffer()) {
-	// Or maybe blit final result to the backbuffer
+		if (rt) {
-	m_state.rtv = static_cast<GSTextureOGL*>(rt);
+			// FIXME DEBUG special case for GL_R16UI
-	m_state.dsv = static_cast<GSTextureOGL*>(ds);
+			if (rt->GetFormat() == GL_R16UI) {
-
+				OMSetFBO(m_fbo, GL_COLOR_ATTACHMENT1);
-	if (static_cast<GSTextureOGL*>(rt)->IsBackbuffer()) {
+				static_cast<GSTextureOGL*>(rt)->Attach(GL_COLOR_ATTACHMENT1);
-		ASSERT(ds == NULL); // no depth-stencil without FBO
+			} else {
-
+				OMSetFBO(m_fbo, GL_COLOR_ATTACHMENT0);
-		OMSetFBO(0);
+				static_cast<GSTextureOGL*>(rt)->Attach(GL_COLOR_ATTACHMENT0);
-
+			}
-	} else {
-		ASSERT(rt != NULL); // a render target must exists
-
-		// FIXME DEBUG special case for GL_R16UI
-		if (rt->GetFormat() == GL_R16UI) {
-			OMSetFBO(m_fbo, GL_COLOR_ATTACHMENT1);
-			static_cast<GSTextureOGL*>(rt)->Attach(GL_COLOR_ATTACHMENT1);
 		} else {
-			OMSetFBO(m_fbo, GL_COLOR_ATTACHMENT0);
+			// Note: NULL rt is only used in DATE so far. Color writing is disabled
-			static_cast<GSTextureOGL*>(rt)->Attach(GL_COLOR_ATTACHMENT0);
+			// on the blend setup
+			OMSetFBO(m_fbo, GL_NONE);
 		}
 
-		if (ds != NULL)
+		// Note: it must be done after OMSetFBO
+		if (ds)
 			static_cast<GSTextureOGL*>(ds)->Attach(GL_DEPTH_STENCIL_ATTACHMENT);
+
+	} else {
+		// Render in the backbuffer
+		OMSetFBO(0);
 	}
 
-	if(m_state.viewport != rt->GetSize())
+
+
+	GSVector2i size = rt ? rt->GetSize() : ds->GetSize();
+	if(m_state.viewport != size)
 	{
-		m_state.viewport = rt->GetSize();
+		m_state.viewport = size;
-		glViewport(0, 0, rt->GetWidth(), rt->GetHeight());
+		glViewport(0, 0, size.x, size.y);
 	}
 
-	GSVector4i r = scissor ? *scissor : GSVector4i(rt->GetSize()).zwxy();
+	GSVector4i r = scissor ? *scissor : GSVector4i(size).zwxy();
 
 	if(!m_state.scissor.eq(r))
 	{
@@ -1249,6 +1238,9 @@ void GSDeviceOGL::CompileShaderFromSource(const std::string& glsl_file, const st
 		version += "#extension GL_ARB_explicit_attrib_location : require\n";
 		version += "#extension GL_ARB_uniform_buffer_object : require\n";
 	}
+#ifdef ENABLE_OGL_STENCIL_DEBUG
+	version += "#define ENABLE_OGL_STENCIL_DEBUG 1\n";
+#endif
 
 	// Allow to puts several shader in 1 files
 	std::string shader_type;

plugins/GSdx/GSDeviceOGL.h

@@ -152,11 +152,11 @@ class GSDepthStencilOGL {
 	GLboolean m_depth_mask;
 	// Note front face and back might be split but it seems they have same parameter configuration
 	bool m_stencil_enable;
-	GLuint m_stencil_mask;
+	const GLuint m_stencil_mask;
 	GLuint m_stencil_func;
-	GLuint m_stencil_ref;
+	const GLuint m_stencil_ref;
-	GLuint m_stencil_sfail_op;
+	const GLuint m_stencil_sfail_op;
-	GLuint m_stencil_spass_dfail_op;
+	const GLuint m_stencil_spass_dfail_op;
 	GLuint m_stencil_spass_dpass_op;
 
 	char* NameOfParam(GLenum p)
@@ -182,7 +182,7 @@ public:
 		, m_stencil_enable(false)
 		, m_stencil_mask(1)
 		, m_stencil_func(0)
-		, m_stencil_ref(0)
+		, m_stencil_ref(1)
 		, m_stencil_sfail_op(GL_KEEP)
 		, m_stencil_spass_dfail_op(GL_KEEP)
 		, m_stencil_spass_dpass_op(GL_KEEP)
@@ -204,14 +204,14 @@ public:
 			glDisable(GL_DEPTH_TEST);
 	}
 
-	void SetupStencil(uint8 sref)
+	void SetupStencil()
 	{
-		uint32 ref = sref;
 		if (m_stencil_enable) {
 			glEnable(GL_STENCIL_TEST);
 			// Note: here the mask control which bitplane is considered by the operation
-			glStencilFunc(m_stencil_func, ref, m_stencil_mask);
+			glStencilFunc(m_stencil_func, m_stencil_ref, m_stencil_mask);
 			glStencilOp(m_stencil_sfail_op, m_stencil_spass_dfail_op, m_stencil_spass_dpass_op);
+			// FIXME only needed once since m_stencil_mask is constant
 			// Control which stencil bitplane are written
 			glStencilMask(m_stencil_mask);
 		} else
@@ -541,7 +541,6 @@ class GSDeviceOGL : public GSDevice
 		GSVector2i viewport;
 		GSVector4i scissor;
 		GSDepthStencilOGL* dss;
-		uint8 sref;
 		GSBlendStateOGL* bs;
 		float bf;
 		// FIXME texture attachment in the FBO

plugins/GSdx/GSTextureOGL.cpp

@@ -292,12 +292,15 @@ bool GSTextureOGL::Map(GSMap& m, const GSVector4i* r)
 		if (m_format == GL_RGBA8) {
 			glPixelStorei(GL_PACK_ALIGNMENT, 4);
 			glReadPixels(0, 0, m_size.x, m_size.y, GL_RGBA, GL_UNSIGNED_BYTE, 0);
+			m.pitch = m_size.x * 4;
 		} else if (m_format == GL_R16UI) {
 			glPixelStorei(GL_PACK_ALIGNMENT, 2);
 			glReadPixels(0, 0, m_size.x, m_size.y, GL_RED_INTEGER, GL_UNSIGNED_SHORT, 0);
+			m.pitch = m_size.x * 2;
 		} else if (m_format == GL_R8) {
 			glPixelStorei(GL_PACK_ALIGNMENT, 1);
 			glReadPixels(0, 0, m_size.x, m_size.y, GL_RED, GL_UNSIGNED_BYTE, 0);
+			m.pitch = m_size.x;
 		} else {
 			fprintf(stderr, "wrong texture pixel format :%x\n", m_format);
 			ASSERT(0);
@@ -306,7 +309,6 @@ bool GSTextureOGL::Map(GSMap& m, const GSVector4i* r)
 
 		// Give access from the CPU
 		m.bits = (uint8*) gl_MapBufferRange(GL_PIXEL_PACK_BUFFER, 0, m_pbo_size, GL_MAP_READ_BIT);
-		m.pitch = m_size.x;
 
 		if ( m.bits ) {
 			return true;

plugins/GSdx/res/convert.glsl

@@ -143,20 +143,28 @@ void ps_main6() // diagonal
     SV_Target0 = c;
 }
 
+// Used for DATE (stencil)
 void ps_main2()
 {
-    if((sample_c().a - 128.0f / 255) < 0) // >= 0x80 pass
+    if((sample_c().a - 127.5f / 255) < 0) // >= 0x80 pass
         discard;
 
-    SV_Target0 = vec4(0.0f, 0.0f, 0.0f, 0.0f);
+#ifdef ENABLE_OGL_STENCIL_DEBUG
+    SV_Target0 = vec4(1.0f, 0.0f, 0.0f, 1.0f);
+#endif
 }
+
+// Used for DATE (stencil)
 void ps_main3()
 {
-    if((127.95f / 255 - sample_c().a) <0) // < 0x80 pass (== 0x80 should not pass)
+    if((127.5f / 255 - sample_c().a) < 0) // < 0x80 pass (== 0x80 should not pass)
         discard;
 
-    SV_Target0 = vec4(0.0f, 0.0f, 0.0f, 0.0f);
+#ifdef ENABLE_OGL_STENCIL_DEBUG
+    SV_Target0 = vec4(1.0f, 0.0f, 0.0f, 1.0f);
+#endif
 }
+
 void ps_main4()
 {
     // FIXME mod and fmod are different when value are negative

plugins/GSdx/res/convert.h

@@ -171,20 +171,28 @@ static const char* convert_glsl =
 	"    SV_Target0 = c;\n"
 	"}\n"
 	"\n"
+	"// Used for DATE (stencil)\n"
 	"void ps_main2()\n"
 	"{\n"
-	"    if((sample_c().a - 128.0f / 255) < 0) // >= 0x80 pass\n"
+	"    if((sample_c().a - 127.5f / 255) < 0) // >= 0x80 pass\n"
 	"        discard;\n"
 	"\n"
-	"    SV_Target0 = vec4(0.0f, 0.0f, 0.0f, 0.0f);\n"
+	"#ifdef ENABLE_OGL_STENCIL_DEBUG\n"
+	"    SV_Target0 = vec4(1.0f, 0.0f, 0.0f, 1.0f);\n"
+	"#endif\n"
 	"}\n"
+	"\n"
+	"// Used for DATE (stencil)\n"
 	"void ps_main3()\n"
 	"{\n"
-	"    if((127.95f / 255 - sample_c().a) <0) // < 0x80 pass (== 0x80 should not pass)\n"
+	"    if((127.5f / 255 - sample_c().a) < 0) // < 0x80 pass (== 0x80 should not pass)\n"
 	"        discard;\n"
 	"\n"
-	"    SV_Target0 = vec4(0.0f, 0.0f, 0.0f, 0.0f);\n"
+	"#ifdef ENABLE_OGL_STENCIL_DEBUG\n"
+	"    SV_Target0 = vec4(1.0f, 0.0f, 0.0f, 1.0f);\n"
+	"#endif\n"
 	"}\n"
+	"\n"
 	"void ps_main4()\n"
 	"{\n"
 	"    // FIXME mod and fmod are different when value are negative\n"