GS: Simulate scan mask

bin/resources/shaders/dx11/tfx.fx

@@ -50,6 +50,7 @@
 #define PS_PABE 0
 #define PS_DITHER 0
 #define PS_ZCLAMP 0
+#define PS_SCANMSK 0
 #endif
 
 #define SW_BLEND (PS_BLEND_A || PS_BLEND_B || PS_BLEND_D)
@@ -720,6 +721,13 @@ PS_OUTPUT ps_main(PS_INPUT input)
 
 	PS_OUTPUT output;
 
+	if (PS_SCANMSK & 2)
+	{
+		// fail depth test on prohibited lines
+		if ((int(input.p.y) & 1) == (PS_SCANMSK & 1))
+			discard;
+	}
+
 	if (PS_SHUFFLE)
 	{
 		uint4 denorm_c = uint4(C);

bin/resources/shaders/opengl/tfx_fs.glsl

@@ -718,6 +718,11 @@ void ps_blend(inout vec4 Color, float As)
 
 void ps_main()
 {
+#if PS_SCANMSK & 2
+    // fail depth test on prohibited lines
+ 	if ((int(gl_FragCoord.y) & 1) == (PS_SCANMSK & 1))
+ 	 	discard;
+#endif
 #if ((PS_DATE & 3) == 1 || (PS_DATE & 3) == 2)
 
 #if PS_WRITE_RG == 1

pcsx2/GS/GSState.cpp

@@ -34,6 +34,7 @@ GSState::GSState()
 	, m_crc(0)
 	, m_options(0)
 	, m_frameskip(0)
+	, m_scanmask_used(false)
 {
 	// m_nativeres seems to be a hack. Unfortunately it impacts draw call number which make debug painful in the replayer.
 	// Let's keep it disabled to ease debug.
@@ -219,6 +220,8 @@ void GSState::Reset()
 	m_vertex.tail = 0;
 	m_vertex.next = 0;
 	m_index.tail = 0;
+
+	m_scanmask_used = false;
 }
 
 void GSState::ResetHandlers()
@@ -1098,6 +1101,8 @@ void GSState::GIFRegHandlerSCANMSK(const GIFReg* RESTRICT r)
 		Flush();
 
 	m_env.SCANMSK = (GSVector4i)r->SCANMSK;
+	if (m_env.SCANMSK.MSK & 2)
+		m_scanmask_used = true;
 }
 
 template <int i>

pcsx2/GS/GSState.h

@@ -162,6 +162,8 @@ protected:
 	GSVector4i m_ofxy;
 	bool tex_flushed;
 
+	bool m_scanmask_used;
+
 	struct
 	{
 		GSVertex* buff;

pcsx2/GS/Renderers/Common/GSDevice.h

@@ -250,7 +250,10 @@ struct alignas(16) GSHWDrawConfig
 				u32 point_sampler : 1;
 				u32 invalid_tex0 : 1; // Lupin the 3rd
 
-				u32 _free2 : 6;
+				// Scan mask
+				u32 scanmsk : 2;
+
+				u32 _free2 : 4;
 			};
 
 			u64 key;

pcsx2/GS/Renderers/Common/GSRenderer.cpp

@@ -220,7 +220,9 @@ bool GSRenderer::Merge(int field)
 			off.x = tex[i]->GetScale().x * frame_diff.x;
 		}
 
-		if (display_diff.y >= 4) // Shouldn't this be >= 2?
+		if (m_scanmask_used && display_diff.y == 1) // Scanmask effect wouldn't look correct if we scale the offset
+			off.y = display_diff.y;
+		else if (display_diff.y >= 4) // Shouldn't this be >= 2?
 		{
 			off.y = tex[i]->GetScale().y * display_diff.y;
 

pcsx2/GS/Renderers/DX11/GSTextureFX11.cpp

@@ -189,6 +189,7 @@ void GSDevice11::SetupPS(PSSelector sel, const GSHWDrawConfig::PSConstantBuffer*
 		sm.AddMacro("PS_PABE", sel.pabe);
 		sm.AddMacro("PS_DITHER", sel.dither);
 		sm.AddMacro("PS_ZCLAMP", sel.zclamp);
+		sm.AddMacro("PS_SCANMSK", sel.scanmsk);
 
 		wil::com_ptr_nothrow<ID3D11PixelShader> ps;
 		CreateShader(m_tfx_source, "tfx.fx", nullptr, "ps_main", sm.GetPtr(), ps.put());

pcsx2/GS/Renderers/HW/GSRendererNew.cpp

@@ -1174,6 +1174,7 @@ void GSRendererNew::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sour
 
 	ResetStates();
 	m_conf.cb_vs.texture_offset = GSVector2(0, 0);
+	m_conf.ps.scanmsk = m_env.SCANMSK.MSK;
 
 	ASSERT(m_dev != NULL);
 
@@ -1300,6 +1301,9 @@ void GSRendererNew::DrawPrims(GSTexture* rt, GSTexture* ds, GSTextureCache::Sour
 		m_conf.blend = {}; // No blending please
 	}
 
+	if (m_conf.ps.scanmsk & 2)
+		DATE_GL42 = false; // to have discard in the shader work correctly
+
 	if (m_conf.ps.dfmt == 1)
 	{
 		// Disable writing of the alpha channel

pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.cpp

@@ -1008,6 +1008,7 @@ GLuint GSDeviceOGL::CompilePS(PSSelector sel)
 		+ format("#define PS_DITHER %d\n", sel.dither)
 		+ format("#define PS_ZCLAMP %d\n", sel.zclamp)
 		+ format("#define PS_PABE %d\n", sel.pabe)
+		+ format("#define PS_SCANMSK %d\n", sel.scanmsk)
 		+ format("#define PS_SCALE_FACTOR %d\n", m_upscale_multiplier)
 	;
 

pcsx2/GS/Renderers/SW/GSRasterizer.cpp

@@ -40,6 +40,7 @@ GSRasterizer::GSRasterizer(IDrawScanline* ds, int id, int threads, GSPerfMon* pe
 	, m_ds(ds)
 	, m_id(id)
 	, m_threads(threads)
+	, m_scanmsk_value(0)
 {
 	memset(&m_pixels, 0, sizeof(m_pixels));
 	m_primcount = 0;
@@ -158,6 +159,7 @@ void GSRasterizer::Draw(GSRasterizerData* data)
 	m_scissor = data->scissor;
 	m_fscissor_x = GSVector4(data->scissor).xzxz();
 	m_fscissor_y = GSVector4(data->scissor).ywyw();
+	m_scanmsk_value = data->scanmsk_value;
 
 	switch (data->primclass)
 	{
@@ -834,7 +836,7 @@ void GSRasterizer::DrawSprite(const GSVertexSW* vertex, const u32* index)
 
 	GSVertexSW scan = v[0];
 
-	if (m_ds->IsSolidRect())
+	if ((m_scanmsk_value & 2) == 0 && m_ds->IsSolidRect())
 	{
 		if (m_threads == 1)
 		{
@@ -1158,6 +1160,7 @@ void GSRasterizer::Flush(const GSVertexSW* vertex, const u32* index, const GSVer
 
 void GSRasterizer::DrawScanline(int pixels, int left, int top, const GSVertexSW& scan)
 {
+	if ((m_scanmsk_value & 2) && (m_scanmsk_value & 1) == (top & 1)) return;
 	m_pixels.actual += pixels;
 	m_pixels.total += ((left + pixels + (PIXELS_PER_LOOP - 1)) & ~(PIXELS_PER_LOOP - 1)) - (left & ~(PIXELS_PER_LOOP - 1));
 	//m_pixels.total += ((left + pixels + (PIXELS_PER_LOOP - 1)) & ~(PIXELS_PER_LOOP - 1)) - left;
@@ -1169,6 +1172,7 @@ void GSRasterizer::DrawScanline(int pixels, int left, int top, const GSVertexSW&
 
 void GSRasterizer::DrawEdge(int pixels, int left, int top, const GSVertexSW& scan)
 {
+	if ((m_scanmsk_value & 2) && (m_scanmsk_value & 1) == (top & 1)) return;
 	m_pixels.actual += 1;
 	m_pixels.total += PIXELS_PER_LOOP - 1;
 

pcsx2/GS/Renderers/SW/GSRasterizer.h

@@ -39,6 +39,7 @@ public:
 	u64 start;
 	int pixels;
 	int counter;
+	u8 scanmsk_value;
 
 	GSRasterizerData()
 		: scissor(GSVector4i::zero())
@@ -52,6 +53,7 @@ public:
 		, frame(0)
 		, start(0)
 		, pixels(0)
+		, scanmsk_value(0)
 	{
 		counter = s_counter++;
 	}
@@ -132,6 +134,7 @@ protected:
 	int m_threads;
 	int m_thread_height;
 	u8* m_scanline;
+	u8 m_scanmsk_value;
 	GSVector4i m_scissor;
 	GSVector4 m_fscissor_x;
 	GSVector4 m_fscissor_y;

pcsx2/GS/Renderers/SW/GSRendererSW.cpp

@@ -344,6 +344,7 @@ void GSRendererSW::Draw()
 	sd->vertex_count = m_vertex.next;
 	sd->index = (u32*)(sd->buff + sizeof(GSVertexSW) * ((m_vertex.next + 1) & ~1));
 	sd->index_count = m_index.tail;
+	sd->scanmsk_value = m_env.SCANMSK.MSK;
 
 	// skip per pixel division if q is constant.
 	// Optimize the division by 1 with a nop. It also means that GS_SPRITE_CLASS must be processed when !m_vt.m_eq.q.