GS/HW: Assume primitive does not overlap if it is a single quad

Enables one-barrier software blending for fullscreen quads.
We can also use tex-is-fb safely in these scenarios too.

Fixes Persona 4 menu background, Hard Hitter Tennis shadows at
Basic blending.

pcsx2/GS/GSState.cpp

@@ -2720,6 +2720,55 @@ void GSState::GrowVertexBuffer()
 	m_index.buff = index;
 }
 
+bool GSState::TrianglesAreQuads() const
+{
+	// If this is a quad, there should only be two distinct values for both X and Y, which
+	// also happen to be the minimum/maximum bounds of the primitive.
+	const GSVertex* const v = m_vertex.buff;
+	for (u32 idx = 0; idx < m_index.tail; idx += 6)
+	{
+		const u16* const i = m_index.buff + idx;
+
+		// Degenerate triangles should've been culled already, so we can check indices.
+		u32 extra_verts = 0;
+		for (u32 j = 3; j < 6; j++)
+		{
+			const u16 idx = i[j];
+			if (idx != i[0] && idx != i[1] && idx != i[2])
+				extra_verts++;
+		}
+		if (extra_verts == 1)
+			return true;
+
+		// As a fallback, they might've used different vertices with a tri list, not strip.
+		// Note that this won't work unless the quad is axis-aligned.
+		u16 distinct_x_values[2] = {v[i[0]].XYZ.X};
+		u16 distinct_y_values[2] = {v[i[0]].XYZ.Y};
+		u32 num_distinct_x_values = 1, num_distinct_y_values = 1;
+		for (u32 j = 1; j < 6; j++)
+		{
+			const GSVertex& jv = v[i[j]];
+			if (jv.XYZ.X != distinct_x_values[0] && jv.XYZ.X != distinct_x_values[1])
+			{
+				if (num_distinct_x_values > 1)
+					return false;
+
+				distinct_x_values[num_distinct_x_values++] = jv.XYZ.X;
+			}
+
+			if (jv.XYZ.Y != distinct_y_values[0] && jv.XYZ.Y != distinct_y_values[1])
+			{
+				if (num_distinct_y_values > 1)
+					return false;
+
+				distinct_y_values[num_distinct_y_values++] = jv.XYZ.Y;
+			}
+		}
+	}
+
+	return true;
+}
+
 GSState::PRIM_OVERLAP GSState::PrimitiveOverlap()
 {
 	// Either 1 triangle or 1 line or 3 POINTs
@@ -2727,7 +2776,9 @@ GSState::PRIM_OVERLAP GSState::PrimitiveOverlap()
 	if (m_vertex.next < 4)
 		return PRIM_OVERLAP_NO;
 
-	if (m_vt.m_primclass != GS_SPRITE_CLASS)
+	if (m_vt.m_primclass == GS_TRIANGLE_CLASS)
+		return (m_index.tail == 6 && TrianglesAreQuads()) ? PRIM_OVERLAP_NO : PRIM_OVERLAP_UNKNOW;
+	else if (m_vt.m_primclass != GS_SPRITE_CLASS)
 		return PRIM_OVERLAP_UNKNOW; // maybe, maybe not
 
 	// Check intersection of sprite primitive only

pcsx2/GS/GSState.h

@@ -418,6 +418,7 @@ public:
 
 	void DumpVertices(const std::string& filename);
 
+	bool TrianglesAreQuads() const;
 	PRIM_OVERLAP PrimitiveOverlap();
 	GIFRegTEX0 GetTex0Layer(u32 lod);
 };

pcsx2/GS/Renderers/HW/GSRendererHW.cpp

@@ -4412,7 +4412,7 @@ __ri void GSRendererHW::HandleTextureHazards(const GSTextureCache::Target* rt, c
 }
 
 bool GSRendererHW::CanUseTexIsFB(const GSTextureCache::Target* rt, const GSTextureCache::Source* tex,
-	const TextureMinMaxResult& tmm) const
+	const TextureMinMaxResult& tmm)
 {
 	// Minimum blending or no barriers -> we can't use tex-is-fb.
 	if (GSConfig.AccurateBlendingUnit == AccBlendLevel::Minimum || !g_gs_device->Features().texture_barrier)
@@ -4460,21 +4460,8 @@ bool GSRendererHW::CanUseTexIsFB(const GSTextureCache::Target* rt, const GSTextu
 
 	// Texture is actually the frame buffer. Stencil emulation to compute shadow (Jak series/tri-ace game)
 	// Will hit the "m_ps_sel.tex_is_fb = 1" path in the draw
-	if (m_vt.m_primclass == GS_TRIANGLE_CLASS)
+	const bool is_quads = (m_vt.m_primclass == GS_SPRITE_CLASS || m_prim_overlap == PRIM_OVERLAP_NO);
-	{
+	if (is_quads)
-		// This pattern is used by several games to emulate a stencil (shadow)
-		// Ratchet & Clank, Jak do alpha integer multiplication (tfx) which is mostly equivalent to +1/-1
-		// Tri-Ace (Star Ocean 3/RadiataStories/VP2) uses a palette to handle the +1/-1
-		if (m_cached_ctx.FRAME.FBMSK == 0x00FFFFFF)
-		{
-			GL_CACHE("Tex-is-fb hack for Jak");
-			return true;
-		}
-
-		GL_CACHE("Triangle draw, not using tex-is-fb");
-		return false;
-	}
-	else if (m_vt.m_primclass == GS_SPRITE_CLASS)
 	{
 		// No bilinear for tex-is-fb.
 		if (m_vt.IsLinear())
@@ -4507,6 +4494,21 @@ bool GSRendererHW::CanUseTexIsFB(const GSTextureCache::Target* rt, const GSTextu
 		return false;
 	}
 
+	if (m_vt.m_primclass == GS_TRIANGLE_CLASS)
+	{
+		// This pattern is used by several games to emulate a stencil (shadow)
+		// Ratchet & Clank, Jak do alpha integer multiplication (tfx) which is mostly equivalent to +1/-1
+		// Tri-Ace (Star Ocean 3/RadiataStories/VP2) uses a palette to handle the +1/-1
+		if (m_cached_ctx.FRAME.FBMSK == 0x00FFFFFF)
+		{
+			GL_CACHE("Tex-is-fb hack for Jak");
+			return true;
+		}
+
+		GL_CACHE("Triangle draw, not using tex-is-fb");
+		return false;
+	}
+
 	return false;
 }
 
@@ -5798,52 +5800,13 @@ bool GSRendererHW::PrimitiveCoversWithoutGaps()
 
 	if (m_vt.m_primclass == GS_POINT_CLASS)
 	{
-		m_primitive_covers_without_gaps = true;
+		m_primitive_covers_without_gaps = (m_vertex.next < 2);
-		return true;
+		return m_primitive_covers_without_gaps.value();
 	}
 	else if (m_vt.m_primclass == GS_TRIANGLE_CLASS)
 	{
-		if (m_index.tail != 6)
+		m_primitive_covers_without_gaps = (m_index.tail == 6 && TrianglesAreQuads());
-		{
+		return m_primitive_covers_without_gaps.value();
-			m_primitive_covers_without_gaps = false;
-			return false;
-		}
-
-		// If this is a quad, there should only be two distinct values for both X and Y, which
-		// also happen to be the minimum/maximum bounds of the primitive.
-		const GSVertex* const v = m_vertex.buff;
-		const u16* const i = m_index.buff;
-		u16 distinct_x_values[2] = {v[i[0]].XYZ.X};
-		u16 distinct_y_values[2] = {v[i[0]].XYZ.Y};
-		u32 num_distinct_x_values = 1, num_distinct_y_values = 1;
-		for (u32 j = 1; j < 6; j++)
-		{
-			const GSVertex& jv = v[i[j]];
-			if (jv.XYZ.X != distinct_x_values[0] && jv.XYZ.X != distinct_x_values[1])
-			{
-				if (num_distinct_x_values > 1)
-				{
-					m_primitive_covers_without_gaps = false;
-					return false;
-				}
-
-				distinct_x_values[num_distinct_x_values++] = jv.XYZ.X;
-			}
-
-			if (jv.XYZ.Y != distinct_y_values[0] && jv.XYZ.Y != distinct_y_values[1])
-			{
-				if (num_distinct_y_values > 1)
-				{
-					m_primitive_covers_without_gaps = false;
-					return false;
-				}
-
-				distinct_y_values[num_distinct_y_values++] = jv.XYZ.Y;
-			}
-		}
-
-		m_primitive_covers_without_gaps = true;
-		return true;
 	}
 	else if (m_vt.m_primclass != GS_SPRITE_CLASS)
 	{

pcsx2/GS/Renderers/HW/GSRendererHW.h

@@ -94,7 +94,7 @@ private:
 		const GSTextureCache::Source* tex, const TextureMinMaxResult& tmm, GSTextureCache::SourceRegion& source_region,
 		bool& target_region, GSVector2i& unscaled_size, float& scale, GSTexture*& src_copy);
 	bool CanUseTexIsFB(const GSTextureCache::Target* rt, const GSTextureCache::Source* tex,
-		const TextureMinMaxResult& tmm) const;
+		const TextureMinMaxResult& tmm);
 
 	void EmulateZbuffer(const GSTextureCache::Target* ds);
 	void EmulateATST(float& AREF, GSHWDrawConfig::PSSelector& ps, bool pass_2);