GS: Blend truncation and dither goes the other way when subtracting

Truncation happens after subtraction, so it's the equivalent of rounding the value to subtract *up* instead of down

bin/resources/shaders/dx11/tfx.fx

@@ -53,6 +53,7 @@
 #define PS_BLEND_C 0
 #define PS_BLEND_D 0
 #define PS_BLEND_MIX 0
+#define PS_ROUND_INV 0
 #define PS_FIXED_ONE_A 0
 #define PS_PABE 0
 #define PS_DITHER 0
@@ -741,7 +742,11 @@ void ps_dither(inout float3 C, float2 pos_xy)
 		else
 			fpos = int2(pos_xy / (float)PS_SCALE_FACTOR);
 
-		C += DitherMatrix[fpos.x & 3][fpos.y & 3];
+		float value = DitherMatrix[fpos.x & 3][fpos.y & 3];
+		if (PS_ROUND_INV != 0)
+			C -= value;
+		else
+			C += value;
 	}
 }
 
@@ -751,6 +756,9 @@ void ps_color_clamp_wrap(inout float3 C)
 	// so we need to limit the color depth on dithered items
 	if (SW_BLEND || PS_DITHER || PS_FBMASK)
 	{
+		if (PS_DFMT == FMT_16 && PS_BLEND_MIX == 0 && PS_ROUND_INV != 0)
+			C += 7.0f; // Need to round up, not down since the shader will invert
+
 		// Standard Clamp
 		if (PS_COLCLIP == 0 && PS_HDR == 0)
 			C = clamp(C, (float3)0.0f, (float3)255.0f);

bin/resources/shaders/opengl/tfx_fs.glsl

@@ -654,7 +654,12 @@ void ps_dither(inout vec3 C)
     #else
     ivec2 fpos = ivec2(gl_FragCoord.xy / float(PS_SCALE_FACTOR));
     #endif
-    C += DitherMatrix[fpos.y&3][fpos.x&3];
+    float value = DitherMatrix[fpos.y&3][fpos.x&3];
+    #if PS_ROUND_INV
+    C -= value;
+    #else
+    C += value;
+    #endif
 #endif
 }
 
@@ -664,6 +669,10 @@ void ps_color_clamp_wrap(inout vec3 C)
     // so we need to limit the color depth on dithered items
 #if SW_BLEND || PS_DITHER || PS_FBMASK
 
+#if PS_DFMT == FMT_16 && PS_BLEND_MIX == 0
+    C += 7.f; // Need to round up, not down since the shader will invert
+#endif
+
     // Correct the Color value based on the output format
 #if PS_COLCLIP == 0 && PS_HDR == 0
     // Standard Clamp

bin/resources/shaders/vulkan/tfx.glsl

@@ -970,7 +970,12 @@ void ps_dither(inout vec3 C)
 			fpos = ivec2(gl_FragCoord.xy / float(PS_SCALE_FACTOR));
 		#endif
 
-		C += DitherMatrix[fpos.y & 3][fpos.x & 3];
+		float value = DitherMatrix[fpos.y & 3][fpos.x & 3];
+		#if PS_ROUND_INV
+			C -= value;
+		#else
+			C += value;
+		#endif
 	#endif
 }
 
@@ -980,6 +985,10 @@ void ps_color_clamp_wrap(inout vec3 C)
 	// so we need to limit the color depth on dithered items
 #if SW_BLEND || PS_DITHER || PS_FBMASK
 
+#if PS_DFMT == FMT_16 && PS_BLEND_MIX == 0 && PS_ROUND_INV != 0
+	C += 7.0f; // Need to round up, not down since the shader will invert
+#endif
+
 	// Correct the Color value based on the output format
 #if PS_COLCLIP == 0 && PS_HDR == 0
 	// Standard Clamp

pcsx2/GS/Renderers/Common/GSDevice.h

@@ -332,6 +332,7 @@ struct alignas(16) GSHWDrawConfig
 				u32 hdr         : 1;
 				u32 colclip     : 1;
 				u32 blend_mix   : 2;
+				u32 round_inv   : 1; // Blending will invert the value, so rounding needs to go the other way
 				u32 pabe        : 1;
 				u32 no_color    : 1; // disables color output entirely (depth only)
 				u32 no_color1   : 1; // disables second color output (when unnecessary)

pcsx2/GS/Renderers/DX11/GSTextureFX11.cpp

@@ -174,6 +174,7 @@ void GSDevice11::SetupPS(const PSSelector& sel, const GSHWDrawConfig::PSConstant
 		sm.AddMacro("PS_BLEND_C", sel.blend_c);
 		sm.AddMacro("PS_BLEND_D", sel.blend_d);
 		sm.AddMacro("PS_BLEND_MIX", sel.blend_mix);
+		sm.AddMacro("PS_ROUND_INV", sel.round_inv);
 		sm.AddMacro("PS_FIXED_ONE_A", sel.fixed_one_a);
 		sm.AddMacro("PS_PABE", sel.pabe);
 		sm.AddMacro("PS_DITHER", sel.dither);

pcsx2/GS/Renderers/DX12/GSDevice12.cpp

@@ -1515,6 +1515,7 @@ const ID3DBlob* GSDevice12::GetTFXPixelShader(const GSHWDrawConfig::PSSelector&
 	sm.AddMacro("PS_BLEND_C", sel.blend_c);
 	sm.AddMacro("PS_BLEND_D", sel.blend_d);
 	sm.AddMacro("PS_BLEND_MIX", sel.blend_mix);
+	sm.AddMacro("PS_ROUND_INV", sel.round_inv);
 	sm.AddMacro("PS_FIXED_ONE_A", sel.fixed_one_a);
 	sm.AddMacro("PS_PABE", sel.pabe);
 	sm.AddMacro("PS_DITHER", sel.dither);

pcsx2/GS/Renderers/HW/GSRendererHW.cpp

@@ -3226,6 +3226,10 @@ void GSRendererHW::EmulateBlending(bool& DATE_PRIMID, bool& DATE_BARRIER, bool&
 							   !GSDevice::IsDualSourceBlendFactor(m_conf.blend.dst_factor);
 	}
 
+	// Notify the shader that it needs to invert rounding
+	if (m_conf.blend.op == GSDevice::OP_REV_SUBTRACT)
+		m_conf.ps.round_inv = 1;
+
 	// DATE_PRIMID interact very badly with sw blending. DATE_PRIMID uses the primitiveID to find the primitive
 	// that write the bad alpha value. Sw blending will force the draw to run primitive by primitive
 	// (therefore primitiveID will be constant to 1).

pcsx2/GS/Renderers/Metal/GSDeviceMTL.mm

@@ -1391,6 +1391,7 @@ void GSDeviceMTL::MRESetHWPipelineState(GSHWDrawConfig::VSSelector vssel, GSHWDr
 		setFnConstantB(m_fn_constants, pssel.hdr,                GSMTLConstantIndex_PS_HDR);
 		setFnConstantB(m_fn_constants, pssel.colclip,            GSMTLConstantIndex_PS_COLCLIP);
 		setFnConstantI(m_fn_constants, pssel.blend_mix,          GSMTLConstantIndex_PS_BLEND_MIX);
+		setFnConstantB(m_fn_constants, pssel.round_inv,          GSMTLConstantIndex_PS_ROUND_INV);
 		setFnConstantB(m_fn_constants, pssel.fixed_one_a,        GSMTLConstantIndex_PS_FIXED_ONE_A);
 		setFnConstantB(m_fn_constants, pssel.pabe,               GSMTLConstantIndex_PS_PABE);
 		setFnConstantB(m_fn_constants, pssel.no_color,           GSMTLConstantIndex_PS_NO_COLOR);

pcsx2/GS/Renderers/Metal/GSMTLSharedHeader.h

@@ -186,6 +186,7 @@ enum GSMTLFnConstants
 	GSMTLConstantIndex_PS_HDR,
 	GSMTLConstantIndex_PS_COLCLIP,
 	GSMTLConstantIndex_PS_BLEND_MIX,
+	GSMTLConstantIndex_PS_ROUND_INV,
 	GSMTLConstantIndex_PS_FIXED_ONE_A,
 	GSMTLConstantIndex_PS_PABE,
 	GSMTLConstantIndex_PS_NO_COLOR,

pcsx2/GS/Renderers/Metal/tfx.metal

@@ -53,6 +53,7 @@ constant uint PS_CLR_HW             [[function_constant(GSMTLConstantIndex_PS_CL
 constant bool PS_HDR                [[function_constant(GSMTLConstantIndex_PS_HDR)]];
 constant bool PS_COLCLIP            [[function_constant(GSMTLConstantIndex_PS_COLCLIP)]];
 constant uint PS_BLEND_MIX          [[function_constant(GSMTLConstantIndex_PS_BLEND_MIX)]];
+constant bool PS_ROUND_INV          [[function_constant(GSMTLConstantIndex_PS_ROUND_INV)]];
 constant bool PS_FIXED_ONE_A        [[function_constant(GSMTLConstantIndex_PS_FIXED_ONE_A)]];
 constant bool PS_PABE               [[function_constant(GSMTLConstantIndex_PS_PABE)]];
 constant bool PS_NO_COLOR           [[function_constant(GSMTLConstantIndex_PS_NO_COLOR)]];
@@ -793,7 +794,11 @@ struct PSMain
 			fpos = ushort2(in.p.xy);
 		else
 			fpos = ushort2(in.p.xy / SCALING_FACTOR);
-		C.rgb += cb.dither_matrix[fpos.y & 3][fpos.x & 3];
+		float value = cb.dither_matrix[fpos.y & 3][fpos.x & 3];;
+		if (PS_ROUND_INV)
+			C.rgb -= value;
+		else
+			C.rgb += value;
 	}
 
 	void ps_color_clamp_wrap(thread float4& C)
@@ -802,6 +807,9 @@ struct PSMain
 		if (!SW_BLEND && !PS_DITHER && !PS_FBMASK)
 			return;
 
+		if (PS_DFMT == FMT_16 && PS_BLEND_MIX == 0 && PS_ROUND_INV)
+			C.rgb += 7.f; // Need to round up, not down since the shader will invert
+
 		// Correct the Color value based on the output format
 		if (!PS_COLCLIP && !PS_HDR)
 			C.rgb = clamp(C.rgb, 0.f, 255.f); // Standard Clamp

pcsx2/GS/Renderers/OpenGL/GSDeviceOGL.cpp

@@ -1067,6 +1067,7 @@ std::string GSDeviceOGL::GetPSSource(const PSSelector& sel)
 		+ fmt::format("#define PS_DITHER {}\n", sel.dither)
 		+ fmt::format("#define PS_ZCLAMP {}\n", sel.zclamp)
 		+ fmt::format("#define PS_BLEND_MIX {}\n", sel.blend_mix)
+		+ fmt::format("#define PS_ROUND_INV {}\n", sel.round_inv)
 		+ fmt::format("#define PS_FIXED_ONE_A {}\n", sel.fixed_one_a)
 		+ fmt::format("#define PS_PABE {}\n", sel.pabe)
 		+ fmt::format("#define PS_SCANMSK {}\n", sel.scanmsk)

pcsx2/GS/Renderers/Vulkan/GSDeviceVK.cpp

@@ -1979,6 +1979,7 @@ VkShaderModule GSDeviceVK::GetTFXFragmentShader(const GSHWDrawConfig::PSSelector
 	AddMacro(ss, "PS_BLEND_C", sel.blend_c);
 	AddMacro(ss, "PS_BLEND_D", sel.blend_d);
 	AddMacro(ss, "PS_BLEND_MIX", sel.blend_mix);
+	AddMacro(ss, "PS_ROUND_INV", sel.round_inv);
 	AddMacro(ss, "PS_FIXED_ONE_A", sel.fixed_one_a);
 	AddMacro(ss, "PS_IIP", sel.iip);
 	AddMacro(ss, "PS_SHUFFLE", sel.shuffle);

pcsx2/ShaderCacheVersion.h

@@ -15,4 +15,4 @@
 
 /// Version number for GS and other shaders. Increment whenever any of the contents of the
 /// shaders change, to invalidate the cache.
-static constexpr u32 SHADER_CACHE_VERSION = 14;
+static constexpr u32 SHADER_CACHE_VERSION = 15;