diff --git a/bin/resources/shaders/dx11/tfx.fx b/bin/resources/shaders/dx11/tfx.fx
index ba3b15a96f..7ed81c3d6a 100644
--- a/bin/resources/shaders/dx11/tfx.fx
+++ b/bin/resources/shaders/dx11/tfx.fx
@@ -852,6 +852,8 @@ void ps_color_clamp_wrap(inout float3 C)
 		else if (PS_COLCLIP == 1 || PS_HDR == 1)
 			C = (float3)((int3)C & (int3)0xFF);
 	}
+	else if (PS_DST_FMT == FMT_16 && PS_DITHER != 3 && PS_BLEND_MIX == 0 && PS_BLEND_HW == 0)
+		C = (float3)((int3)C & (int3)0xF8);
 }
 
 void ps_blend(inout float4 Color, inout float4 As_rgba, float2 pos_xy)
diff --git a/bin/resources/shaders/opengl/tfx_fs.glsl b/bin/resources/shaders/opengl/tfx_fs.glsl
index bb004e3891..cf10c58f25 100644
--- a/bin/resources/shaders/opengl/tfx_fs.glsl
+++ b/bin/resources/shaders/opengl/tfx_fs.glsl
@@ -775,6 +775,8 @@ void ps_color_clamp_wrap(inout vec3 C)
 	C = vec3(ivec3(C) & ivec3(0xFF));
 #endif
 
+#elif PS_DST_FMT == FMT_16 && PS_DITHER != 3 && PS_BLEND_MIX == 0 && PS_BLEND_HW == 0
+	C = vec3(ivec3(C) & ivec3(0xF8));
 #endif
 }
 
diff --git a/bin/resources/shaders/vulkan/tfx.glsl b/bin/resources/shaders/vulkan/tfx.glsl
index a7050b193d..519af4681b 100644
--- a/bin/resources/shaders/vulkan/tfx.glsl
+++ b/bin/resources/shaders/vulkan/tfx.glsl
@@ -1045,6 +1045,8 @@ void ps_color_clamp_wrap(inout vec3 C)
 	C = vec3(ivec3(C) & ivec3(0xFF));
 #endif
 
+#elif PS_DST_FMT == FMT_16 && PS_DITHER != 3 && PS_BLEND_MIX == 0 && PS_BLEND_HW == 0
+	C = vec3(ivec3(C) & ivec3(0xF8));
 #endif
 }
 
diff --git a/pcsx2/GS/Renderers/Metal/tfx.metal b/pcsx2/GS/Renderers/Metal/tfx.metal
index 48cd9c7095..447d3018a2 100644
--- a/pcsx2/GS/Renderers/Metal/tfx.metal
+++ b/pcsx2/GS/Renderers/Metal/tfx.metal
@@ -890,28 +890,31 @@ struct PSMain
 
 	void ps_color_clamp_wrap(thread float4& C)
 	{
-		// When dithering the bottom 3 bits become meaningless and cause lines in the picture so we need to limit the color depth on dithered items
-		if (!SW_BLEND && !(PS_DITHER > 0 && PS_DITHER < 3) && !PS_FBMASK)
-			return;
+		// When dithering the bottom 3 bits become meaningless and cause lines in the picture
+		// so we need to limit the color depth on dithered items
+		if (SW_BLEND || (PS_DITHER > 0 && PS_DITHER < 3) || PS_FBMASK)
+		{
+			if (PS_DST_FMT == FMT_16 && PS_BLEND_MIX == 0 && PS_ROUND_INV)
+				C.rgb += 7.f; // Need to round up, not down since the shader will invert
 
-		if (PS_DST_FMT == FMT_16 && PS_BLEND_MIX == 0 && PS_ROUND_INV)
-			C.rgb += 7.f; // Need to round up, not down since the shader will invert
+			// Correct the Color value based on the output format
+			if (PS_COLCLIP == 0 && PS_HDR == 0)
+				C.rgb = clamp(C.rgb, 0.f, 255.f); // Standard Clamp
 
-		// Correct the Color value based on the output format
-		if (!PS_COLCLIP && !PS_HDR)
-			C.rgb = clamp(C.rgb, 0.f, 255.f); // Standard Clamp
+			// FIXME rouding of negative float?
+			// compiler uses trunc but it might need floor
 
-		// FIXME rouding of negative float?
-		// compiler uses trunc but it might need floor
-
-		// Warning: normally blending equation is mult(A, B) = A * B >> 7. GPU have the full accuracy
-		// GS: Color = 1, Alpha = 255 => output 1
-		// GPU: Color = 1/255, Alpha = 255/255 * 255/128 => output 1.9921875
-		if (PS_DST_FMT == FMT_16 && PS_DITHER < 3 && (PS_BLEND_MIX == 0 || PS_DITHER))
+			// Warning: normally blending equation is mult(A, B) = A * B >> 7. GPU have the full accuracy
+			// GS: Color = 1, Alpha = 255 => output 1
+			// GPU: Color = 1/255, Alpha = 255/255 * 255/128 => output 1.9921875
 			// In 16 bits format, only 5 bits of colors are used. It impacts shadows computation of Castlevania
+			if (PS_DST_FMT == FMT_16 && PS_DITHER != 3 && (PS_BLEND_MIX == 0 || PS_DITHER))
+				C.rgb = float3(short3(C.rgb) & 0xF8);
+			else if (PS_COLCLIP == 1 || PS_HDR == 1)
+				C.rgb = float3(short3(C.rgb) & 0xFF);
+		}
+		else if (PS_DST_FMT == FMT_16 && PS_DITHER != 3 && PS_BLEND_MIX == 0 && PS_BLEND_HW == 0)
 			C.rgb = float3(short3(C.rgb) & 0xF8);
-		else if (PS_COLCLIP || PS_HDR)
-			C.rgb = float3(short3(C.rgb) & 0xFF);
 	}
 
 	template <typename T>
diff --git a/pcsx2/ShaderCacheVersion.h b/pcsx2/ShaderCacheVersion.h
index 66693958f6..e4de26746b 100644
--- a/pcsx2/ShaderCacheVersion.h
+++ b/pcsx2/ShaderCacheVersion.h
@@ -3,4 +3,4 @@
 
 /// Version number for GS and other shaders. Increment whenever any of the contents of the
 /// shaders change, to invalidate the cache.
-static constexpr u32 SHADER_CACHE_VERSION = 60;
+static constexpr u32 SHADER_CACHE_VERSION = 61;