From badede2e2d18feb2b33c00c0cb7cbcf39ba95acd Mon Sep 17 00:00:00 2001 From: refractionpcsx2 Date: Sun, 14 Jan 2024 04:41:24 +0000 Subject: [PATCH] GS/HW: ignore lower 3 bits of 16bit color for AEM check --- bin/resources/shaders/dx11/tfx.fx | 2 +- bin/resources/shaders/opengl/tfx_fs.glsl | 2 +- bin/resources/shaders/vulkan/tfx.glsl | 2 +- pcsx2/GS/Renderers/Metal/tfx.metal | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/resources/shaders/dx11/tfx.fx b/bin/resources/shaders/dx11/tfx.fx index 9d9a45105d..80915b1ef7 100644 --- a/bin/resources/shaders/dx11/tfx.fx +++ b/bin/resources/shaders/dx11/tfx.fx @@ -631,7 +631,7 @@ float4 sample_color(float2 st, float uv_w) } else if(PS_AEM_FMT == FMT_16) { - c[i].a = c[i].a >= 0.5 ? TA.y : !PS_AEM || any(c[i].rgb) ? TA.x : 0; + c[i].a = c[i].a >= 0.5 ? TA.y : !PS_AEM || any(int3(c[i].rgb * 255.0f) & 0xF8) ? TA.x : 0; } } diff --git a/bin/resources/shaders/opengl/tfx_fs.glsl b/bin/resources/shaders/opengl/tfx_fs.glsl index e12a4ae6d1..33728f964e 100644 --- a/bin/resources/shaders/opengl/tfx_fs.glsl +++ b/bin/resources/shaders/opengl/tfx_fs.glsl @@ -580,7 +580,7 @@ vec4 sample_color(vec2 st) c[i].a = ( (PS_AEM == 0) || any(bvec3(c[i].rgb)) ) ? TA.x : 0.0f; //c[i].a = ( (PS_AEM == 0) || (sum > 0.0f) ) ? TA.x : 0.0f; #elif (PS_AEM_FMT == FMT_16) - c[i].a = c[i].a >= 0.5 ? TA.y : ( (PS_AEM == 0) || any(bvec3(c[i].rgb)) ) ? TA.x : 0.0f; + c[i].a = c[i].a >= 0.5 ? TA.y : ( (PS_AEM == 0) || any(bvec3(ivec3(c[i].rgb * 255.0f) & ivec3(0xF8))) ) ? TA.x : 0.0f; //c[i].a = c[i].a >= 0.5 ? TA.y : ( (PS_AEM == 0) || (sum > 0.0f) ) ? TA.x : 0.0f; #endif } diff --git a/bin/resources/shaders/vulkan/tfx.glsl b/bin/resources/shaders/vulkan/tfx.glsl index a40bd9535d..73c80a03f6 100644 --- a/bin/resources/shaders/vulkan/tfx.glsl +++ b/bin/resources/shaders/vulkan/tfx.glsl @@ -817,7 +817,7 @@ vec4 sample_color(vec2 st) #if (PS_AEM_FMT == FMT_24) c[i].a = (PS_AEM == 0 || any(bvec3(c[i].rgb))) ? TA.x : 0.0f; #elif (PS_AEM_FMT == FMT_16) - c[i].a = (c[i].a >= 0.5) ? TA.y : ((PS_AEM == 0 || any(bvec3(c[i].rgb))) ? TA.x : 0.0f); + c[i].a = (c[i].a >= 0.5) ? TA.y : ((PS_AEM == 0 || any(bvec3(ivec3(c[i].rgb * 255.0f) & ivec3(0xF8)))) ? TA.x : 0.0f); #endif } diff --git a/pcsx2/GS/Renderers/Metal/tfx.metal b/pcsx2/GS/Renderers/Metal/tfx.metal index 0dc78180ad..294c49b048 100644 --- a/pcsx2/GS/Renderers/Metal/tfx.metal +++ b/pcsx2/GS/Renderers/Metal/tfx.metal @@ -702,7 +702,7 @@ struct PSMain if (PS_AEM_FMT == FMT_24) c[i].a = !PS_AEM || any(c[i].rgb != 0) ? cb.ta.x : 0.f; else if (PS_AEM_FMT == FMT_16) - c[i].a = c[i].a >= 0.5 ? cb.ta.y : !PS_AEM || any(c[i].rgb != 0) ? cb.ta.x : 0.f; + c[i].a = c[i].a >= 0.5 ? cb.ta.y : !PS_AEM || any((int3(c[i].rgb * 255.0f) & 0xF8) != 0) ? cb.ta.x : 0.f; } if (PS_LTF)