From 906225d92f9321c429c7d34c821f8e3eba45979c Mon Sep 17 00:00:00 2001 From: Erik Abair Date: Mon, 2 Jun 2025 12:56:12 -0700 Subject: [PATCH] nv2a: Handle color keying with X-alpha textures --- hw/xbox/nv2a/pgraph/gl/renderer.h | 1 + hw/xbox/nv2a/pgraph/gl/shaders.c | 91 +++++++++++++++++-------------- hw/xbox/nv2a/pgraph/glsl/psh.c | 32 +++++++---- hw/xbox/nv2a/pgraph/psh.h | 1 + hw/xbox/nv2a/pgraph/shaders.c | 13 +++++ hw/xbox/nv2a/pgraph/vk/renderer.h | 1 + hw/xbox/nv2a/pgraph/vk/shaders.c | 16 ++++-- 7 files changed, 98 insertions(+), 57 deletions(-) diff --git a/hw/xbox/nv2a/pgraph/gl/renderer.h b/hw/xbox/nv2a/pgraph/gl/renderer.h index 24fe37ad59..a722a292ac 100644 --- a/hw/xbox/nv2a/pgraph/gl/renderer.h +++ b/hw/xbox/nv2a/pgraph/gl/renderer.h @@ -129,6 +129,7 @@ typedef struct ShaderBinding { GLint material_alpha_loc; GLint color_key_loc[4]; + GLint color_key_ignore_alpha_loc[4]; } ShaderBinding; typedef struct VertexKey { diff --git a/hw/xbox/nv2a/pgraph/gl/shaders.c b/hw/xbox/nv2a/pgraph/gl/shaders.c index ecfb6c74b8..9dc4e2487c 100644 --- a/hw/xbox/nv2a/pgraph/gl/shaders.c +++ b/hw/xbox/nv2a/pgraph/gl/shaders.c @@ -209,6 +209,10 @@ static void update_shader_constant_locations(ShaderBinding *binding) snprintf(tmp, sizeof(tmp), "colorKey[%d]", i); binding->color_key_loc[i] = glGetUniformLocation(binding->gl_program, tmp); + + snprintf(tmp, sizeof(tmp), "colorKeyIgnoreAlpha[%d]", i); + binding->color_key_ignore_alpha_loc[i] = + glGetUniformLocation(binding->gl_program, tmp); } } @@ -703,6 +707,7 @@ static void shader_update_constants(PGRAPHState *pg, ShaderBinding *binding, bool binding_changed) { PGRAPHGLState *r = pg->gl_renderer_state; + ShaderState *state = &binding->state; int i, j; /* update combiner constants */ @@ -732,13 +737,6 @@ static void shader_update_constants(PGRAPHState *pg, ShaderBinding *binding, glUniform1i(binding->alpha_ref_loc, alpha_ref); } - for (int i = 0; i < 4; ++i) { - if (binding->color_key_loc[i] != -1) { - glUniform1ui(binding->color_key_loc[i], - pgraph_reg_r(pg, NV_PGRAPH_COLORKEYCOLOR0 + i * 4)); - } - } - /* For each texture stage */ for (i = 0; i < NV2A_MAX_TEXTURES; i++) { GLint loc; @@ -778,6 +776,15 @@ static void shader_update_constants(PGRAPHState *pg, ShaderBinding *binding, assert(r->texture_binding[i] != NULL); glUniform1f(loc, (float)r->texture_binding[i]->scale); } + + if (binding->color_key_loc[i] != -1) { + glUniform1ui(binding->color_key_loc[i], + pgraph_reg_r(pg, NV_PGRAPH_COLORKEYCOLOR0 + i * 4)); + } + if (binding->color_key_ignore_alpha_loc[i] != -1) { + glUniform1i(binding->color_key_ignore_alpha_loc[i], + state->psh.colorkey_ignore_alpha[i]); + } } if (binding->fog_color_loc != -1) { @@ -807,7 +814,7 @@ static void shader_update_constants(PGRAPHState *pg, ShaderBinding *binding, assert(0); } - if (binding->state.fixed_function) { + if (state->fixed_function) { /* update lighting constants */ struct { uint32_t* v; @@ -973,41 +980,41 @@ static bool test_shaders_dirty(PGRAPHState *pg) #define CF_x__update(type, src, name, x) \ for (int i = 0; i < x; i++) { CNAME(name)[i] = src[i]; } -#define DIRTY_REGS \ - CR_1(NV_PGRAPH_COMBINECTL) \ - CR_1(NV_PGRAPH_SHADERCTL) \ - CR_1(NV_PGRAPH_SHADOWCTL) \ - CR_1(NV_PGRAPH_COMBINESPECFOG0) \ - CR_1(NV_PGRAPH_COMBINESPECFOG1) \ - CR_1(NV_PGRAPH_CONTROL_0) \ - CR_1(NV_PGRAPH_CONTROL_3) \ - CR_1(NV_PGRAPH_CSV0_C) \ - CR_1(NV_PGRAPH_CSV0_D) \ - CR_1(NV_PGRAPH_CSV1_A) \ - CR_1(NV_PGRAPH_CSV1_B) \ - CR_1(NV_PGRAPH_SETUPRASTER) \ - CR_1(NV_PGRAPH_SHADERPROG) \ - CR_1(NV_PGRAPH_ZCOMPRESSOCCLUDE) \ - CR_8(NV_PGRAPH_COMBINECOLORI0) \ - CR_8(NV_PGRAPH_COMBINECOLORO0) \ - CR_8(NV_PGRAPH_COMBINEALPHAI0) \ - CR_8(NV_PGRAPH_COMBINEALPHAO0) \ - CR_8(NV_PGRAPH_COMBINEFACTOR0) \ - CR_8(NV_PGRAPH_COMBINEFACTOR1) \ - CR_1(NV_PGRAPH_SHADERCLIPMODE) \ - CR_4(NV_PGRAPH_TEXCTL0_0) \ - CR_4(NV_PGRAPH_TEXFMT0) \ - CR_4(NV_PGRAPH_TEXFILTER0) \ - CR_8(NV_PGRAPH_WINDOWCLIPX0) \ - CR_8(NV_PGRAPH_WINDOWCLIPY0) \ - CF(pg->primitive_mode, primitive_mode) \ - CF(pg->surface_scale_factor, surface_scale_factor) \ - CF(pg->compressed_attrs, compressed_attrs) \ - CFA(pg->texture_matrix_enable, texture_matrix_enable) \ - CR_4(NV_PGRAPH_COLORKEYCOLOR0) + #define DIRTY_REGS \ + CR_1(NV_PGRAPH_COMBINECTL) \ + CR_1(NV_PGRAPH_SHADERCTL) \ + CR_1(NV_PGRAPH_SHADOWCTL) \ + CR_1(NV_PGRAPH_COMBINESPECFOG0) \ + CR_1(NV_PGRAPH_COMBINESPECFOG1) \ + CR_1(NV_PGRAPH_CONTROL_0) \ + CR_1(NV_PGRAPH_CONTROL_3) \ + CR_1(NV_PGRAPH_CSV0_C) \ + CR_1(NV_PGRAPH_CSV0_D) \ + CR_1(NV_PGRAPH_CSV1_A) \ + CR_1(NV_PGRAPH_CSV1_B) \ + CR_1(NV_PGRAPH_SETUPRASTER) \ + CR_1(NV_PGRAPH_SHADERPROG) \ + CR_1(NV_PGRAPH_ZCOMPRESSOCCLUDE) \ + CR_8(NV_PGRAPH_COMBINECOLORI0) \ + CR_8(NV_PGRAPH_COMBINECOLORO0) \ + CR_8(NV_PGRAPH_COMBINEALPHAI0) \ + CR_8(NV_PGRAPH_COMBINEALPHAO0) \ + CR_8(NV_PGRAPH_COMBINEFACTOR0) \ + CR_8(NV_PGRAPH_COMBINEFACTOR1) \ + CR_1(NV_PGRAPH_SHADERCLIPMODE) \ + CR_4(NV_PGRAPH_TEXCTL0_0) \ + CR_4(NV_PGRAPH_TEXFMT0) \ + CR_4(NV_PGRAPH_TEXFILTER0) \ + CR_8(NV_PGRAPH_WINDOWCLIPX0) \ + CR_8(NV_PGRAPH_WINDOWCLIPY0) \ + CF(pg->primitive_mode, primitive_mode) \ + CF(pg->surface_scale_factor, surface_scale_factor) \ + CF(pg->compressed_attrs, compressed_attrs) \ + CFA(pg->texture_matrix_enable, texture_matrix_enable) \ + CR_4(NV_PGRAPH_COLORKEYCOLOR0) -#define CR_x(reg, x) CR_x__define(reg, x) -#define CF_x(type, src, name, x) CF_x__define(type, src, name, x) + #define CR_x(reg, x) CR_x__define(reg, x) + #define CF_x(type, src, name, x) CF_x__define(type, src, name, x) DIRTY_REGS #undef CR_x #undef CF_x diff --git a/hw/xbox/nv2a/pgraph/glsl/psh.c b/hw/xbox/nv2a/pgraph/glsl/psh.c index 30ad62fed2..b15797e0b8 100644 --- a/hw/xbox/nv2a/pgraph/glsl/psh.c +++ b/hw/xbox/nv2a/pgraph/glsl/psh.c @@ -176,8 +176,8 @@ enum PS_DOTMAPPING enum PS_COLORKEYMODE { COLOR_KEY_NONE = 0, - COLOR_KEY_KILL_ALPHA = (1 << 0), - COLOR_KEY_KILL_COLOR_AND_ALPHA = (1 << 1), + COLOR_KEY_KILL_ALPHA = 0x01, + COLOR_KEY_KILL_COLOR_AND_ALPHA = 0x02, COLOR_KEY_DISCARD = 0x03, }; @@ -740,12 +740,17 @@ static void apply_convolution_filter(const struct PixelShader *ps, MString *vars static void define_colorkey_comparator(MString *preflight) { - mstring_append(preflight, - "bool check_color_key(vec4 texel, uint color_key) {\n" - " uvec4 components = uvec4(round(texel * 255.0));\n" - " return uint((components.a << 24) + (components.r << " - "16) + (components.g << 8) + components.b) == color_key;\n" - "}\n"); + // clang-format off + mstring_append( + preflight, + "bool check_color_key(vec4 texel, uint color_key, bool ignore_alpha) {\n" + " uvec4 components = uvec4(round(texel * 255.0));\n" + " if (ignore_alpha) {\n" + " return uint((components.r << 16) + (components.g << 8) + components.b) == (color_key & 0x00FFFFFFu);\n" + " }\n" + " return uint((components.a << 24) + (components.r << 16) + (components.g << 8) + components.b) == color_key;\n" + "}\n"); + // clang-format on } static MString* psh_convert(struct PixelShader *ps) @@ -771,8 +776,9 @@ static MString* psh_convert(struct PixelShader *ps) "%sivec4 clipRegion[8];\n" "%svec4 clipRange;\n" "%sfloat depthOffset;\n" - "%suint colorKey[4];\n", - u, u, u, u, u, u); + "%suint colorKey[4];\n" + "%sint colorKeyIgnoreAlpha[4];\n", + u, u, u, u, u, u, u); for (int i = 0; i < 4; i++) { mstring_append_fmt(preflight, "%smat2 bumpMat%d;\n" "%sfloat bumpScale%d;\n" @@ -1212,8 +1218,12 @@ static MString* psh_convert(struct PixelShader *ps) color_key_comparator_defined = true; } + // clang-format off mstring_append_fmt( - vars, "if (check_color_key(t%d, colorKey[%d])) {\n", i, i); + vars, + "if (check_color_key(t%d, colorKey[%d], colorKeyIgnoreAlpha[%d] != 0)) {\n", + i, i, i); + // clang-format on switch (color_key_mode) { case COLOR_KEY_DISCARD: diff --git a/hw/xbox/nv2a/pgraph/psh.h b/hw/xbox/nv2a/pgraph/psh.h index 8ec895aff9..6e4d5b1027 100644 --- a/hw/xbox/nv2a/pgraph/psh.h +++ b/hw/xbox/nv2a/pgraph/psh.h @@ -70,6 +70,7 @@ typedef struct PshState { bool compare_mode[4][4]; bool alphakill[4]; int colorkeymode[4]; + bool colorkey_ignore_alpha[4]; enum ConvolutionFilter conv_tex[4]; bool tex_x8y24[4]; int dim_tex[4]; diff --git a/hw/xbox/nv2a/pgraph/shaders.c b/hw/xbox/nv2a/pgraph/shaders.c index c9c68e31ec..87ef2f465c 100644 --- a/hw/xbox/nv2a/pgraph/shaders.c +++ b/hw/xbox/nv2a/pgraph/shaders.c @@ -24,6 +24,17 @@ #include "pgraph.h" #include "shaders.h" +// TODO: https://github.com/xemu-project/xemu/issues/2260 +// Investigate how color keying is handled for components with no alpha or +// only alpha. +static bool color_format_ignores_alpha(unsigned int color_format) +{ + return color_format == NV097_SET_TEXTURE_FORMAT_COLOR_SZ_X1R5G5B5 || + color_format == NV097_SET_TEXTURE_FORMAT_COLOR_SZ_X8R8G8B8 || + color_format == NV097_SET_TEXTURE_FORMAT_COLOR_LU_IMAGE_X1R5G5B5 || + color_format == NV097_SET_TEXTURE_FORMAT_COLOR_LU_IMAGE_X8R8G8B8; +} + ShaderState pgraph_get_shader_state(PGRAPHState *pg) { bool vertex_program = GET_MASK(pgraph_reg_r(pg, NV_PGRAPH_CSV0_D), @@ -229,6 +240,8 @@ ShaderState pgraph_get_shader_state(PGRAPHState *pg) state.psh.rect_tex[i] = f.linear; state.psh.tex_x8y24[i] = color_format == NV097_SET_TEXTURE_FORMAT_COLOR_LU_IMAGE_DEPTH_X8_Y24_FIXED || color_format == NV097_SET_TEXTURE_FORMAT_COLOR_LU_IMAGE_DEPTH_X8_Y24_FLOAT; + state.psh.colorkey_ignore_alpha[i] = + color_format_ignores_alpha(color_format); uint32_t border_source = GET_MASK(tex_fmt, NV_PGRAPH_TEXFMT0_BORDER_SOURCE); diff --git a/hw/xbox/nv2a/pgraph/vk/renderer.h b/hw/xbox/nv2a/pgraph/vk/renderer.h index 73cd94b37a..59eb59e50f 100644 --- a/hw/xbox/nv2a/pgraph/vk/renderer.h +++ b/hw/xbox/nv2a/pgraph/vk/renderer.h @@ -196,6 +196,7 @@ typedef struct ShaderBinding { int material_alpha_loc; int color_key_loc; + int color_key_ignore_alpha_loc; int uniform_attrs_loc; } ShaderBinding; diff --git a/hw/xbox/nv2a/pgraph/vk/shaders.c b/hw/xbox/nv2a/pgraph/vk/shaders.c index 5d54f869a1..5706b0c989 100644 --- a/hw/xbox/nv2a/pgraph/vk/shaders.c +++ b/hw/xbox/nv2a/pgraph/vk/shaders.c @@ -315,6 +315,9 @@ static void update_shader_constant_locations(ShaderBinding *binding) binding->color_key_loc = uniform_index(&binding->fragment->uniforms, "colorKey"); + binding->color_key_ignore_alpha_loc = + uniform_index(&binding->fragment->uniforms, "colorKeyIgnoreAlpha"); + binding->uniform_attrs_loc = uniform_index(&binding->vertex->uniforms, "inlineValue"); @@ -495,14 +498,19 @@ static void shader_update_constants(PGRAPHState *pg, ShaderBinding *binding, } if (binding->color_key_loc != -1) { uint32_t color_key_colors[4] = { - pg->regs_[NV_PGRAPH_COLORKEYCOLOR0], - pg->regs_[NV_PGRAPH_COLORKEYCOLOR1], - pg->regs_[NV_PGRAPH_COLORKEYCOLOR2], - pg->regs_[NV_PGRAPH_COLORKEYCOLOR3], + pgraph_reg_r(pg, NV_PGRAPH_COLORKEYCOLOR0), + pgraph_reg_r(pg, NV_PGRAPH_COLORKEYCOLOR1), + pgraph_reg_r(pg, NV_PGRAPH_COLORKEYCOLOR2), + pgraph_reg_r(pg, NV_PGRAPH_COLORKEYCOLOR3), }; uniform1uiv(&binding->fragment->uniforms, binding->color_key_loc, 4, color_key_colors); } + if (binding->color_key_ignore_alpha_loc != -1) { + uniform1iv(&binding->fragment->uniforms, + binding->color_key_ignore_alpha_loc, 4, + (int32_t *)&state->psh.colorkey_ignore_alpha); + } /* For each texture stage */ for (int i = 0; i < NV2A_MAX_TEXTURES; i++) {