nv2a: Handle PGRAPH color keying

hw/xbox/nv2a/nv2a_regs.h

@@ -409,6 +409,10 @@
 #       define NV_PGRAPH_CLEARRECTY_YMIN                          0x00000FFF
 #       define NV_PGRAPH_CLEARRECTY_YMAX                          0x0FFF0000
 #define NV_PGRAPH_COLORCLEARVALUE                        0x0000186C
+#define NV_PGRAPH_COLORKEYCOLOR0                         0x00001870
+#define NV_PGRAPH_COLORKEYCOLOR1                         0x00001874
+#define NV_PGRAPH_COLORKEYCOLOR2                         0x00001878
+#define NV_PGRAPH_COLORKEYCOLOR3                         0x0000187C
 #define NV_PGRAPH_COMBINEFACTOR0                         0x00001880
 #define NV_PGRAPH_COMBINEFACTOR1                         0x000018A0
 #define NV_PGRAPH_COMBINEALPHAI0                         0x000018C0
@@ -533,6 +537,7 @@
 #define NV_PGRAPH_TEXADDRESS2                            0x000019C4
 #define NV_PGRAPH_TEXADDRESS3                            0x000019C8
 #define NV_PGRAPH_TEXCTL0_0                              0x000019CC
+#   define NV_PGRAPH_TEXCTL0_0_COLORKEYMODE                     0x03
 #   define NV_PGRAPH_TEXCTL0_0_ALPHAKILLEN                      (1 << 2)
 #   define NV_PGRAPH_TEXCTL0_0_MAX_LOD_CLAMP                    0x0003FFC0
 #   define NV_PGRAPH_TEXCTL0_0_MIN_LOD_CLAMP                    0x3FFC0000
@@ -1065,6 +1070,7 @@
 #   define NV097_SET_COMBINER_FACTOR1                         0x00000A80
 #   define NV097_SET_COMBINER_ALPHA_OCW                       0x00000AA0
 #   define NV097_SET_COMBINER_COLOR_ICW                       0x00000AC0
+#   define NV097_SET_COLOR_KEY_COLOR                          0x00000AE0
 #   define NV097_SET_VIEWPORT_SCALE                           0x00000AF0
 #   define NV097_SET_TRANSFORM_PROGRAM                        0x00000B00
 #   define NV097_SET_TRANSFORM_CONSTANT                       0x00000B80

hw/xbox/nv2a/pgraph/gl/renderer.h

@@ -127,6 +127,9 @@ typedef struct ShaderBinding {
 
     GLint point_params_loc[8];
     GLint material_alpha_loc;
+
+    GLint color_key_loc[4];
+    GLint color_key_mask_loc[4];
 } ShaderBinding;
 
 typedef struct VertexKey {

hw/xbox/nv2a/pgraph/gl/shaders.c

@@ -133,6 +133,7 @@ static void update_shader_constant_locations(ShaderBinding *binding)
         }
     }
     binding->alpha_ref_loc = glGetUniformLocation(binding->gl_program, "alphaRef");
+
     for (int i = 1; i < NV2A_MAX_TEXTURES; i++) {
         snprintf(tmp, sizeof(tmp), "bumpMat%d", i);
         binding->bump_mat_loc[i] = glGetUniformLocation(binding->gl_program, tmp);
@@ -203,6 +204,16 @@ static void update_shader_constant_locations(ShaderBinding *binding)
         binding->material_alpha_loc = -1;
         binding->specular_power_loc = -1;
     }
+
+    for (int i = 0; i < 4; ++i) {
+        snprintf(tmp, sizeof(tmp), "colorKey[%d]", i);
+        binding->color_key_loc[i] =
+            glGetUniformLocation(binding->gl_program, tmp);
+
+        snprintf(tmp, sizeof(tmp), "colorKeyMask[%d]", i);
+        binding->color_key_mask_loc[i] =
+            glGetUniformLocation(binding->gl_program, tmp);
+    }
 }
 
 static void generate_shaders(ShaderBinding *binding)
@@ -696,6 +707,7 @@ static void shader_update_constants(PGRAPHState *pg, ShaderBinding *binding,
                                     bool binding_changed)
 {
     PGRAPHGLState *r = pg->gl_renderer_state;
+    ShaderState *state = &binding->state;
     int i, j;
 
     /* update combiner constants */
@@ -725,7 +737,6 @@ static void shader_update_constants(PGRAPHState *pg, ShaderBinding *binding,
         glUniform1i(binding->alpha_ref_loc, alpha_ref);
     }
 
-
     /* For each texture stage */
     for (i = 0; i < NV2A_MAX_TEXTURES; i++) {
         GLint loc;
@@ -765,6 +776,15 @@ static void shader_update_constants(PGRAPHState *pg, ShaderBinding *binding,
             assert(r->texture_binding[i] != NULL);
             glUniform1f(loc, (float)r->texture_binding[i]->scale);
         }
+
+        if (binding->color_key_loc[i] != -1) {
+            glUniform1ui(binding->color_key_loc[i],
+                         pgraph_reg_r(pg, NV_PGRAPH_COLORKEYCOLOR0 + i * 4));
+        }
+        if (binding->color_key_mask_loc[i] != -1) {
+            glUniform1ui(binding->color_key_mask_loc[i],
+                         state->psh.colorkey_mask[i]);
+        }
     }
 
     if (binding->fog_color_loc != -1) {
@@ -794,7 +814,7 @@ static void shader_update_constants(PGRAPHState *pg, ShaderBinding *binding,
         assert(0);
     }
 
-    if (binding->state.fixed_function) {
+    if (state->fixed_function) {
         /* update lighting constants */
         struct {
             uint32_t* v;
@@ -990,7 +1010,8 @@ static bool test_shaders_dirty(PGRAPHState *pg)
         CF(pg->primitive_mode, primitive_mode) \
         CF(pg->surface_scale_factor, surface_scale_factor) \
         CF(pg->compressed_attrs, compressed_attrs) \
-        CFA(pg->texture_matrix_enable, texture_matrix_enable)
+        CFA(pg->texture_matrix_enable, texture_matrix_enable) \
+        CR_4(NV_PGRAPH_COLORKEYCOLOR0)
 
     #define CR_x(reg, x) CR_x__define(reg, x)
     #define CF_x(type, src, name, x) CF_x__define(type, src, name, x)

hw/xbox/nv2a/pgraph/glsl/psh.c

@@ -174,6 +174,13 @@ enum PS_DOTMAPPING
     PS_DOTMAPPING_HILO_HEMISPHERE=     0x07L, // - * * *
 };
 
+enum PS_COLORKEYMODE {
+    COLOR_KEY_NONE = 0,
+    COLOR_KEY_KILL_ALPHA = 1,
+    COLOR_KEY_KILL_COLOR_AND_ALPHA = 2,
+    COLOR_KEY_DISCARD = 3,
+};
+
 
 // Structures to describe the PS definition
 
@@ -731,6 +738,19 @@ static void apply_convolution_filter(const struct PixelShader *ps, MString *vars
         "}\n", tex, tex, tex, tex, tex_remap, tex);
 }
 
+static void define_colorkey_comparator(MString *preflight)
+{
+    // clang-format off
+    mstring_append(
+        preflight,
+        "bool check_color_key(vec4 texel, uint color_key, uint color_key_mask) {\n"
+        "    uvec4 c = uvec4(texel * 255.0 + 0.5);\n"
+        "    uint color = (c.a << 24) | (c.r << 16) | (c.g << 8) | c.b;\n"
+        "    return (color & color_key_mask) == (color_key & color_key_mask);\n"
+        "}\n");
+    // clang-format on
+}
+
 static MString* psh_convert(struct PixelShader *ps)
 {
     const char *u = ps->state.vulkan ? "" : "uniform "; // FIXME: Remove
@@ -748,12 +768,15 @@ static MString* psh_convert(struct PixelShader *ps)
                            "layout(location = 0) out vec4 fragColor;\n");
     }
 
-    mstring_append_fmt(preflight, "%sint alphaRef;\n"
+    mstring_append_fmt(preflight,
-                                  "%svec4  fogColor;\n"
+                       "%sint alphaRef;\n"
-                                  "%sivec4 clipRegion[8];\n"
+                       "%svec4  fogColor;\n"
-                                  "%svec4  clipRange;\n"
+                       "%sivec4 clipRegion[8];\n"
-                                  "%sfloat depthOffset;\n",
+                       "%svec4  clipRange;\n"
-                                  u, u, u, u, u);
+                       "%sfloat depthOffset;\n"
+                       "%suint colorKey[4];\n"
+                       "%suint colorKeyMask[4];\n",
+                       u, u, u, u, u, u, u);
     for (int i = 0; i < 4; i++) {
         mstring_append_fmt(preflight, "%smat2  bumpMat%d;\n"
                                       "%sfloat bumpScale%d;\n"
@@ -934,6 +957,8 @@ static MString* psh_convert(struct PixelShader *ps)
 
     ps->code = mstring_new();
 
+    bool color_key_comparator_defined = false;
+
     for (int i = 0; i < 4; i++) {
 
         const char *sampler_type = get_sampler_type(ps->tex_modes[i], &ps->state, i);
@@ -1184,6 +1209,40 @@ static MString* psh_convert(struct PixelShader *ps)
                                    i);
             }
 
+            enum PS_COLORKEYMODE color_key_mode = ps->state.colorkey_mode[i];
+            if (color_key_mode != COLOR_KEY_NONE) {
+                if (!color_key_comparator_defined) {
+                    define_colorkey_comparator(preflight);
+                    color_key_comparator_defined = true;
+                }
+
+                // clang-format off
+                mstring_append_fmt(
+                    vars,
+                    "if (check_color_key(t%d, colorKey[%d], colorKeyMask[%d])) {\n",
+                    i, i, i);
+                // clang-format on
+
+                switch (color_key_mode) {
+                case COLOR_KEY_DISCARD:
+                    mstring_append(vars, "  discard;\n");
+                    break;
+
+                case COLOR_KEY_KILL_ALPHA:
+                    mstring_append_fmt(vars, "  t%d.a = 0.0;\n", i);
+                    break;
+
+                case COLOR_KEY_KILL_COLOR_AND_ALPHA:
+                    mstring_append_fmt(vars, "  t%d = vec4(0.0);\n", i);
+                    break;
+
+                default:
+                    assert(!"Unhandled key mode.");
+                }
+
+                mstring_append(vars, "}\n");
+            }
+
             if (ps->state.rect_tex[i]) {
                 mstring_append_fmt(preflight,
                 "vec2 norm%d(vec2 coord) {\n"

hw/xbox/nv2a/pgraph/methods.h.inc

@@ -105,6 +105,7 @@ DEF_METHOD_RANGE(NV097, SET_COMBINER_FACTOR0, 8)
 DEF_METHOD_RANGE(NV097, SET_COMBINER_FACTOR1, 8)
 DEF_METHOD_RANGE(NV097, SET_COMBINER_ALPHA_OCW, 8)
 DEF_METHOD_RANGE(NV097, SET_COMBINER_COLOR_ICW, 8)
+DEF_METHOD_RANGE(NV097, SET_COLOR_KEY_COLOR, 4)
 DEF_METHOD_RANGE(NV097, SET_VIEWPORT_SCALE, 4)
 DEF_METHOD_RANGE(NV097, SET_TRANSFORM_PROGRAM, 32)
 DEF_METHOD_RANGE(NV097, SET_TRANSFORM_CONSTANT, 32)

hw/xbox/nv2a/pgraph/pgraph.c

@@ -1966,6 +1966,12 @@ DEF_METHOD_INC(NV097, SET_COMBINER_COLOR_ICW)
     pgraph_reg_w(pg, NV_PGRAPH_COMBINECOLORI0 + slot*4, parameter);
 }
 
+DEF_METHOD_INC(NV097, SET_COLOR_KEY_COLOR)
+{
+    int slot = (method - NV097_SET_COLOR_KEY_COLOR) / 4;
+    pgraph_reg_w(pg, NV_PGRAPH_COLORKEYCOLOR0 + slot * 4, parameter);
+}
+
 DEF_METHOD_INC(NV097, SET_VIEWPORT_SCALE)
 {
     int slot = (method - NV097_SET_VIEWPORT_SCALE) / 4;

hw/xbox/nv2a/pgraph/psh.h

@@ -69,6 +69,8 @@ typedef struct PshState {
     bool snorm_tex[4];
     bool compare_mode[4][4];
     bool alphakill[4];
+    int colorkey_mode[4];
+    uint32_t colorkey_mask[4];
     enum ConvolutionFilter conv_tex[4];
     bool tex_x8y24[4];
     int dim_tex[4];

hw/xbox/nv2a/pgraph/shaders.c

@@ -24,6 +24,23 @@
 #include "pgraph.h"
 #include "shaders.h"
 
+// TODO: https://github.com/xemu-project/xemu/issues/2260
+//   Investigate how color keying is handled for components with no alpha or
+//   only alpha.
+static uint32_t get_colorkey_mask(unsigned int color_format)
+{
+    switch (color_format) {
+    case NV097_SET_TEXTURE_FORMAT_COLOR_SZ_X1R5G5B5:
+    case NV097_SET_TEXTURE_FORMAT_COLOR_SZ_X8R8G8B8:
+    case NV097_SET_TEXTURE_FORMAT_COLOR_LU_IMAGE_X1R5G5B5:
+    case NV097_SET_TEXTURE_FORMAT_COLOR_LU_IMAGE_X8R8G8B8:
+        return 0x00FFFFFF;
+
+    default:
+        return 0xFFFFFFFF;
+    }
+}
+
 ShaderState pgraph_get_shader_state(PGRAPHState *pg)
 {
     bool vertex_program = GET_MASK(pgraph_reg_r(pg, NV_PGRAPH_CSV0_D),
@@ -219,6 +236,7 @@ ShaderState pgraph_get_shader_state(PGRAPHState *pg)
         }
 
         state.psh.alphakill[i] = ctl_0 & NV_PGRAPH_TEXCTL0_0_ALPHAKILLEN;
+        state.psh.colorkey_mode[i] = ctl_0 & NV_PGRAPH_TEXCTL0_0_COLORKEYMODE;
 
         uint32_t tex_fmt = pgraph_reg_r(pg, NV_PGRAPH_TEXFMT0 + i * 4);
         state.psh.dim_tex[i] = GET_MASK(tex_fmt, NV_PGRAPH_TEXFMT0_DIMENSIONALITY);
@@ -228,6 +246,7 @@ ShaderState pgraph_get_shader_state(PGRAPHState *pg)
         state.psh.rect_tex[i] = f.linear;
         state.psh.tex_x8y24[i] = color_format == NV097_SET_TEXTURE_FORMAT_COLOR_LU_IMAGE_DEPTH_X8_Y24_FIXED ||
                                 color_format == NV097_SET_TEXTURE_FORMAT_COLOR_LU_IMAGE_DEPTH_X8_Y24_FLOAT;
+        state.psh.colorkey_mask[i] = get_colorkey_mask(color_format);
 
         uint32_t border_source =
             GET_MASK(tex_fmt, NV_PGRAPH_TEXFMT0_BORDER_SOURCE);

hw/xbox/nv2a/pgraph/vk/glsl.h

@@ -202,4 +202,10 @@ void uniform4i(ShaderUniformLayout *layout, int idx, int v0, int v1, int v2, int
 	uniform1iv(layout, idx, 4, values);
 }
 
+static inline void uniform1uiv(ShaderUniformLayout *layout, int idx,
+                               size_t count, uint32_t *values)
+{
+    uniform_copy(layout, idx, values, sizeof(uint32_t), count);
+}
+
 #endif

hw/xbox/nv2a/pgraph/vk/renderer.h

@@ -195,6 +195,9 @@ typedef struct ShaderBinding {
     int clip_region_loc;
     int material_alpha_loc;
 
+    int color_key_loc;
+    int color_key_mask_loc;
+
     int uniform_attrs_loc;
 } ShaderBinding;
 

hw/xbox/nv2a/pgraph/vk/shaders.c

@@ -312,6 +312,12 @@ static void update_shader_constant_locations(ShaderBinding *binding)
     binding->material_alpha_loc =
         uniform_index(&binding->vertex->uniforms, "material_alpha");
 
+    binding->color_key_loc =
+        uniform_index(&binding->fragment->uniforms, "colorKey");
+
+    binding->color_key_mask_loc =
+        uniform_index(&binding->fragment->uniforms, "colorKeyMask");
+
     binding->uniform_attrs_loc =
         uniform_index(&binding->vertex->uniforms, "inlineValue");
 
@@ -490,7 +496,20 @@ static void shader_update_constants(PGRAPHState *pg, ShaderBinding *binding,
         uniform1i(&binding->fragment->uniforms, binding->alpha_ref_loc,
                          alpha_ref);
     }
-
+    if (binding->color_key_loc != -1) {
+        uint32_t color_key_colors[4] = {
+            pgraph_reg_r(pg, NV_PGRAPH_COLORKEYCOLOR0),
+            pgraph_reg_r(pg, NV_PGRAPH_COLORKEYCOLOR1),
+            pgraph_reg_r(pg, NV_PGRAPH_COLORKEYCOLOR2),
+            pgraph_reg_r(pg, NV_PGRAPH_COLORKEYCOLOR3),
+        };
+        uniform1uiv(&binding->fragment->uniforms, binding->color_key_loc, 4,
+                    color_key_colors);
+    }
+    if (binding->color_key_mask_loc != -1) {
+        uniform1uiv(&binding->fragment->uniforms, binding->color_key_mask_loc,
+                    4, state->psh.colorkey_mask);
+    }
 
     /* For each texture stage */
     for (int i = 0; i < NV2A_MAX_TEXTURES; i++) {