rsx: Fix alpha ref

- The alpha ref register is compared directly to the ROP output register in realhw
- alpha ref content must match bit-width of ROP register, which means fp16 values are possible

rpcs3/Emu/RSX/RSXThread.cpp

@@ -742,7 +742,7 @@ namespace rsx
 	{
 		//TODO: Properly support alpha-to-coverage and alpha-to-one behavior in shaders
 		auto fragment_alpha_func = rsx::method_registers.alpha_func();
-		auto alpha_ref = rsx::method_registers.alpha_ref() / 255.f;
+		auto alpha_ref = rsx::method_registers.alpha_ref();
 		auto rop_control = rsx::method_registers.alpha_test_enabled()? 1u : 0u;
 
 		if (rsx::method_registers.msaa_alpha_to_coverage_enabled() && !backend_config.supports_hw_a2c)

rpcs3/Emu/RSX/rsx_decode.h

@@ -2521,15 +2521,26 @@ struct registers_decoder<NV4097_SET_ALPHA_REF>
 	public:
 		decoded_type(u32 value) : value(value) {}
 
-		u8 alpha_ref() const
+		f32 alpha_ref8() const
 		{
-			return bf_decoder<0, 8>(value);
+			return bf_decoder<0, 8>(value) / 255.f;
+		}
+
+		f32 alpha_ref16() const
+		{
+			return rsx::decode_fp16(bf_decoder<0, 16>(value));
+		}
+
+		f32 alpha_ref32() const
+		{
+			return std::bit_cast<f32>(value);
 		}
 	};
 
 	static std::string dump(decoded_type &&decoded_values)
 	{
-		return "Alpha: ref = " + std::to_string(decoded_values.alpha_ref());
+		return "Alpha: ref unorm8 = " + std::to_string(decoded_values.alpha_ref8()) +
+			" f16 = " + std::to_string(decoded_values.alpha_ref16());
 	}
 };
 

rpcs3/Emu/RSX/rsx_methods.h

@@ -654,7 +654,14 @@ namespace rsx
 
 		bool alpha_test_enabled() const
 		{
-			return decode<NV4097_SET_ALPHA_TEST_ENABLE>().alpha_test_enabled();
+			switch (surface_color())
+			{
+			case rsx::surface_color_format::x32:
+			case rsx::surface_color_format::w32z32y32x32:
+				return false;
+			default:
+				return decode<NV4097_SET_ALPHA_TEST_ENABLE>().alpha_test_enabled();
+			}
 		}
 
 		bool stencil_test_enabled() const
@@ -1104,9 +1111,18 @@ namespace rsx
 			return decode<NV4097_SET_POINT_SPRITE_CONTROL>().enabled();
 		}
 
-		u8 alpha_ref() const
+		f32 alpha_ref() const
 		{
-			return decode<NV4097_SET_ALPHA_REF>().alpha_ref();
+			switch (surface_color())
+			{
+			case rsx::surface_color_format::x32:
+			case rsx::surface_color_format::w32z32y32x32:
+				return decode<NV4097_SET_ALPHA_REF>().alpha_ref32();
+			case rsx::surface_color_format::w16z16y16x16:
+				return decode<NV4097_SET_ALPHA_REF>().alpha_ref16();
+			default:
+				return decode<NV4097_SET_ALPHA_REF>().alpha_ref8();
+			}
 		}
 
 		surface_target surface_color_target() const

rpcs3/Emu/RSX/rsx_utils.h

@@ -772,6 +772,44 @@ namespace rsx
 		return bits / To(1u << frac);
 	}
 
+	static inline f32 decode_fp16(u16 bits)
+	{
+		if (bits == 0)
+		{
+			return 0.f;
+		}
+
+		// Extract components
+		unsigned int sign = (bits >> 15) & 1;
+		unsigned int exp = (bits >> 10) & 0x1f;
+		unsigned int mantissa = bits & 0x3ff;
+
+		float base = (sign != 0) ? -1.f : 1.f;
+		float scale;
+
+		if (exp == 0x1F)
+		{
+			// specials (nan, inf)
+			u32 nan = 0x7F800000 | mantissa;
+			nan |= (sign << 31);
+			return std::bit_cast<f32>(nan);
+		}
+		else if (exp > 0)
+		{
+			// normal number, borrows a '1' from the hidden mantissa bit
+			base *= std::exp2f(f32(exp) - 15.f);
+			scale = (float(mantissa) / 1024.f) + 1.f;
+		}
+		else
+		{
+			// subnormal number, borrows a '0' from the hidden mantissa bit
+			base *= std::exp2f(1.f - 15.f);
+			scale = float(mantissa) / 1024.f;
+		}
+
+		return base * scale;
+	}
+
 	template <int N>
 	void unpack_bitset(const std::bitset<N>& block, u64* values)
 	{