vk: Integrate custom border colors when supported

rpcs3/Emu/RSX/VK/VKDraw.cpp

@@ -296,7 +296,7 @@ void VKGSRender::load_texture_env()
 				const auto wrap_s = vk::vk_wrap_mode(tex.wrap_s());
 				const auto wrap_t = vk::vk_wrap_mode(tex.wrap_t());
 				const auto wrap_r = vk::vk_wrap_mode(tex.wrap_r());
-				const auto border_color = vk::get_border_color(tex.border_color());
+				const auto border_color = vk::border_color_t(tex.border_color());
 
 				// Check if non-point filtering can even be used on this format
 				bool can_sample_linear;
@@ -427,7 +427,7 @@ void VKGSRender::load_texture_env()
 				const VkBool32 unnormalized_coords = !!(tex.format() & CELL_GCM_TEXTURE_UN);
 				const auto min_lod = tex.min_lod();
 				const auto max_lod = tex.max_lod();
-				const auto border_color = vk::get_border_color(tex.border_color());
+				const auto border_color = vk::border_color_t(tex.border_color());
 
 				if (vs_sampler_handles[i])
 				{

rpcs3/Emu/RSX/VK/VKFormats.cpp

@@ -78,17 +78,7 @@ namespace vk
 		}
 		default:
 		{
-			const auto color4 = rsx::decode_border_color(color);
+			return VK_BORDER_COLOR_FLOAT_CUSTOM_EXT;
-			if ((color4.r + color4.g + color4.b) > 1.35f)
-			{
-				//If color elements are brighter than roughly 0.5 average, use white border
-				return VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE;
-			}
-
-			if (color4.a > 0.5f)
-				return VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK;
-
-			return VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
 		}
 		}
 	}

rpcs3/Emu/RSX/VK/vkutils/sampler.cpp

@@ -4,9 +4,42 @@
 
 namespace vk
 {
+	extern VkBorderColor get_border_color(u32);
+
+	static VkBorderColor get_closest_border_color_enum(const color4f& color4)
+	{
+		if ((color4.r + color4.g + color4.b) > 1.35f)
+		{
+			//If color elements are brighter than roughly 0.5 average, use white border
+			return VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE;
+		}
+
+		if (color4.a > 0.5f)
+			return VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK;
+
+		return VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
+	}
+
+	border_color_t::border_color_t(u32 encoded_color)
+	{
+		value = vk::get_border_color(encoded_color);
+
+		if (value != VK_BORDER_COLOR_FLOAT_CUSTOM_EXT)
+		{
+			// Nothing to do
+			return;
+		}
+
+		color_value = rsx::decode_border_color(encoded_color);
+		if (!g_render_device->get_custom_border_color_support())
+		{
+			value = get_closest_border_color_enum(color_value);
+		}
+	}
+
 	sampler::sampler(const vk::render_device& dev, VkSamplerAddressMode clamp_u, VkSamplerAddressMode clamp_v, VkSamplerAddressMode clamp_w,
 		VkBool32 unnormalized_coordinates, float mipLodBias, float max_anisotropy, float min_lod, float max_lod,
-		VkFilter min_filter, VkFilter mag_filter, VkSamplerMipmapMode mipmap_mode, VkBorderColor border_color,
+		VkFilter min_filter, VkFilter mag_filter, VkSamplerMipmapMode mipmap_mode, const vk::border_color_t& border_color,
 		VkBool32 depth_compare, VkCompareOp depth_compare_mode)
 		: m_device(dev)
 	{
@@ -25,7 +58,20 @@ namespace vk
 		info.minFilter = min_filter;
 		info.mipmapMode = mipmap_mode;
 		info.compareOp = depth_compare_mode;
-		info.borderColor = border_color;
+		info.borderColor = border_color.value;
+
+		VkSamplerCustomBorderColorCreateInfoEXT custom_color_info;
+		if (border_color.value == VK_BORDER_COLOR_FLOAT_CUSTOM_EXT)
+		{
+			custom_color_info =
+			{
+				.sType = VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT,
+				.format = VK_FORMAT_UNDEFINED
+			};
+
+			std::memcpy(custom_color_info.customBorderColor.float32, border_color.color_value.rgba, sizeof(float) * 4);
+			info.pNext = &custom_color_info;
+		}
 
 		CHECK_RESULT(vkCreateSampler(m_device, &info, nullptr, &value));
 		vmm_notify_object_allocated(VMM_ALLOCATION_POOL_SAMPLER);
@@ -39,7 +85,7 @@ namespace vk
 
 	bool sampler::matches(VkSamplerAddressMode clamp_u, VkSamplerAddressMode clamp_v, VkSamplerAddressMode clamp_w,
 		VkBool32 unnormalized_coordinates, float mipLodBias, float max_anisotropy, float min_lod, float max_lod,
-		VkFilter min_filter, VkFilter mag_filter, VkSamplerMipmapMode mipmap_mode, VkBorderColor border_color,
+		VkFilter min_filter, VkFilter mag_filter, VkSamplerMipmapMode mipmap_mode, const vk::border_color_t& border_color,
 		VkBool32 depth_compare, VkCompareOp depth_compare_mode)
 	{
 		if (info.magFilter != mag_filter || info.minFilter != min_filter || info.mipmapMode != mipmap_mode ||

rpcs3/Emu/RSX/VK/vkutils/sampler.h

@@ -5,6 +5,29 @@
 
 namespace vk
 {
+	struct border_color_t
+	{
+		VkBorderColor value;
+		color4f color_value;
+
+		border_color_t(u32 encoded_color);
+
+		bool operator == (const border_color_t& that) const
+		{
+			if (this->value != that.value)
+			{
+				return false;
+			}
+
+			if (this->value != VK_BORDER_COLOR_FLOAT_CUSTOM_EXT)
+			{
+				return true;
+			}
+
+			return this->color_value == that.color_value;
+		}
+	};
+
 	struct sampler
 	{
 		VkSampler value;
@@ -12,14 +35,14 @@ namespace vk
 
 		sampler(const vk::render_device& dev, VkSamplerAddressMode clamp_u, VkSamplerAddressMode clamp_v, VkSamplerAddressMode clamp_w,
 			VkBool32 unnormalized_coordinates, float mipLodBias, float max_anisotropy, float min_lod, float max_lod,
-			VkFilter min_filter, VkFilter mag_filter, VkSamplerMipmapMode mipmap_mode, VkBorderColor border_color,
+			VkFilter min_filter, VkFilter mag_filter, VkSamplerMipmapMode mipmap_mode, const border_color_t& border_color,
 			VkBool32 depth_compare = false, VkCompareOp depth_compare_mode = VK_COMPARE_OP_NEVER);
 
 		~sampler();
 
 		bool matches(VkSamplerAddressMode clamp_u, VkSamplerAddressMode clamp_v, VkSamplerAddressMode clamp_w,
 			VkBool32 unnormalized_coordinates, float mipLodBias, float max_anisotropy, float min_lod, float max_lod,
-			VkFilter min_filter, VkFilter mag_filter, VkSamplerMipmapMode mipmap_mode, VkBorderColor border_color,
+			VkFilter min_filter, VkFilter mag_filter, VkSamplerMipmapMode mipmap_mode, const border_color_t& border_color,
 			VkBool32 depth_compare = false, VkCompareOp depth_compare_mode = VK_COMPARE_OP_NEVER);
 
 		sampler(const sampler&) = delete;