[GPU] Normalize sampler clamp modes

src/xenia/gpu/d3d12/d3d12_texture_cache.cc

@@ -965,10 +965,19 @@ D3D12TextureCache::SamplerParameters D3D12TextureCache::GetSamplerParameters(
 
   SamplerParameters parameters;
 
-  parameters.clamp_x = fetch.clamp_x;
-  parameters.clamp_y = fetch.clamp_y;
-  parameters.clamp_z = fetch.clamp_z;
-  parameters.border_color = fetch.border_color;
+  xenos::ClampMode fetch_clamp_x, fetch_clamp_y, fetch_clamp_z;
+  texture_util::GetClampModesForDimension(fetch, fetch_clamp_x, fetch_clamp_y,
+                                          fetch_clamp_z);
+  parameters.clamp_x = NormalizeClampMode(fetch_clamp_x);
+  parameters.clamp_y = NormalizeClampMode(fetch_clamp_y);
+  parameters.clamp_z = NormalizeClampMode(fetch_clamp_z);
+  if (xenos::ClampModeUsesBorder(parameters.clamp_x) ||
+      xenos::ClampModeUsesBorder(parameters.clamp_y) ||
+      xenos::ClampModeUsesBorder(parameters.clamp_z)) {
+    parameters.border_color = fetch.border_color;
+  } else {
+    parameters.border_color = xenos::BorderColor::k_ABGR_Black;
+  }
 
   uint32_t mip_min_level;
   texture_util::GetSubresourcesFromFetchConstant(fetch, nullptr, nullptr,
@@ -976,6 +985,7 @@ D3D12TextureCache::SamplerParameters D3D12TextureCache::GetSamplerParameters(
                                                  &mip_min_level, nullptr);
   parameters.mip_min_level = mip_min_level;
 
+  // TODO(Triang3l): Disable filtering for texture formats not supporting it.
   xenos::AnisoFilter aniso_filter =
       binding.aniso_filter == xenos::AnisoFilter::kUseFetchConst
           ? fetch.aniso_filter
@@ -2204,6 +2214,22 @@ D3D12_CPU_DESCRIPTOR_HANDLE D3D12TextureCache::GetTextureDescriptorCPUHandle(
   return provider.OffsetViewDescriptor(heap_start, heap_offset);
 }
 
+xenos::ClampMode D3D12TextureCache::NormalizeClampMode(
+    xenos::ClampMode clamp_mode) const {
+  if (clamp_mode == xenos::ClampMode::kClampToHalfway) {
+    // No GL_CLAMP (clamp to half edge, half border) equivalent in Direct3D 12,
+    // but there's no Direct3D 9 equivalent anyway, and too weird to be suitable
+    // for intentional real usage.
+    return xenos::ClampMode::kClampToEdge;
+  }
+  if (clamp_mode == xenos::ClampMode::kMirrorClampToHalfway ||
+      clamp_mode == xenos::ClampMode::kMirrorClampToBorder) {
+    // No Direct3D 12 equivalents.
+    return xenos::ClampMode::kMirrorClampToEdge;
+  }
+  return clamp_mode;
+}
+
 }  // namespace d3d12
 }  // namespace gpu
 }  // namespace xe

src/xenia/gpu/d3d12/d3d12_texture_cache.h

@@ -463,6 +463,8 @@ class D3D12TextureCache final : public TextureCache {
     return *scaled_resolve_buffer;
   }
 
+  xenos::ClampMode NormalizeClampMode(xenos::ClampMode clamp_mode) const;
+
   static const HostFormat host_formats_[64];
 
   D3D12CommandProcessor& command_processor_;

src/xenia/gpu/texture_util.cc

@@ -612,6 +612,29 @@ uint8_t SwizzleSigns(const xenos::xe_gpu_texture_fetch_t& fetch) {
   return signs;
 }
 
+void GetClampModesForDimension(const xenos::xe_gpu_texture_fetch_t& fetch,
+                               xenos::ClampMode& clamp_x_out,
+                               xenos::ClampMode& clamp_y_out,
+                               xenos::ClampMode& clamp_z_out) {
+  clamp_x_out = xenos::ClampMode::kClampToEdge;
+  clamp_y_out = xenos::ClampMode::kClampToEdge;
+  clamp_z_out = xenos::ClampMode::kClampToEdge;
+  switch (fetch.dimension) {
+    case xenos::DataDimension::k3D:
+      clamp_z_out = fetch.clamp_z;
+      [[fallthrough]];
+    case xenos::DataDimension::k2DOrStacked:
+      clamp_y_out = fetch.clamp_y;
+      [[fallthrough]];
+    case xenos::DataDimension::k1D:
+      clamp_x_out = fetch.clamp_x;
+      break;
+    default:
+      // Not applicable to cube textures.
+      break;
+  }
+}
+
 }  // namespace texture_util
 }  // namespace gpu
 }  // namespace xe

src/xenia/gpu/texture_util.h

@@ -334,6 +334,13 @@ constexpr bool IsAnySignSigned(uint8_t packed_signs) {
   return ((xor_signed | (xor_signed >> 1)) & 0b01010101) != 0b01010101;
 }
 
+// Returns normalized clamp modes specified in the fetch constant based on the
+// texture data dimension in it.
+void GetClampModesForDimension(const xenos::xe_gpu_texture_fetch_t& fetch,
+                               xenos::ClampMode& clamp_x_out,
+                               xenos::ClampMode& clamp_y_out,
+                               xenos::ClampMode& clamp_z_out);
+
 }  // namespace texture_util
 }  // namespace gpu
 }  // namespace xe

src/xenia/gpu/xenos.h

@@ -99,6 +99,11 @@ enum class ClampMode : uint32_t {
   kMirrorClampToBorder = 7,
 };
 
+constexpr bool ClampModeUsesBorder(ClampMode clamp_mode) {
+  return clamp_mode == ClampMode::kClampToBorder ||
+         clamp_mode == ClampMode::kMirrorClampToBorder;
+}
+
 // TEX_FORMAT_COMP, known as GPUSIGN on the Xbox 360.
 enum class TextureSign : uint32_t {
   kUnsigned = 0,