GS/HW: Correct min lod/minification filter for mipmaps

pcsx2/GS/Renderers/Common/GSDevice.h

@@ -288,6 +288,43 @@ struct alignas(16) GSHWDrawConfig
 			out.biln = 1;
 			return out;
 		}
+
+		/// Returns true if the effective minification filter is linear.
+		__fi bool IsMinFilterLinear() const
+		{
+			if (triln < static_cast<u8>(GS_MIN_FILTER::Nearest_Mipmap_Nearest))
+			{
+				// use the same filter as mag when mipmapping is off
+				return biln;
+			}
+			else
+			{
+				// Linear_Mipmap_Nearest or Linear_Mipmap_Linear
+				return (triln >= static_cast<u8>(GS_MIN_FILTER::Linear_Mipmap_Nearest));
+			}
+		}
+
+		/// Returns true if the effective magnification filter is linear.
+		__fi bool IsMagFilterLinear() const
+		{
+			// magnification uses biln regardless of mip mode (they're only used for minification)
+			return biln;
+		}
+
+		/// Returns true if the effective mipmap filter is linear.
+		__fi bool IsMipFilterLinear() const
+		{
+			return (triln == static_cast<u8>(GS_MIN_FILTER::Nearest_Mipmap_Linear) ||
+					triln == static_cast<u8>(GS_MIN_FILTER::Linear_Mipmap_Linear));
+		}
+
+		/// Returns the maximum LOD for this sampler (0 if mipmapping is disabled).
+		__fi float GetMaxLOD() const
+		{
+			// See https://www.khronos.org/registry/vulkan/specs/1.2-extensions/man/html/VkSamplerCreateInfo.html#_description
+			// for the reasoning behind 0.25f here.
+			return triln >= static_cast<u8>(GS_MIN_FILTER::Nearest_Mipmap_Nearest) ? 1000.0f : 0.25f;
+		}
 	};
 	struct DepthStencilSelector
 	{

pcsx2/GS/Renderers/DX11/GSTextureFX11.cpp

@@ -221,23 +221,37 @@ void GSDevice11::SetupPS(PSSelector sel, const GSHWDrawConfig::PSConstantBuffer*
 		}
 		else
 		{
-			D3D11_SAMPLER_DESC sd;
+			D3D11_SAMPLER_DESC sd = {};
-
-			memset(&sd, 0, sizeof(sd));
 
 			const int anisotropy = GSConfig.MaxAnisotropy;
 			if (anisotropy && ssel.aniso)
+			{
 				sd.Filter = D3D11_FILTER_ANISOTROPIC;
-			else if (ssel.biln)
+			}
-				sd.Filter = D3D11_FILTER_MIN_MAG_LINEAR_MIP_POINT;
 			else
-				sd.Filter = D3D11_FILTER_MIN_MAG_MIP_POINT;
+			{
+				static constexpr std::array<D3D11_FILTER, 8> filters = {{
+					D3D11_FILTER_MIN_MAG_MIP_POINT, // 000 / min=point,mag=point,mip=point
+					D3D11_FILTER_MIN_LINEAR_MAG_MIP_POINT, // 001 / min=linear,mag=point,mip=point
+					D3D11_FILTER_MIN_POINT_MAG_LINEAR_MIP_POINT, // 010 / min=point,mag=linear,mip=point
+					D3D11_FILTER_MIN_MAG_LINEAR_MIP_POINT, // 011 / min=linear,mag=linear,mip=point
+					D3D11_FILTER_MIN_MAG_POINT_MIP_LINEAR, // 100 / min=point,mag=point,mip=linear
+					D3D11_FILTER_MIN_LINEAR_MAG_POINT_MIP_LINEAR, // 101 / min=linear,mag=point,mip=linear
+					D3D11_FILTER_MIN_POINT_MAG_MIP_LINEAR, // 110 / min=point,mag=linear,mip=linear
+					D3D11_FILTER_MIN_MAG_MIP_LINEAR, // 111 / min=linear,mag=linear,mip=linear
+				}};
+
+				const u8 index = (static_cast<u8>(ssel.IsMipFilterLinear()) << 2) |
+								 (static_cast<u8>(ssel.IsMagFilterLinear()) << 1) |
+								 static_cast<u8>(ssel.IsMinFilterLinear());
+				sd.Filter = filters[index];
+			}
 
 			sd.AddressU = ssel.tau ? D3D11_TEXTURE_ADDRESS_WRAP : D3D11_TEXTURE_ADDRESS_CLAMP;
 			sd.AddressV = ssel.tav ? D3D11_TEXTURE_ADDRESS_WRAP : D3D11_TEXTURE_ADDRESS_CLAMP;
 			sd.AddressW = D3D11_TEXTURE_ADDRESS_CLAMP;
-			sd.MinLOD = -FLT_MAX;
+			sd.MinLOD = 0.0f;
-			sd.MaxLOD = FLT_MAX;
+			sd.MaxLOD = ssel.GetMaxLOD();
 			sd.MaxAnisotropy = std::clamp(anisotropy, 1, 16);
 			sd.ComparisonFunc = D3D11_COMPARISON_NEVER;
 

pcsx2/GS/Renderers/Vulkan/GSDeviceVK.cpp

@@ -952,20 +952,11 @@ VkSampler GSDeviceVK::GetSampler(GSHWDrawConfig::SamplerSelector ss)
 
 	const bool aniso = (ss.aniso && GSConfig.MaxAnisotropy > 1);
 
-	static constexpr std::array<VkSamplerMipmapMode, 6> mipmap_modes = {{
-		VK_SAMPLER_MIPMAP_MODE_NEAREST, // Nearest
-		VK_SAMPLER_MIPMAP_MODE_NEAREST, // Linear
-		VK_SAMPLER_MIPMAP_MODE_NEAREST, // Nearest_Mipmap_Nearest
-		VK_SAMPLER_MIPMAP_MODE_LINEAR, // Nearest_Mipmap_Linear
-		VK_SAMPLER_MIPMAP_MODE_NEAREST, // Linear_Mipmap_Nearest
-		VK_SAMPLER_MIPMAP_MODE_LINEAR, // Linear_Mipmap_Linear
-	}};
-
 	const VkSamplerCreateInfo ci = {
 		VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO, nullptr, 0,
-		ss.biln ? VK_FILTER_LINEAR : VK_FILTER_NEAREST, // min
+		ss.IsMinFilterLinear() ? VK_FILTER_LINEAR : VK_FILTER_NEAREST, // min
-		ss.biln ? VK_FILTER_LINEAR : VK_FILTER_NEAREST, // max
+		ss.IsMagFilterLinear() ? VK_FILTER_LINEAR : VK_FILTER_NEAREST, // mag
-		mipmap_modes[ss.triln], // mip
+		ss.IsMipFilterLinear() ? VK_SAMPLER_MIPMAP_MODE_LINEAR : VK_SAMPLER_MIPMAP_MODE_NEAREST, // mip
 		static_cast<VkSamplerAddressMode>(
 			ss.tau ? VK_SAMPLER_ADDRESS_MODE_REPEAT : VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE), // u
 		static_cast<VkSamplerAddressMode>(
@@ -976,8 +967,8 @@ VkSampler GSDeviceVK::GetSampler(GSHWDrawConfig::SamplerSelector ss)
 		aniso ? static_cast<float>(GSConfig.MaxAnisotropy) : 1.0f, // anisotropy
 		VK_FALSE, // compare enable
 		VK_COMPARE_OP_ALWAYS, // compare op
-		-1000.0f, // min lod
+		0.0f, // min lod
-		(ss.triln >= static_cast<u8>(GS_MIN_FILTER::Nearest_Mipmap_Nearest)) ? 1000.0f : 0.0f, // max lod
+		ss.GetMaxLOD(), // max lod
 		VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK, // border
 		VK_FALSE // unnormalized coordinates
 	};