diff --git a/Source/Core/VideoBackends/D3D/D3DState.cpp b/Source/Core/VideoBackends/D3D/D3DState.cpp
index f3aed1c1f2..90256dc5c9 100644
--- a/Source/Core/VideoBackends/D3D/D3DState.cpp
+++ b/Source/Core/VideoBackends/D3D/D3DState.cpp
@@ -344,10 +344,10 @@ ID3D11SamplerState* StateCache::Get(SamplerState state)
   sampdc.MinLOD = state.tm1.min_lod / 16.f;
   sampdc.MipLODBias = state.tm0.lod_bias / 256.f;
 
-  if (state.tm0.anisotropic_filtering)
+  if (state.tm0.anisotropic_filtering != 0)
   {
     sampdc.Filter = D3D11_FILTER_ANISOTROPIC;
-    sampdc.MaxAnisotropy = 1u << g_ActiveConfig.iMaxAnisotropy;
+    sampdc.MaxAnisotropy = 1u << state.tm0.anisotropic_filtering;
   }
 
   ComPtr<ID3D11SamplerState> res;
diff --git a/Source/Core/VideoBackends/D3D12/DescriptorHeapManager.cpp b/Source/Core/VideoBackends/D3D12/DescriptorHeapManager.cpp
index 9827057626..a8fb30e210 100644
--- a/Source/Core/VideoBackends/D3D12/DescriptorHeapManager.cpp
+++ b/Source/Core/VideoBackends/D3D12/DescriptorHeapManager.cpp
@@ -128,10 +128,10 @@ static void GetD3DSamplerDesc(D3D12_SAMPLER_DESC* desc, const SamplerState& stat
   desc->MipLODBias = static_cast<s32>(state.tm0.lod_bias) / 256.f;
   desc->ComparisonFunc = D3D12_COMPARISON_FUNC_NEVER;
 
-  if (state.tm0.anisotropic_filtering)
+  if (state.tm0.anisotropic_filtering != 0)
   {
     desc->Filter = D3D12_FILTER_ANISOTROPIC;
-    desc->MaxAnisotropy = 1u << g_ActiveConfig.iMaxAnisotropy;
+    desc->MaxAnisotropy = 1u << state.tm0.anisotropic_filtering;
   }
 }
 
diff --git a/Source/Core/VideoBackends/OGL/SamplerCache.cpp b/Source/Core/VideoBackends/OGL/SamplerCache.cpp
index 2673fd2ddd..0c344f4719 100644
--- a/Source/Core/VideoBackends/OGL/SamplerCache.cpp
+++ b/Source/Core/VideoBackends/OGL/SamplerCache.cpp
@@ -102,10 +102,10 @@ void SamplerCache::SetParameters(GLuint sampler_id, const SamplerState& params)
     glSamplerParameterf(sampler_id, GL_TEXTURE_LOD_BIAS, params.tm0.lod_bias / 256.f);
   }
 
-  if (params.tm0.anisotropic_filtering && g_ogl_config.bSupportsAniso)
+  if (params.tm0.anisotropic_filtering != 0 && g_ogl_config.bSupportsAniso)
   {
     glSamplerParameterf(sampler_id, GL_TEXTURE_MAX_ANISOTROPY_EXT,
-                        static_cast<float>(1 << g_ActiveConfig.iMaxAnisotropy));
+                        static_cast<float>(1 << params.tm0.anisotropic_filtering));
   }
 }
 
diff --git a/Source/Core/VideoBackends/Vulkan/ObjectCache.cpp b/Source/Core/VideoBackends/Vulkan/ObjectCache.cpp
index c32365f3fc..a9127070c7 100644
--- a/Source/Core/VideoBackends/Vulkan/ObjectCache.cpp
+++ b/Source/Core/VideoBackends/Vulkan/ObjectCache.cpp
@@ -398,11 +398,11 @@ VkSampler ObjectCache::GetSampler(const SamplerState& info)
   };
 
   // Can we use anisotropic filtering with this sampler?
-  if (info.tm0.anisotropic_filtering && g_vulkan_context->SupportsAnisotropicFiltering())
+  if (info.tm0.anisotropic_filtering != 0 && g_vulkan_context->SupportsAnisotropicFiltering())
   {
     // Cap anisotropy to device limits.
     create_info.anisotropyEnable = VK_TRUE;
-    create_info.maxAnisotropy = std::min(static_cast<float>(1 << g_ActiveConfig.iMaxAnisotropy),
+    create_info.maxAnisotropy = std::min(static_cast<float>(1 << info.tm0.anisotropic_filtering),
                                          g_vulkan_context->GetMaxSamplerAnisotropy());
   }
 
diff --git a/Source/Core/VideoCommon/RenderState.cpp b/Source/Core/VideoCommon/RenderState.cpp
index 89f36f2b18..d9551b4da9 100644
--- a/Source/Core/VideoCommon/RenderState.cpp
+++ b/Source/Core/VideoCommon/RenderState.cpp
@@ -309,10 +309,15 @@ void SamplerState::Generate(const BPMemory& bp, u32 index)
   };
   tm0.wrap_u = filter_invalid_wrap(bp_tm0.wrap_s);
   tm0.wrap_v = filter_invalid_wrap(bp_tm0.wrap_t);
+  if (bp_tm0.max_aniso == MaxAniso::Two)
+    tm0.anisotropic_filtering = 1;
+  else if (bp_tm0.max_aniso == MaxAniso::Four)
+    tm0.anisotropic_filtering = 2;
+  else
+    tm0.anisotropic_filtering = 0;
 
   tm0.diag_lod = bp_tm0.diag_lod;
-  tm0.anisotropic_filtering = false;  // TODO: Respect BP anisotropic filtering mode
-  tm0.lod_clamp = bp_tm0.lod_clamp;   // TODO: What does this do?
+  tm0.lod_clamp = bp_tm0.lod_clamp;  // TODO: What does this do?
 }
 
 namespace RenderState
diff --git a/Source/Core/VideoCommon/RenderState.h b/Source/Core/VideoCommon/RenderState.h
index 4a682b1a0f..5ce468463a 100644
--- a/Source/Core/VideoCommon/RenderState.h
+++ b/Source/Core/VideoCommon/RenderState.h
@@ -205,7 +205,7 @@ struct SamplerState
     BitField<7, 1, LODType> diag_lod;
     BitField<8, 16, s32> lod_bias;         // multiplied by 256, higher precision than normal
     BitField<24, 1, bool, u32> lod_clamp;  // TODO: This isn't currently implemented
-    BitField<25, 1, bool, u32> anisotropic_filtering;  // TODO: This doesn't use the BP one yet
+    BitField<25, 4, u32> anisotropic_filtering;
     u32 hex = 0;
   };
   union TM1
diff --git a/Source/Core/VideoCommon/TextureCacheBase.cpp b/Source/Core/VideoCommon/TextureCacheBase.cpp
index b99533eb02..871bd1780c 100644
--- a/Source/Core/VideoCommon/TextureCacheBase.cpp
+++ b/Source/Core/VideoCommon/TextureCacheBase.cpp
@@ -1053,11 +1053,7 @@ SamplerState TextureCacheBase::GetSamplerState(u32 index, float custom_tex_scale
     state.tm0.mag_filter = FilterMode::Linear;
     if (tm0.mipmap_filter != MipMode::None)
       state.tm0.mipmap_filter = FilterMode::Linear;
-    state.tm0.anisotropic_filtering = true;
-  }
-  else
-  {
-    state.tm0.anisotropic_filtering = false;
+    state.tm0.anisotropic_filtering = g_ActiveConfig.iMaxAnisotropy;
   }
 
   if (has_arbitrary_mips && tm0.mipmap_filter != MipMode::None)
@@ -1068,9 +1064,6 @@ SamplerState TextureCacheBase::GetSamplerState(u32 index, float custom_tex_scale
     // Correct this with the upscaling factor of custom textures.
     s32 lod_offset = std::log2(g_framebuffer_manager->GetEFBScale() / custom_tex_scale) * 256.f;
     state.tm0.lod_bias = std::clamp<s32>(state.tm0.lod_bias + lod_offset, -32768, 32767);
-
-    // Anisotropic also pushes mips farther away so it cannot be used either
-    state.tm0.anisotropic_filtering = false;
   }
 
   return state;