diff --git a/Source/Core/VideoBackends/D3D/PSTextureEncoder.cpp b/Source/Core/VideoBackends/D3D/PSTextureEncoder.cpp
index aaefb10e9c..e3eb0fd7d3 100644
--- a/Source/Core/VideoBackends/D3D/PSTextureEncoder.cpp
+++ b/Source/Core/VideoBackends/D3D/PSTextureEncoder.cpp
@@ -126,8 +126,11 @@ void PSTextureEncoder::Encode(u8* dst, u32 format, u32 native_width, u32 bytes_p
     D3D::context->UpdateSubresource(m_encodeParams, 0, nullptr, &params, 0, 0);
     D3D::stateman->SetPixelConstants(m_encodeParams);
 
-    // Use linear filtering if (bScaleByHalf), use point filtering otherwise
-    if (scaleByHalf)
+    // We also linear filtering for both box filtering and downsampling higher resolutions to 1x
+    // TODO: This only produces perfect downsampling for 1.5x and 2x IR, other resolution will
+    //       need more complex down filtering to average all pixels and produce the correct result.
+    // Also, box filtering won't be correct for anything other than 1x IR
+    if (scaleByHalf || g_ActiveConfig.iEFBScale != SCALE_1X)
       D3D::SetLinearCopySampler();
     else
       D3D::SetPointCopySampler();
diff --git a/Source/Core/VideoBackends/D3D12/PSTextureEncoder.cpp b/Source/Core/VideoBackends/D3D12/PSTextureEncoder.cpp
index d07c884e7d..25c5dd92c5 100644
--- a/Source/Core/VideoBackends/D3D12/PSTextureEncoder.cpp
+++ b/Source/Core/VideoBackends/D3D12/PSTextureEncoder.cpp
@@ -151,8 +151,11 @@ void PSTextureEncoder::Encode(u8* dst, u32 format, u32 native_width, u32 bytes_p
 
   D3D::command_list_mgr->SetCommandListDirtyState(COMMAND_LIST_STATE_PS_CBV, true);
 
-  // Use linear filtering if (bScaleByHalf), use point filtering otherwise
-  if (scale_by_half)
+  // We also linear filtering for both box filtering and downsampling higher resolutions to 1x
+  // TODO: This only produces perfect downsampling for 1.5x and 2x IR, other resolution will
+  //       need more complex down filtering to average all pixels and produce the correct result.
+  // Also, box filtering won't be correct for anything other than 1x IR
+  if (scale_by_half || g_ActiveConfig.iEFBScale != SCALE_1X)
     D3D::SetLinearCopySampler();
   else
     D3D::SetPointCopySampler();
diff --git a/Source/Core/VideoBackends/OGL/TextureConverter.cpp b/Source/Core/VideoBackends/OGL/TextureConverter.cpp
index 361a51c266..8ea8cd5d8b 100644
--- a/Source/Core/VideoBackends/OGL/TextureConverter.cpp
+++ b/Source/Core/VideoBackends/OGL/TextureConverter.cpp
@@ -227,7 +227,11 @@ static void EncodeToRamUsingShader(GLuint srcTexture, u8* destAddr, u32 dst_line
   glActiveTexture(GL_TEXTURE9);
   glBindTexture(GL_TEXTURE_2D_ARRAY, srcTexture);
 
-  if (linearFilter)
+  // We also linear filtering for both box filtering and downsampling higher resolutions to 1x
+  // TODO: This only produces perfect downsampling for 1.5x and 2x IR, other resolution will
+  //       need more complex down filtering to average all pixels and produce the correct result.
+  // Also, box filtering won't be correct for anything other than 1x IR
+  if (linearFilter || g_ActiveConfig.iEFBScale != SCALE_1X)
     g_sampler_cache->BindLinearSampler(9);
   else
     g_sampler_cache->BindNearestSampler(9);