diff --git a/Source/Core/VideoBackends/D3D/Render.cpp b/Source/Core/VideoBackends/D3D/Render.cpp
index 23f9ceae2d..c7204257bd 100644
--- a/Source/Core/VideoBackends/D3D/Render.cpp
+++ b/Source/Core/VideoBackends/D3D/Render.cpp
@@ -1154,7 +1154,7 @@ void Renderer::SetDitherMode()
 	// TODO: Set dither mode to bpmem.blendmode.dither
 }
 
-void Renderer::SetSamplerState(int stage, int texindex)
+void Renderer::SetSamplerState(int stage, int texindex, bool custom_tex)
 {
 	const FourTexUnits &tex = bpmem.tex[texindex];
 	const TexMode0 &tm0 = tex.texMode0[stage];
@@ -1179,6 +1179,12 @@ void Renderer::SetSamplerState(int stage, int texindex)
 	gx_state.sampler[stage].max_lod = (u32)tm1.max_lod;
 	gx_state.sampler[stage].min_lod = (u32)tm1.min_lod;
 	gx_state.sampler[stage].lod_bias = (s32)tm0.lod_bias;
+
+	// custom textures may have higher resolution, so disable the max_lod
+	if (custom_tex)
+	{
+		gx_state.sampler[stage].max_lod = 255;
+	}
 }
 
 void Renderer::SetInterlacingMode()
diff --git a/Source/Core/VideoBackends/D3D/Render.h b/Source/Core/VideoBackends/D3D/Render.h
index 020dc9d85a..c4ba243521 100644
--- a/Source/Core/VideoBackends/D3D/Render.h
+++ b/Source/Core/VideoBackends/D3D/Render.h
@@ -19,7 +19,7 @@ public:
 	void SetDepthMode() override;
 	void SetLogicOpMode() override;
 	void SetDitherMode() override;
-	void SetSamplerState(int stage,int texindex) override;
+	void SetSamplerState(int stage, int texindex, bool custom_tex) override;
 	void SetInterlacingMode() override;
 	void SetViewport() override;
 
diff --git a/Source/Core/VideoBackends/OGL/Render.cpp b/Source/Core/VideoBackends/OGL/Render.cpp
index 3e50f70481..a1f63c6d32 100644
--- a/Source/Core/VideoBackends/OGL/Render.cpp
+++ b/Source/Core/VideoBackends/OGL/Render.cpp
@@ -1868,13 +1868,13 @@ void Renderer::SetDitherMode()
 		glDisable(GL_DITHER);
 }
 
-void Renderer::SetSamplerState(int stage, int texindex)
+void Renderer::SetSamplerState(int stage, int texindex, bool custom_tex)
 {
 	auto const& tex = bpmem.tex[texindex];
 	auto const& tm0 = tex.texMode0[stage];
 	auto const& tm1 = tex.texMode1[stage];
 
-	g_sampler_cache->SetSamplerState((texindex * 4) + stage, tm0, tm1);
+	g_sampler_cache->SetSamplerState((texindex * 4) + stage, tm0, tm1, custom_tex);
 }
 
 void Renderer::SetInterlacingMode()
diff --git a/Source/Core/VideoBackends/OGL/Render.h b/Source/Core/VideoBackends/OGL/Render.h
index c3f7e59ae4..834f73e3a3 100644
--- a/Source/Core/VideoBackends/OGL/Render.h
+++ b/Source/Core/VideoBackends/OGL/Render.h
@@ -58,7 +58,7 @@ public:
 	void SetDepthMode() override;
 	void SetLogicOpMode() override;
 	void SetDitherMode() override;
-	void SetSamplerState(int stage,int texindex) override;
+	void SetSamplerState(int stage, int texindex, bool custom_tex) override;
 	void SetInterlacingMode() override;
 	void SetViewport() override;
 
diff --git a/Source/Core/VideoBackends/OGL/SamplerCache.cpp b/Source/Core/VideoBackends/OGL/SamplerCache.cpp
index a20b42d29f..71e039c449 100644
--- a/Source/Core/VideoBackends/OGL/SamplerCache.cpp
+++ b/Source/Core/VideoBackends/OGL/SamplerCache.cpp
@@ -20,7 +20,7 @@ SamplerCache::~SamplerCache()
 	Clear();
 }
 
-void SamplerCache::SetSamplerState(int stage, const TexMode0& tm0, const TexMode1& tm1)
+void SamplerCache::SetSamplerState(int stage, const TexMode0& tm0, const TexMode1& tm1, bool custom_tex)
 {
 	// TODO: can this go somewhere else?
 	if (m_last_max_anisotropy != g_ActiveConfig.iMaxAnisotropy)
@@ -38,6 +38,12 @@ void SamplerCache::SetSamplerState(int stage, const TexMode0& tm0, const TexMode
 		params.tm0.mag_filter |= 0x1;
 	}
 
+	// custom textures may have higher resolution, so disable the max_lod
+	if (custom_tex)
+	{
+		params.tm1.max_lod = 255;
+	}
+
 	// TODO: Should keep a circular buffer for each stage of recently used samplers.
 
 	auto& active_sampler = m_active_samplers[stage];
diff --git a/Source/Core/VideoBackends/OGL/SamplerCache.h b/Source/Core/VideoBackends/OGL/SamplerCache.h
index f8e77a0b93..ec033bd6be 100644
--- a/Source/Core/VideoBackends/OGL/SamplerCache.h
+++ b/Source/Core/VideoBackends/OGL/SamplerCache.h
@@ -14,7 +14,7 @@ public:
 	SamplerCache();
 	~SamplerCache();
 
-	void SetSamplerState(int stage, const TexMode0& tm0, const TexMode1& tm1);
+	void SetSamplerState(int stage, const TexMode0& tm0, const TexMode1& tm1, bool custom_tex);
 	void Clear();
 
 private:
diff --git a/Source/Core/VideoCommon/PixelShaderManager.cpp b/Source/Core/VideoCommon/PixelShaderManager.cpp
index 694f78e68f..77d92d0156 100644
--- a/Source/Core/VideoCommon/PixelShaderManager.cpp
+++ b/Source/Core/VideoCommon/PixelShaderManager.cpp
@@ -134,7 +134,7 @@ void PixelShaderManager::SetDestAlpha()
 	dirty = true;
 }
 
-void PixelShaderManager::SetTexDims(int texmapid, u32 width, u32 height, u32 wraps, u32 wrapt)
+void PixelShaderManager::SetTexDims(int texmapid, u32 width, u32 height)
 {
 	// TODO: move this check out to callee. There we could just call this function on texture changes
 	// or better, use textureSize() in glsl
diff --git a/Source/Core/VideoCommon/PixelShaderManager.h b/Source/Core/VideoCommon/PixelShaderManager.h
index 0f3dba3699..79781f0d1c 100644
--- a/Source/Core/VideoCommon/PixelShaderManager.h
+++ b/Source/Core/VideoCommon/PixelShaderManager.h
@@ -33,7 +33,7 @@ public:
 	static void SetTevKonstColor(int index, int component, s32 value);
 	static void SetAlpha();
 	static void SetDestAlpha();
-	static void SetTexDims(int texmapid, u32 width, u32 height, u32 wraps, u32 wrapt);
+	static void SetTexDims(int texmapid, u32 width, u32 height);
 	static void SetZTextureBias();
 	static void SetViewportChanged();
 	static void SetEfbScaleChanged();
diff --git a/Source/Core/VideoCommon/RenderBase.h b/Source/Core/VideoCommon/RenderBase.h
index 1f89ef828c..42d1c2e438 100644
--- a/Source/Core/VideoCommon/RenderBase.h
+++ b/Source/Core/VideoCommon/RenderBase.h
@@ -57,7 +57,7 @@ public:
 	virtual void SetDepthMode() = 0;
 	virtual void SetLogicOpMode() = 0;
 	virtual void SetDitherMode() = 0;
-	virtual void SetSamplerState(int stage,int texindex) = 0;
+	virtual void SetSamplerState(int stage, int texindex, bool custom_tex) = 0;
 	virtual void SetInterlacingMode() = 0;
 	virtual void SetViewport() = 0;
 
diff --git a/Source/Core/VideoCommon/TextureCacheBase.cpp b/Source/Core/VideoCommon/TextureCacheBase.cpp
index e35a94424b..53168f45e7 100644
--- a/Source/Core/VideoCommon/TextureCacheBase.cpp
+++ b/Source/Core/VideoCommon/TextureCacheBase.cpp
@@ -266,8 +266,8 @@ TextureCache::TCacheEntryBase* TextureCache::Load(const u32 stage)
 	const int texformat = tex.texImage0[id].format;
 	const u32 tlutaddr = tex.texTlut[id].tmem_offset << 9;
 	const u32 tlutfmt = tex.texTlut[id].tlut_format;
-	u32 tex_levels = (tex.texMode1[id].max_lod + 0xf) / 0x10 + 1;
-	const bool use_mipmaps = (tex.texMode0[id].min_filter & 3) != 0 && tex_levels > 0;
+	const bool use_mipmaps = (tex.texMode0[id].min_filter & 3) != 0;
+	u32 tex_levels = use_mipmaps ? ((tex.texMode1[id].max_lod + 0xf) / 0x10 + 1) : 1;
 	const bool from_tmem = tex.texImage1[id].image_type != 0;
 
 	if (0 == address)
@@ -472,11 +472,8 @@ TextureCache::TCacheEntryBase* TextureCache::Load(const u32 stage)
 		}
 	}
 
-	u32 texLevels = use_mipmaps ? tex_levels : 1;
-	const bool using_custom_lods = hires_tex && hires_tex->m_levels.size() >= texLevels;
-	// Only load native mips if their dimensions fit to our virtual texture dimensions
-	const bool use_native_mips = use_mipmaps && !using_custom_lods && (width == nativeW && height == nativeH);
-	texLevels = (use_native_mips || using_custom_lods) ? texLevels : 1; // TODO: Should be forced to 1 for non-pow2 textures (e.g. efb copies with automatically adjusted IR)
+	// how many levels the allocated texture shall have
+	const u32 texLevels = hires_tex ? (u32)hires_tex->m_levels.size() : tex_levels;
 
 	// create the entry/texture
 	TCacheEntryConfig config;
@@ -493,6 +490,7 @@ TextureCache::TCacheEntryBase* TextureCache::Load(const u32 stage)
 	entry->SetDimensions(nativeW, nativeH, tex_levels);
 	entry->hash = tex_hash ^ tlut_hash;
 	entry->is_efb_copy = false;
+	entry->is_custom_tex = hires_tex != nullptr;
 
 	// load texture
 	entry->Load(width, height, expandedWidth, 0);
@@ -510,10 +508,19 @@ TextureCache::TCacheEntryBase* TextureCache::Load(const u32 stage)
 		DumpTexture(entry, basename, 0);
 	}
 
-	u32 level = 1;
-	// load mips - TODO: Loading mipmaps from tmem is untested!
-	if (use_native_mips)
+	if (hires_tex)
 	{
+		for (u32 level = 1; level != texLevels; ++level)
+		{
+			auto& l = hires_tex->m_levels[level];
+			CheckTempSize(l.data_size);
+			memcpy(temp, l.data, l.data_size);
+			entry->Load(l.width, l.height, l.width, level);
+		}
+	}
+	else
+	{
+		// load mips - TODO: Loading mipmaps from tmem is untested!
 		src_data += texture_size;
 
 		const u8* ptr_even = nullptr;
@@ -524,7 +531,7 @@ TextureCache::TCacheEntryBase* TextureCache::Load(const u32 stage)
 			ptr_odd = &texMem[bpmem.tex[stage / 4].texImage2[stage % 4].tmem_odd * TMEM_LINE_SIZE];
 		}
 
-		for (; level != texLevels; ++level)
+		for (u32 level = 1; level != texLevels; ++level)
 		{
 			const u32 mip_width = CalculateLevelSize(width, level);
 			const u32 mip_height = CalculateLevelSize(height, level);
@@ -544,16 +551,6 @@ TextureCache::TCacheEntryBase* TextureCache::Load(const u32 stage)
 				DumpTexture(entry, basename, level);
 		}
 	}
-	else if (using_custom_lods)
-	{
-		for (; level != texLevels; ++level)
-		{
-			auto& l = hires_tex->m_levels[level];
-			CheckTempSize(l.data_size);
-			memcpy(temp, l.data, l.data_size);
-			entry->Load(l.width, l.height, l.width, level);
-		}
-	}
 
 	INCSTAT(stats.numTexturesUploaded);
 	SETSTAT(stats.numTexturesAlive, textures.size());
@@ -872,6 +869,7 @@ void TextureCache::CopyRenderTargetToTexture(u32 dstAddr, unsigned int dstFormat
 
 	entry->frameCount = FRAMECOUNT_INVALID;
 	entry->is_efb_copy = true;
+	entry->is_custom_tex = false;
 
 	entry->FromRenderTarget(dstAddr, dstFormat, srcFormat, srcRect, isIntensity, scaleByHalf, cbufid, colmat);
 
diff --git a/Source/Core/VideoCommon/TextureCacheBase.h b/Source/Core/VideoCommon/TextureCacheBase.h
index c721228c78..e048a3ff3c 100644
--- a/Source/Core/VideoCommon/TextureCacheBase.h
+++ b/Source/Core/VideoCommon/TextureCacheBase.h
@@ -52,6 +52,7 @@ public:
 		u64 hash;
 		u32 format;
 		bool is_efb_copy;
+		bool is_custom_tex;
 
 		unsigned int native_width, native_height; // Texture dimensions from the GameCube's point of view
 		unsigned int native_levels;
diff --git a/Source/Core/VideoCommon/VertexManagerBase.cpp b/Source/Core/VideoCommon/VertexManagerBase.cpp
index 520654448f..0eebc05663 100644
--- a/Source/Core/VideoCommon/VertexManagerBase.cpp
+++ b/Source/Core/VideoCommon/VertexManagerBase.cpp
@@ -210,13 +210,12 @@ void VertexManager::Flush()
 		TextureCache::UnbindTextures();
 		for (unsigned int i : usedtextures)
 		{
-			g_renderer->SetSamplerState(i & 3, i >> 2);
 			const TextureCache::TCacheEntryBase* tentry = TextureCache::Load(i);
 
 			if (tentry)
 			{
-				// 0s are probably for no manual wrapping needed.
-				PixelShaderManager::SetTexDims(i, tentry->native_width, tentry->native_height, 0, 0);
+				g_renderer->SetSamplerState(i & 3, i >> 2, tentry->is_custom_tex);
+				PixelShaderManager::SetTexDims(i, tentry->native_width, tentry->native_height);
 			}
 			else
 			{