diff --git a/pcsx2/GS/GSLocalMemory.h b/pcsx2/GS/GSLocalMemory.h
index 61f6c59531..f1b3eb8a28 100644
--- a/pcsx2/GS/GSLocalMemory.h
+++ b/pcsx2/GS/GSLocalMemory.h
@@ -981,10 +981,15 @@ public:
 		off.loopPixels(r, vm32(), (u32*)src, pitch, [&](u32* dst, u32* src) { *dst = *src; });
 	}
 
+	void WritePixel32(u8* RESTRICT src, u32 pitch, const GSOffset& off, const GSVector4i& r, u32 write_mask)
+	{
+		off.loopPixels(r, vm32(), (u32*)src, pitch, [&](u32* dst, u32* src) { *dst = (*dst & ~write_mask) | (*src & write_mask); });
+	}
+
 	void WritePixel24(u8* RESTRICT src, u32 pitch, const GSOffset& off, const GSVector4i& r)
 	{
 		off.loopPixels(r, vm32(), (u32*)src, pitch,
-		[&](u32* dst, u32* src)
+			[&](u32* dst, u32* src)
 		{
 			*dst = (*dst & 0xff000000) | (*src & 0x00ffffff);
 		});
diff --git a/pcsx2/GS/Renderers/HW/GSRendererHW.cpp b/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
index 39bc4970c0..3b6b33c81a 100644
--- a/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
+++ b/pcsx2/GS/Renderers/HW/GSRendererHW.cpp
@@ -1959,6 +1959,8 @@ void GSRendererHW::Draw()
 		if(can_update_size)
 			rt->UpdateValidity(m_r);
 
+		rt->UpdateValidBits(~fm & fm_mask);
+
 		m_tc->InvalidateVideoMem(context->offset.fb, m_r, false, false);
 
 		m_tc->InvalidateVideoMemType(GSTextureCache::DepthStencil, context->FRAME.Block());
@@ -1971,6 +1973,8 @@ void GSRendererHW::Draw()
 		if (can_update_size)
 			ds->UpdateValidity(m_r);
 
+		ds->UpdateValidBits(GSLocalMemory::m_psm[context->ZBUF.PSM].fmsk);
+
 		m_tc->InvalidateVideoMem(context->offset.zb, m_r, false, false);
 
 		m_tc->InvalidateVideoMemType(GSTextureCache::RenderTarget, context->ZBUF.Block());
diff --git a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
index 7f31739c70..dc6ebe9939 100644
--- a/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
+++ b/pcsx2/GS/Renderers/HW/GSTextureCache.cpp
@@ -1005,6 +1005,7 @@ void GSTextureCache::ExpandTarget(const GIFRegBITBLTBUF& BITBLTBUF, const GSVect
 				AddDirtyRectTarget(dst, r, TEX0.PSM, TEX0.TBW);
 				GetTargetHeight(TEX0.TBP0, TEX0.TBW, TEX0.PSM, aligned_height);
 				dst->UpdateValidity(r);
+				dst->UpdateValidBits(GSLocalMemory::m_psm[TEX0.PSM].fmsk);
 			}
 		}
 	}
@@ -1015,6 +1016,7 @@ void GSTextureCache::ExpandTarget(const GIFRegBITBLTBUF& BITBLTBUF, const GSVect
 				static_cast<int>(dst->m_texture->GetHeight() / dst->m_texture->GetScale().y))));
 		AddDirtyRectTarget(dst, clamped_r, TEX0.PSM, TEX0.TBW);
 		dst->UpdateValidity(clamped_r);
+		dst->UpdateValidBits(GSLocalMemory::m_psm[TEX0.PSM].fmsk);
 	}
 }
 // Goal: Depth And Target at the same address is not possible. On GS it is
@@ -1699,6 +1701,7 @@ bool GSTextureCache::Move(u32 SBP, u32 SBW, u32 SPSM, int sx, int sy, u32 DBP, u
 		{
 			dst->m_texture->SetScale(scale);
 			dst->UpdateValidity(GSVector4i(dx, dy, dx + w, dy + h));
+			dst->m_valid_bits = src->m_valid_bits;
 		}
 	}
 
@@ -2726,6 +2729,16 @@ void GSTextureCache::Read(Target* t, const GSVector4i& r)
 			return;
 	}
 
+	// Don't overwrite bits which aren't used in the target's format.
+	// Stops Burnout 3's sky from breaking when flushing targets to local memory.
+	const GSLocalMemory::psm_t& psm = GSLocalMemory::m_psm[TEX0.PSM];
+	const u32 write_mask = t->m_valid_bits & psm.fmsk;
+	if (psm.bpp > 16 && write_mask == 0)
+	{
+		Console.Warning("Not reading back target %x PSM %s due to no write mask", TEX0.TBP0, psm_str(TEX0.PSM));
+		return;
+	}
+
 	// Yes lots of logging, but I'm not confident with this code
 	GL_PUSH("Texture Cache Read. Format(0x%x)", TEX0.PSM);
 
@@ -2771,11 +2784,9 @@ void GSTextureCache::Read(Target* t, const GSVector4i& r)
 	{
 		case PSM_PSMCT32:
 		case PSM_PSMZ32:
-			g_gs_renderer->m_mem.WritePixel32(bits, pitch, off, r);
-			break;
 		case PSM_PSMCT24:
 		case PSM_PSMZ24:
-			g_gs_renderer->m_mem.WritePixel24(bits, pitch, off, r);
+			g_gs_renderer->m_mem.WritePixel32(bits, pitch, off, r, write_mask);
 			break;
 		case PSM_PSMCT16:
 		case PSM_PSMCT16S:
@@ -3343,6 +3354,10 @@ void GSTextureCache::Target::UpdateValidity(const GSVector4i& rect, bool can_res
 	// GL_CACHE("UpdateValidity (0x%x->0x%x) from R:%d,%d Valid: %d,%d", m_TEX0.TBP0, m_end_block, rect.z, rect.w, m_valid.z, m_valid.w);
 }
 
+void GSTextureCache::Target::UpdateValidBits(u32 bits_written)
+{
+	m_valid_bits |= bits_written;
+}
 
 bool GSTextureCache::Target::ResizeTexture(int new_width, int new_height, bool recycle_old)
 {
diff --git a/pcsx2/GS/Renderers/HW/GSTextureCache.h b/pcsx2/GS/Renderers/HW/GSTextureCache.h
index 6c9c7fd37e..19e89c7d7e 100644
--- a/pcsx2/GS/Renderers/HW/GSTextureCache.h
+++ b/pcsx2/GS/Renderers/HW/GSTextureCache.h
@@ -235,13 +235,14 @@ public:
 	{
 	public:
 		const int m_type = 0;
+		const bool m_depth_supported = false;
+		bool m_dirty_alpha = true;
+		bool m_is_frame = false;
 		bool m_used = false;
 		GSDirtyRectList m_dirty;
 		GSVector4i m_valid{};
 		GSVector4i m_drawn_since_read{};
-		const bool m_depth_supported = false;
-		bool m_dirty_alpha = true;
-		bool m_is_frame = false;
+		u32 m_valid_bits = 0;
 		int readbacks_since_draw = 0;
 
 	public:
@@ -257,6 +258,7 @@ public:
 
 		void ResizeValidity(const GSVector4i& rect);
 		void UpdateValidity(const GSVector4i& rect, bool can_resize = true);
+		void UpdateValidBits(u32 bits_written);
 
 		void Update(bool reset_age);