diff --git a/pcsx2/IPU/IPU.h b/pcsx2/IPU/IPU.h
index 203318ca08..42da83e1d0 100644
--- a/pcsx2/IPU/IPU.h
+++ b/pcsx2/IPU/IPU.h
@@ -121,7 +121,7 @@ struct alignas(16) tIPU_BP {
 
 	__fi bool FillBuffer(u32 bits)
 	{
-		while (FP < 2)
+		while (FP <= ((BP + bits) / 128))
 		{
 			if (ipu_fifo.in.read(&internal_qwc[FP]) == 0)
 			{