diff --git a/pcsx2/CDVD/CDVD.cpp b/pcsx2/CDVD/CDVD.cpp
index aa06b107eb..14803570fd 100644
--- a/pcsx2/CDVD/CDVD.cpp
+++ b/pcsx2/CDVD/CDVD.cpp
@@ -1364,13 +1364,18 @@ static uint cdvdStartSeek(uint newsector, CDVD_MODE_TYPE mode)
 		}
 		else
 		{
-			psxRegs.interrupt &= ~(1 << IopEvt_CdvdSectorReady);
-			cdvd.nextSectorsBuffered = 0;
+			if (delta >= cdvd.nextSectorsBuffered)
+			{
+				psxRegs.interrupt &= ~(1 << IopEvt_CdvdSectorReady);
+				cdvd.nextSectorsBuffered = 0;
+			}
+			else
+				cdvd.nextSectorsBuffered -= delta;
 		}
 	}
 
 	// Only do this on reads, the seek kind of accounts for this and then it reads the sectors after
-	if ((delta || cdvd.Action == cdvdAction_Seek) && !isSeeking)
+	if ((delta || cdvd.Action == cdvdAction_Seek) && !isSeeking && !cdvd.nextSectorsBuffered)
 	{
 		const u32 rotationalLatency = cdvdRotationalLatency((CDVD_MODE_TYPE)cdvdIsDVD());
 		//DevCon.Warning("%s rotational latency at sector %d is %d cycles", (cdvd.SpindlCtrl & CDVD_SPINDLE_CAV) ? "CAV" : "CLV", cdvd.SeekToSector, rotationalLatency);