diff --git a/Source/Core/AudioCommon/Mixer.cpp b/Source/Core/AudioCommon/Mixer.cpp
index b52371d994..955d2a0efb 100644
--- a/Source/Core/AudioCommon/Mixer.cpp
+++ b/Source/Core/AudioCommon/Mixer.cpp
@@ -93,6 +93,8 @@ void Mixer::MixerFifo::Mix(s16* samples, std::size_t num_samples)
       std::clamp((buffer_size_samples) / (GRANULE_SIZE >> 1), static_cast<std::size_t>(4),
                  static_cast<std::size_t>(MAX_GRANULE_QUEUE_SIZE));
 
+  bool fade_audio = m_queue_fading.load(std::memory_order_relaxed);
+
   m_granule_queue_size.store(buffer_size_granules, std::memory_order_relaxed);
 
   while (num_samples-- > 0)
@@ -106,9 +108,9 @@ void Mixer::MixerFifo::Mix(s16* samples, std::size_t num_samples)
     // If either index is less than the index jump, that means we reached
     // the end of the of the buffer and need to load the next granule.
     if (front_index < index_jump)
-      Dequeue(&m_front);
+      fade_audio = Dequeue(&m_front);
     else if (back_index < index_jump)
-      Dequeue(&m_back);
+      fade_audio = Dequeue(&m_back);
 
     // The Granules are pre-windowed, so we can just add them together
     const std::size_t ft = front_index >> GRANULE_FRAC_BITS;
@@ -137,7 +139,7 @@ void Mixer::MixerFifo::Mix(s16* samples, std::size_t num_samples)
                          s5 * StereoPair{(+0.0f + 0.0f * t1 + 1.0f * t2 - 1.0f * t3) / 12.0f});
 
     // Apply Fade In / Fade Out depending on if we are looping
-    if (m_queue_looping.load(std::memory_order_relaxed))
+    if (fade_audio)
       m_fade_volume += fade_out_mul * (0.0f - m_fade_volume);
     else
       m_fade_volume += fade_in_mul * (1.0f - m_fade_volume);
@@ -513,10 +515,11 @@ void Mixer::MixerFifo::Enqueue()
     m_queue[head][i] = m_next_buffer[(i + start_index) & GRANULE_MASK] * GRANULE_WINDOW[i];
 
   m_queue_head.store(next_head, std::memory_order_release);
+  m_queue_fading.store(false, std::memory_order_relaxed);
   m_queue_looping.store(false, std::memory_order_relaxed);
 }
 
-void Mixer::MixerFifo::Dequeue(Granule* granule)
+bool Mixer::MixerFifo::Dequeue(Granule* granule)
 {
   const std::size_t granule_queue_size = m_granule_queue_size.load(std::memory_order_relaxed);
   const std::size_t head = m_queue_head.load(std::memory_order_acquire);
@@ -542,16 +545,22 @@ void Mixer::MixerFifo::Dequeue(Granule* granule)
       // This provides smoother audio playback than suddenly stopping.
       const std::size_t gap = std::max<std::size_t>(2, granule_queue_size >> 1) - 1;
       next_tail = (head - gap) & GRANULE_QUEUE_MASK;
+
+      bool looping = m_queue_looping.load(std::memory_order_relaxed);
+      m_queue_fading.store(looping, std::memory_order_relaxed);
       m_queue_looping.store(true, std::memory_order_relaxed);
     }
     else
     {
       std::fill(granule->begin(), granule->end(), StereoPair{0.0f, 0.0f});
+      m_queue_fading.store(false, std::memory_order_relaxed);
       m_queue_looping.store(false, std::memory_order_relaxed);
-      return;
+      return false;
     }
   }
 
   *granule = m_queue[tail];
   m_queue_tail.store(next_tail, std::memory_order_release);
+
+  return m_queue_fading.load(std::memory_order_relaxed);
 }
diff --git a/Source/Core/AudioCommon/Mixer.h b/Source/Core/AudioCommon/Mixer.h
index ba74c3d71b..1081e10551 100644
--- a/Source/Core/AudioCommon/Mixer.h
+++ b/Source/Core/AudioCommon/Mixer.h
@@ -126,11 +126,12 @@ private:
     std::array<Granule, MAX_GRANULE_QUEUE_SIZE> m_queue;
     std::atomic<std::size_t> m_queue_head{0};
     std::atomic<std::size_t> m_queue_tail{0};
+    std::atomic<bool> m_queue_fading{false};
     std::atomic<bool> m_queue_looping{false};
     float m_fade_volume = 1.0;
 
     void Enqueue();
-    void Dequeue(Granule* granule);
+    bool Dequeue(Granule* granule);
 
     // Volume ranges from 0-256
     std::atomic<s32> m_LVolume{256};