diff --git a/src/core/spu.cpp b/src/core/spu.cpp
index c88121b16..675b0ffef 100644
--- a/src/core/spu.cpp
+++ b/src/core/spu.cpp
@@ -453,6 +453,7 @@ void SPU::CreateOutputStream()
   }
 
   s_audio_stream->SetOutputVolume(System::GetAudioOutputVolume());
+  s_audio_stream->SetNominalRate(System::GetAudioNominalRate());
   s_audio_stream->SetPaused(System::IsPaused());
 }
 
diff --git a/src/core/system.cpp b/src/core/system.cpp
index 029a20fa3..986fbf00f 100644
--- a/src/core/system.cpp
+++ b/src/core/system.cpp
@@ -2681,6 +2681,11 @@ float System::GetTargetSpeed()
   return s_target_speed;
 }
 
+float System::GetAudioNominalRate()
+{
+  return s_throttler_enabled ? s_target_speed : 1.0f;
+}
+
 void System::UpdatePerformanceCounters()
 {
   const float frame_time = static_cast<float>(s_frame_timer.GetTimeMillisecondsAndReset());
@@ -2869,15 +2874,7 @@ void System::UpdateSpeedLimiterState()
   // Update audio output.
   AudioStream* stream = SPU::GetOutputStream();
   stream->SetOutputVolume(GetAudioOutputVolume());
-
-  // Adjust nominal rate when resampling, or syncing to host.
-  const bool rate_adjust =
-    (s_syncing_to_host || g_settings.audio_stream_parameters.stretch_mode == AudioStretchMode::Resample) &&
-    s_target_speed > 0.0f;
-  stream->SetNominalRate(rate_adjust ? s_target_speed : 1.0f);
-
-  if (old_target_speed < s_target_speed)
-    stream->UpdateTargetTempo(s_target_speed);
+  stream->SetNominalRate(GetAudioNominalRate());
 
   UpdateThrottlePeriod();
   ResetThrottler();
diff --git a/src/core/system.h b/src/core/system.h
index d1b436f4e..ab1400702 100644
--- a/src/core/system.h
+++ b/src/core/system.h
@@ -285,6 +285,7 @@ void SingleStepCPU();
 
 /// Sets target emulation speed.
 float GetTargetSpeed();
+float GetAudioNominalRate();
 
 /// Adjusts the throttle frequency, i.e. how many times we should sleep per second.
 void SetThrottleFrequency(float frequency);
diff --git a/src/util/audio_stream.cpp b/src/util/audio_stream.cpp
index c800f523e..256377736 100644
--- a/src/util/audio_stream.cpp
+++ b/src/util/audio_stream.cpp
@@ -541,25 +541,8 @@ void AudioStream::SetNominalRate(float tempo)
   m_nominal_rate = tempo;
   if (m_parameters.stretch_mode == AudioStretchMode::Resample)
     m_soundtouch->setRate(tempo);
-}
-
-void AudioStream::UpdateTargetTempo(float tempo)
-{
-  if (m_parameters.stretch_mode != AudioStretchMode::TimeStretch)
-    return;
-
-  // undo sqrt()
-  if (tempo)
-    tempo *= tempo;
-
-  m_average_position = AVERAGING_WINDOW;
-  m_average_available = AVERAGING_WINDOW;
-  std::fill_n(m_average_fullness.data(), AVERAGING_WINDOW, tempo);
-  m_soundtouch->setTempo(tempo);
-  m_stretch_reset = 0;
-  m_stretch_inactive = false;
-  m_stretch_ok_count = 0;
-  m_dynamic_target_usage = static_cast<float>(m_target_buffer_size) * m_nominal_rate;
+  else if (m_parameters.stretch_mode == AudioStretchMode::TimeStretch && m_stretch_inactive)
+    m_soundtouch->setTempo(tempo);
 }
 
 void AudioStream::SetStretchMode(AudioStretchMode mode)
diff --git a/src/util/audio_stream.h b/src/util/audio_stream.h
index 3e4af2f13..586a0d6c9 100644
--- a/src/util/audio_stream.h
+++ b/src/util/audio_stream.h
@@ -190,7 +190,6 @@ public:
   /// Nominal rate is used for both resampling and timestretching, input samples are assumed to be this amount faster
   /// than the sample rate.
   void SetNominalRate(float tempo);
-  void UpdateTargetTempo(float tempo);
 
   void SetStretchMode(AudioStretchMode mode);