Implement basic time stretching + Tweaks

This commit is contained in:
Rui Pinheiro 2018-12-16 21:12:58 +00:00 committed by kd-11
parent 5159d3559e
commit 892deb1552
9 changed files with 249 additions and 39 deletions

View File

@ -18,6 +18,7 @@ public:
NON_BLOCKING = 0x1,
IS_PLAYING = 0x2,
GET_NUM_ENQUEUED_SAMPLES = 0x4,
SET_FREQUENCY_RATIO = 0x8,
};
virtual ~AudioBackend() = default;
@ -46,6 +47,12 @@ public:
return 0;
}
virtual f32 SetFrequencyRatio(f32 new_ratio) // returns the new ratio
{
fmt::throw_exception("SetFrequencyRatio() not implemented");
return 1.0f;
}
// Helper methods
static u32 get_sampling_rate()
{
@ -68,4 +75,44 @@ public:
{
return g_cfg.audio.downmix_to_2ch ? 2 : 8;
}
bool has_capability(Capabilities cap) const
{
return (cap & GetCapabilities()) != 0;
}
void dump_capabilities(std::string& out) const
{
u32 count = 0;
u32 capabilities = GetCapabilities();
if (capabilities & NON_BLOCKING)
{
fmt::append(out, "NON_BLOCKING");
count++;
}
if (capabilities & IS_PLAYING)
{
fmt::append(out, "%sIS_PLAYING", count > 0 ? " | " : "");
count++;
}
if (capabilities & GET_NUM_ENQUEUED_SAMPLES)
{
fmt::append(out, "%sGET_NUM_ENQUEUED_SAMPLES", count > 0 ? " | " : "");
count++;
}
if (capabilities & SET_FREQUENCY_RATIO)
{
fmt::append(out, "%sSET_FREQUENCY_RATIO", count > 0 ? " | " : "");
count++;
}
if (count == 0)
{
fmt::append(out, "NONE");
}
}
};

View File

@ -8,7 +8,7 @@ public:
NullAudioBackend() {}
virtual ~NullAudioBackend() {}
virtual const char* GetName() const override { return "NullAudioBackend"; }
virtual const char* GetName() const override { return "Null"; }
static const u32 capabilities = NON_BLOCKING;
virtual u32 GetCapabilities() const override { return capabilities; };

View File

@ -177,4 +177,19 @@ u64 XAudio2Backend::xa27_enqueued_samples()
return (AUDIO_BUFFER_SAMPLES - state.SamplesPlayed % AUDIO_BUFFER_SAMPLES) + (state.BuffersQueued * AUDIO_BUFFER_SAMPLES);
}
f32 XAudio2Backend::xa27_set_freq_ratio(f32 new_ratio)
{
new_ratio = std::clamp(new_ratio, XAUDIO2_MIN_FREQ_RATIO, XAUDIO2_DEFAULT_FREQ_RATIO);
HRESULT hr = s_tls_source_voice->SetFrequencyRatio(new_ratio);
if (FAILED(hr))
{
LOG_ERROR(GENERAL, "XAudio2Backend : SetFrequencyRatio() failed(0x%08x)", (u32)hr);
Emu.Pause();
return 1.0f;
}
return new_ratio;
}
#endif

View File

@ -189,4 +189,19 @@ u64 XAudio2Backend::xa28_enqueued_samples()
return (AUDIO_BUFFER_SAMPLES - state.SamplesPlayed % AUDIO_BUFFER_SAMPLES) + (state.BuffersQueued * AUDIO_BUFFER_SAMPLES);
}
f32 XAudio2Backend::xa28_set_freq_ratio(f32 new_ratio)
{
new_ratio = std::clamp(new_ratio, XAUDIO2_MIN_FREQ_RATIO, XAUDIO2_DEFAULT_FREQ_RATIO);
HRESULT hr = s_tls_source_voice->SetFrequencyRatio(new_ratio);
if (FAILED(hr))
{
LOG_ERROR(GENERAL, "XAudio2Backend : SetFrequencyRatio() failed(0x%08x)", (u32)hr);
Emu.Pause();
return 1.0f;
}
return new_ratio;
}
#endif

View File

@ -21,6 +21,7 @@ XAudio2Backend::XAudio2Backend()
m_funcs.is_playing = &xa28_is_playing;
m_funcs.add = &xa28_add;
m_funcs.enqueued_samples = &xa28_enqueued_samples;
m_funcs.set_freq_ratio = &xa28_set_freq_ratio;
LOG_SUCCESS(GENERAL, "XAudio 2.9 initialized");
return;
@ -38,6 +39,7 @@ XAudio2Backend::XAudio2Backend()
m_funcs.is_playing = &xa28_is_playing;
m_funcs.add = &xa28_add;
m_funcs.enqueued_samples = &xa28_enqueued_samples;
m_funcs.set_freq_ratio = &xa28_set_freq_ratio;
LOG_SUCCESS(GENERAL, "XAudio 2.8 initialized");
return;
@ -55,6 +57,7 @@ XAudio2Backend::XAudio2Backend()
m_funcs.is_playing = &xa27_is_playing;
m_funcs.add = &xa27_add;
m_funcs.enqueued_samples = &xa27_enqueued_samples;
m_funcs.set_freq_ratio = &xa27_set_freq_ratio;
LOG_SUCCESS(GENERAL, "XAudio 2.7 initialized");
return;
@ -109,4 +112,9 @@ u64 XAudio2Backend::GetNumEnqueuedSamples()
return m_funcs.enqueued_samples();
}
f32 XAudio2Backend::SetFrequencyRatio(f32 new_ratio)
{
return m_funcs.set_freq_ratio(new_ratio);
}
#endif

View File

@ -16,6 +16,7 @@ class XAudio2Backend : public AudioBackend
bool(*is_playing)();
bool(*add)(const void*, int);
u64(*enqueued_samples)();
f32(*set_freq_ratio)(f32);
};
vtable m_funcs;
@ -29,6 +30,7 @@ class XAudio2Backend : public AudioBackend
static bool xa27_is_playing();
static bool xa27_add(const void*, int);
static u64 xa27_enqueued_samples();
static f32 xa27_set_freq_ratio(f32);
static void xa28_init(void*);
static void xa28_destroy();
@ -39,14 +41,15 @@ class XAudio2Backend : public AudioBackend
static bool xa28_is_playing();
static bool xa28_add(const void*, int);
static u64 xa28_enqueued_samples();
static f32 xa28_set_freq_ratio(f32);
public:
XAudio2Backend();
virtual ~XAudio2Backend() override;
virtual const char* GetName() const override { return "XAudio2Backend"; };
virtual const char* GetName() const override { return "XAudio2"; };
static const u32 capabilities = NON_BLOCKING | IS_PLAYING | GET_NUM_ENQUEUED_SAMPLES;
static const u32 capabilities = NON_BLOCKING | IS_PLAYING | GET_NUM_ENQUEUED_SAMPLES | SET_FREQUENCY_RATIO;
virtual u32 GetCapabilities() const override { return capabilities; };
virtual void Open() override;
@ -60,6 +63,7 @@ public:
virtual void Flush() override;
virtual u64 GetNumEnqueuedSamples() override;
virtual f32 SetFrequencyRatio(f32 new_ratio) override;
};
#endif

View File

@ -60,18 +60,13 @@ audio_ringbuffer::audio_ringbuffer(cell_audio_config& _cfg)
m_dump.reset(new AudioDumper(cfg.audio_channels));
}
// Sanity check configuration vs. capabilities
backend_capabilities = backend->GetCapabilities();
if (cfg.buffering_enabled)
// Initialize backend
{
if (!(backend_capabilities & AudioBackend::NON_BLOCKING) || !(backend_capabilities & AudioBackend::IS_PLAYING))
{
// We need a non-blocking backend to be able to do buffering correctly
fmt::throw_exception("Audio backend %s does not support buffering.", backend->GetName());
}
std::string str;
backend->dump_capabilities(str);
cellAudio.error("cellAudio initializing. Backend: %s, Capabilities: %s", backend->GetName(), str.c_str());
}
// Initialize backend
backend->Open();
backend_open = true;
@ -93,6 +88,21 @@ audio_ringbuffer::~audio_ringbuffer()
backend->Close();
}
f32 audio_ringbuffer::set_frequency_ratio(f32 new_ratio)
{
if(!has_capability(AudioBackend::SET_FREQUENCY_RATIO))
{
ASSERT(new_ratio == 1.0f);
frequency_ratio = 1.0f;
}
else
{
frequency_ratio = backend->SetFrequencyRatio(new_ratio);
//cellAudio.error("set_frequency_ratio(%1.2f) -> %1.2f", new_ratio, frequency_ratio);
}
return frequency_ratio;
}
void audio_ringbuffer::enqueue(const float* in_buffer)
{
AUDIT(cur_pos < cfg.num_allocated_buffers);
@ -140,6 +150,11 @@ void audio_ringbuffer::play()
if (playing)
return;
if (frequency_ratio != 1.0f)
{
set_frequency_ratio(1.0f);
}
playing = true;
ASSERT(enqueued_samples > 0);
@ -156,6 +171,12 @@ void audio_ringbuffer::flush()
playing = false;
backend->Flush();
if (frequency_ratio != 1.0f)
{
set_frequency_ratio(1.0f);
}
enqueued_samples = 0;
}
@ -184,7 +205,7 @@ u64 audio_ringbuffer::update()
// Calculate how many audio samples have played since last time
if (cfg.buffering_enabled && (playing || new_playing))
{
if (backend_capabilities & AudioBackend::GET_NUM_ENQUEUED_SAMPLES)
if (has_capability(AudioBackend::GET_NUM_ENQUEUED_SAMPLES))
{
// Backend supports querying for the remaining playtime, so just ask it
enqueued_samples = backend->GetNumEnqueuedSamples();
@ -194,7 +215,7 @@ u64 audio_ringbuffer::update()
const u64 play_delta = timestamp - (play_timestamp > update_timestamp ? play_timestamp : update_timestamp);
// NOTE: Only works with a fixed sampling rate
const u64 delta_samples_tmp = (play_delta * cfg.audio_sampling_rate) + last_remainder;
const u64 delta_samples_tmp = play_delta * static_cast<u64>(cfg.audio_sampling_rate * frequency_ratio) + last_remainder;
last_remainder = delta_samples_tmp % 1'000'000;
const u64 delta_samples = delta_samples_tmp / 1'000'000;
@ -390,6 +411,14 @@ void cell_audio_thread::advance(u64 timestamp, bool reset)
m_indexes[port.number] = port.cur_pos;
}
if (cfg.buffering_enabled)
{
// Calculate rolling average of enqueued playtime
const u32 enqueued_playtime = ringbuffer->get_enqueued_playtime();
m_average_playtime = cfg.period_average_alpha * enqueued_playtime + (1.0f - cfg.period_average_alpha) * m_average_playtime;
//cellAudio.error("m_average_playtime=%4.2f, enqueued_playtime=%u", m_average_playtime, enqueued_playtime);
}
m_counter++;
m_last_period_end = timestamp;
m_dynamic_period = 0;
@ -421,6 +450,23 @@ void cell_audio_thread::operator()()
// Allocate ringbuffer
ringbuffer.reset(new audio_ringbuffer(cfg));
// Check backend capabilities
if (cfg.buffering_enabled)
{
if (!has_capability(AudioBackend::NON_BLOCKING) || !has_capability(AudioBackend::IS_PLAYING))
{
// We need a non-blocking backend to be able to do buffering correctly
// We also need to be able to query the current playing state
fmt::throw_exception("Audio backend %s does not support buffering.", ringbuffer->get_backend_name());
}
if (cfg.time_stretching_enabled && !has_capability(AudioBackend::SET_FREQUENCY_RATIO))
{
// We need to be able to set a dynamic frequency ratio to be able to do time stretching
fmt::throw_exception("Audio backend %s does not support time stretching", ringbuffer->get_backend_name());
}
}
// Initialize loop variables
m_counter = 0;
m_start_time = ringbuffer->get_timestamp();
@ -459,8 +505,10 @@ void cell_audio_thread::operator()()
}
else
{
const u64 enqueued_playtime = ringbuffer->get_enqueued_samples() * 1'000'000 / cfg.audio_sampling_rate;
const u64 enqueued_buffers = (enqueued_playtime) / cfg.audio_block_period;
const u64 enqueued_samples = ringbuffer->get_enqueued_samples();
f32 frequency_ratio = ringbuffer->get_frequency_ratio();
u64 enqueued_playtime = ringbuffer->get_enqueued_playtime();
const u64 enqueued_buffers = enqueued_samples / AUDIO_BUFFER_SAMPLES;
const bool playing = ringbuffer->is_playing();
@ -471,32 +519,70 @@ void cell_audio_thread::operator()()
const u32 incomplete = std::get<3>(tag_info);
// Wait for a dynamic period - try to maintain an average as close as possible to 5.(3)ms
if (m_dynamic_period == 0)
if (!playing)
{
if (!playing)
// When the buffer is empty, always use the correct block period
m_dynamic_period = cfg.audio_block_period;
}
else
{
// Ratio between the rolling average of the audio period, and the desired audio period
const f32 average_playtime_ratio = m_average_playtime / cfg.audio_buffer_length;
// Use the above adjusted ratio to decide how much buffer we should be aiming for
f32 desired_duration_adjusted = cfg.desired_buffer_duration + (cfg.audio_block_period / 2.0f);
if (average_playtime_ratio < 1.0f)
{
// When the buffer is empty, always use the correct block period
m_dynamic_period = cfg.audio_block_period;
desired_duration_adjusted /= average_playtime_ratio;
}
else
if (cfg.time_stretching_enabled)
{
// Calculate what the playtime is without a frequency ratio
const u64 raw_enqueued_playtime = ringbuffer->get_enqueued_playtime(/* raw= */ true);
// 1.0 means exactly as desired
// <1.0 means not as full as desired
// >1.0 means more full than desired
const f32 desired_duration_rate = (enqueued_playtime) / static_cast<f32>(cfg.desired_buffer_duration);
const f32 desired_duration_rate = raw_enqueued_playtime / desired_duration_adjusted;
if (desired_duration_rate >= 1.0f)
// update frequency ratio if necessary
f32 new_ratio = frequency_ratio;
if (desired_duration_rate < cfg.time_stretching_threshold)
{
// more full than desired
const f32 multiplier = 1.0f / desired_duration_rate;
m_dynamic_period = cfg.maximum_block_period - static_cast<u64>((cfg.maximum_block_period - cfg.audio_block_period) * multiplier);
new_ratio = ringbuffer->set_frequency_ratio(desired_duration_rate * cfg.time_stretching_frequency_scale_factor);
}
else
else if (frequency_ratio != 1.0f)
{
// not as full as desired
const f32 multiplier = desired_duration_rate;
m_dynamic_period = cfg.minimum_block_period + static_cast<u64>((cfg.audio_block_period - cfg.minimum_block_period) * multiplier);
new_ratio = ringbuffer->set_frequency_ratio(1.0f);
}
if (new_ratio != frequency_ratio)
{
// ratio changed, calculate new dynamic period
frequency_ratio = new_ratio;
enqueued_playtime = ringbuffer->get_enqueued_playtime();
m_dynamic_period = 0;
}
}
// 1.0 means exactly as desired
// <1.0 means not as full as desired
// >1.0 means more full than desired
const f32 desired_duration_rate = enqueued_playtime / desired_duration_adjusted;
if (desired_duration_rate >= 1.0f)
{
// more full than desired
const f32 multiplier = 1.0f / desired_duration_rate;
m_dynamic_period = cfg.maximum_block_period - static_cast<u64>((cfg.maximum_block_period - cfg.audio_block_period) * multiplier);
}
else
{
// not as full as desired
const f32 multiplier = desired_duration_rate * desired_duration_rate;
m_dynamic_period = cfg.minimum_block_period + static_cast<u64>((cfg.audio_block_period - cfg.minimum_block_period) * multiplier);
}
}
@ -553,7 +639,9 @@ void cell_audio_thread::operator()()
continue;
}
//cellAudio.error("time_since_last=%llu, dynamic_period=%llu => %3.2f%%", time_since_last_period, m_dynamic_period, (((f32)m_dynamic_period) / audio_block_period) * 100);
/*cellAudio.error("time_since_last=%llu, dynamic_period=%llu => %3.2f%%, average_period=%4.2f => %3.2f%%", time_since_last_period,
m_dynamic_period, (((f32)m_dynamic_period) / cfg.audio_block_period) * 100.0f,
m_average_period, (m_average_period / cfg.audio_block_period) * 100.0f);*/
//cellAudio.error("active=%u, untouched=%u, in_progress=%d, incomplete=%d, enqueued_buffers=%u", active_ports, untouched, in_progress, incomplete, enqueued_buffers);
// Store number of untouched buffers for future reference
@ -581,6 +669,7 @@ void cell_audio_thread::operator()()
ringbuffer->flush();
ringbuffer->enqueue_silence(cfg.desired_full_buffers);
finish_port_volume_stepping();
m_average_playtime = ringbuffer->get_enqueued_playtime();
}
}

View File

@ -90,7 +90,7 @@ enum : u32
AUDIO_BLOCK_SIZE_2CH = 2 * AUDIO_BUFFER_SAMPLES,
AUDIO_BLOCK_SIZE_8CH = 8 * AUDIO_BUFFER_SAMPLES,
PORT_BUFFER_TAG_COUNT = 4,
PORT_BUFFER_TAG_COUNT = 8,
PORT_BUFFER_TAG_LAST_2CH = AUDIO_BLOCK_SIZE_2CH - 1,
PORT_BUFFER_TAG_DELTA_2CH = PORT_BUFFER_TAG_LAST_2CH / (PORT_BUFFER_TAG_COUNT - 1),
@ -176,7 +176,7 @@ struct cell_audio_config
const u32 audio_channels = AudioBackend::get_channels();
const u32 audio_sampling_rate = AudioBackend::get_sampling_rate();
const u64 audio_block_period = AUDIO_BUFFER_SAMPLES * 1000000 / audio_sampling_rate;
const u32 audio_block_period = AUDIO_BUFFER_SAMPLES * 1000000 / audio_sampling_rate;
const u64 desired_buffer_duration = g_cfg.audio.enable_buffering ? g_cfg.audio.desired_buffer_duration : 0;
const u32 audio_buffer_length = AUDIO_BUFFER_SAMPLES * audio_channels;
@ -184,10 +184,16 @@ struct cell_audio_config
const bool buffering_enabled = g_cfg.audio.enable_buffering && (desired_buffer_duration >= audio_block_period);
const u64 minimum_block_period = audio_block_period / 2; // the block period will not be dynamically lowered below this value (usecs)
const u64 maximum_block_period = audio_block_period + (audio_block_period - minimum_block_period); // the block period will not be dynamically increased above this value (usecs)
const u64 maximum_block_period = (6 * audio_block_period) / 5; // the block period will not be dynamically increased above this value (usecs)
const u32 desired_full_buffers = buffering_enabled ? static_cast<u32>(desired_buffer_duration / audio_block_period) + 1 : 1;
const u32 num_allocated_buffers = desired_full_buffers + EXTRA_AUDIO_BUFFERS; // number of ringbuffer buffers
const f32 period_average_alpha = 0.02f; // alpha factor for the m_average_period rolling average
const bool time_stretching_enabled = buffering_enabled && g_cfg.audio.enable_time_stretching && (g_cfg.audio.time_stretching_threshold > 0);
const f32 time_stretching_threshold = g_cfg.audio.time_stretching_threshold / 100.0f; // we only apply time stretching below this buffer fill rate (adjusted for average period)
const f32 time_stretching_frequency_scale_factor = 1.0f / time_stretching_threshold;
};
class audio_ringbuffer
@ -208,19 +214,19 @@ private:
bool playing = false;
bool emu_paused = false;
u32 backend_capabilities;
u64 update_timestamp = 0;
u64 play_timestamp = 0;
u64 last_remainder = 0;
u64 enqueued_samples = 0;
f32 frequency_ratio = 1.0f;
u32 cur_pos = 0;
bool backend_is_playing() const
{
return (backend_capabilities & AudioBackend::IS_PLAYING) ? backend->IsPlaying() : playing;
return has_capability(AudioBackend::IS_PLAYING) ? backend->IsPlaying() : playing;
}
public:
@ -232,6 +238,7 @@ public:
void flush();
u64 update();
void enqueue_silence(u32 buf_count = 1);
f32 set_frequency_ratio(f32 new_ratio);
float* get_buffer(u32 num) const
{
@ -256,14 +263,31 @@ public:
return enqueued_samples;
}
u64 get_enqueued_playtime(bool raw = false) const
{
AUDIT(cfg.buffering_enabled);
u64 sampling_rate = raw ? cfg.audio_sampling_rate : static_cast<u64>(cfg.audio_sampling_rate * frequency_ratio);
return enqueued_samples * 1'000'000 / sampling_rate;
}
bool is_playing() const
{
return playing;
}
u32 capabilities() const
f32 get_frequency_ratio() const
{
return backend_capabilities;
return frequency_ratio;
}
u32 has_capability(AudioBackend::Capabilities cap) const
{
return backend->has_capability(cap);
}
const char* get_backend_name() const
{
return backend->GetName();
}
};
@ -296,6 +320,7 @@ public:
u64 m_counter = 0;
u64 m_start_time = 0;
u64 m_dynamic_period = 0;
f32 m_average_playtime;
void operator()();
@ -323,6 +348,11 @@ public:
return nullptr;
}
bool has_capability(AudioBackend::Capabilities cap) const
{
return ringbuffer->has_capability(cap);
}
};
using cell_audio = named_thread<cell_audio_thread>;

View File

@ -531,6 +531,8 @@ struct cfg_root : cfg::node
cfg::_bool enable_buffering{this, "Enable Buffering", true};
cfg::_int <0, 250'000> desired_buffer_duration{this, "Desired Audio Buffer Duration", 100'000};
cfg::_int<1, 1000> sampling_period_multiplier{this, "Sampling Period Multiplier", 100};
cfg::_bool enable_time_stretching{this, "Enable Time Stretching", true};
cfg::_int<0, 100> time_stretching_threshold{this, "Time Stretching Threshold", 75};
} audio{this};