From: Steinar H. Gunderson Date: Sat, 13 Aug 2016 17:22:07 +0000 (+0200) Subject: Move the R128 and correlation measurements into AudioMixer. X-Git-Tag: 1.4.0~98 X-Git-Url: https://git.sesse.net/?p=nageru;a=commitdiff_plain;h=c015407a3953235df07a601baa6aa8e02ba7b561 Move the R128 and correlation measurements into AudioMixer. --- diff --git a/audio_mixer.cpp b/audio_mixer.cpp index 1c570fa..5a78045 100644 --- a/audio_mixer.cpp +++ b/audio_mixer.cpp @@ -77,13 +77,38 @@ void convert_fixed32_to_fp32(float *dst, size_t out_channel, size_t out_num_chan } } +float find_peak(const float *samples, size_t num_samples) +{ + float m = fabs(samples[0]); + for (size_t i = 1; i < num_samples; ++i) { + m = max(m, fabs(samples[i])); + } + return m; +} + +void deinterleave_samples(const vector &in, vector *out_l, vector *out_r) +{ + size_t num_samples = in.size() / 2; + out_l->resize(num_samples); + out_r->resize(num_samples); + + const float *inptr = in.data(); + float *lptr = &(*out_l)[0]; + float *rptr = &(*out_r)[0]; + for (size_t i = 0; i < num_samples; ++i) { + *lptr++ = *inptr++; + *rptr++ = *inptr++; + } +} + } // namespace AudioMixer::AudioMixer(unsigned num_cards) : num_cards(num_cards), level_compressor(OUTPUT_FREQUENCY), limiter(OUTPUT_FREQUENCY), - compressor(OUTPUT_FREQUENCY) + compressor(OUTPUT_FREQUENCY), + correlation(OUTPUT_FREQUENCY) { locut.init(FILTER_HPF, 2); @@ -108,6 +133,13 @@ AudioMixer::AudioMixer(unsigned num_cards) // Look for ALSA cards. available_alsa_cards = ALSAInput::enumerate_devices(); + + r128.init(2, OUTPUT_FREQUENCY); + r128.integr_start(); + + // hlen=16 is pretty low quality, but we use quite a bit of CPU otherwise, + // and there's a limit to how important the peak meter is. + peak_resampler.setup(OUTPUT_FREQUENCY, OUTPUT_FREQUENCY * 4, /*num_channels=*/2, /*hlen=*/16, /*frel=*/1.0); } AudioMixer::~AudioMixer() @@ -418,7 +450,7 @@ vector AudioMixer::get_output(double pts, unsigned num_samples, Resamplin // Note that there's a feedback loop here, so we choose a very slow filter // (half-time of 30 seconds). double target_loudness_factor, alpha; - double loudness_lu = loudness_lufs - ref_level_lufs; + double loudness_lu = r128.loudness_M() - ref_level_lufs; double current_makeup_lu = to_db(final_makeup_gain); target_loudness_factor = final_makeup_gain * from_db(-loudness_lu); @@ -446,9 +478,74 @@ vector AudioMixer::get_output(double pts, unsigned num_samples, Resamplin final_makeup_gain = m; } + update_meters(samples_out); + return samples_out; } +void AudioMixer::update_meters(const vector &samples) +{ + // Upsample 4x to find interpolated peak. + peak_resampler.inp_data = const_cast(samples.data()); + peak_resampler.inp_count = samples.size() / 2; + + vector interpolated_samples; + interpolated_samples.resize(samples.size()); + { + unique_lock lock(audio_measure_mutex); + + while (peak_resampler.inp_count > 0) { // About four iterations. + peak_resampler.out_data = &interpolated_samples[0]; + peak_resampler.out_count = interpolated_samples.size() / 2; + peak_resampler.process(); + size_t out_stereo_samples = interpolated_samples.size() / 2 - peak_resampler.out_count; + peak = max(peak, find_peak(interpolated_samples.data(), out_stereo_samples * 2)); + peak_resampler.out_data = nullptr; + } + } + + // Find R128 levels and L/R correlation. + vector left, right; + deinterleave_samples(samples, &left, &right); + float *ptrs[] = { left.data(), right.data() }; + { + unique_lock lock(audio_measure_mutex); + r128.process(left.size(), ptrs); + correlation.process_samples(samples); + } + + send_audio_level_callback(); +} + +void AudioMixer::reset_meters() +{ + unique_lock lock(audio_measure_mutex); + peak_resampler.reset(); + peak = 0.0f; + r128.reset(); + r128.integr_start(); + correlation.reset(); +} + +void AudioMixer::send_audio_level_callback() +{ + if (audio_level_callback == nullptr) { + return; + } + + unique_lock lock(audio_measure_mutex); + double loudness_s = r128.loudness_S(); + double loudness_i = r128.integrated(); + double loudness_range_low = r128.range_min(); + double loudness_range_high = r128.range_max(); + + audio_level_callback(loudness_s, to_db(peak), + loudness_i, loudness_range_low, loudness_range_high, + gain_staging_db, + to_db(final_makeup_gain), + correlation.get_correlation()); +} + map AudioMixer::get_devices() const { lock_guard lock(audio_mutex); diff --git a/audio_mixer.h b/audio_mixer.h index 6e1d5a9..75b468a 100644 --- a/audio_mixer.h +++ b/audio_mixer.h @@ -7,9 +7,6 @@ // all together into one final audio signal. // // All operations on AudioMixer (except destruction) are thread-safe. -// -// TODO: There might be more audio stuff that should be moved here -// from Mixer. #include #include @@ -19,11 +16,14 @@ #include #include #include +#include #include "alsa_input.h" #include "bmusb/bmusb.h" +#include "correlation_measurer.h" #include "db.h" #include "defs.h" +#include "ebu_r128_proc.h" #include "filter.h" #include "resampling_queue.h" #include "stereocompressor.h" @@ -77,6 +77,7 @@ public: AudioMixer(unsigned num_cards); ~AudioMixer(); void reset_resampler(DeviceSpec device_spec); + void reset_meters(); // Add audio (or silence) to the given device's queue. Can return false if // the lock wasn't successfully taken; if so, you should simply try again. @@ -88,9 +89,6 @@ public: std::vector get_output(double pts, unsigned num_samples, ResamplingQueue::RateAdjustmentPolicy rate_adjustment_policy); - // See comments inside get_output(). - void set_current_loudness(double level_lufs) { loudness_lufs = level_lufs; } - void set_fader_volume(unsigned bus_index, float level_db) { fader_volume_db[bus_index] = level_db; } std::map get_devices() const; void set_name(DeviceSpec device_spec, const std::string &name); @@ -203,6 +201,15 @@ public: return final_makeup_gain_auto; } + typedef std::function audio_level_callback_t; + void set_audio_level_callback(audio_level_callback_t callback) + { + audio_level_callback = callback; + } + private: struct AudioDevice { std::unique_ptr resampling_queue; @@ -221,6 +228,8 @@ private: void reset_resampler_mutex_held(DeviceSpec device_spec); void reset_alsa_mutex_held(DeviceSpec device_spec); std::map get_devices_mutex_held() const; + void update_meters(const std::vector &samples); + void send_audio_level_callback(); unsigned num_cards; @@ -245,8 +254,6 @@ private: static constexpr float ref_level_dbfs = -14.0f; // Chosen so that we end up around 0 LU in practice. static constexpr float ref_level_lufs = -23.0f; // 0 LU, more or less by definition. - std::atomic loudness_lufs{ref_level_lufs}; - StereoCompressor limiter; std::atomic limiter_threshold_dbfs{ref_level_dbfs + 4.0f}; // 4 dB. std::atomic limiter_enabled{true}; @@ -259,6 +266,13 @@ private: InputMapping input_mapping; // Under audio_mutex. std::atomic fader_volume_db[MAX_BUSES] {{ 0.0f }}; + + audio_level_callback_t audio_level_callback = nullptr; + mutable std::mutex audio_measure_mutex; + Ebu_r128_proc r128; // Under audio_measure_mutex. + CorrelationMeasurer correlation; // Under audio_measure_mutex. + Resampler peak_resampler; // Under audio_measure_mutex. + std::atomic peak{0.0f}; }; #endif // !defined(_AUDIO_MIXER_H) diff --git a/mainwindow.cpp b/mainwindow.cpp index 5eb631a..28f0393 100644 --- a/mainwindow.cpp +++ b/mainwindow.cpp @@ -217,7 +217,7 @@ void MainWindow::mixer_created(Mixer *mixer) global_mixer->get_audio_mixer()->set_compressor_enabled(state == Qt::Checked); }); connect(ui->reset_meters_button, &QPushButton::clicked, this, &MainWindow::reset_meters_button_clicked); - mixer->set_audio_level_callback(bind(&MainWindow::audio_level_callback, this, _1, _2, _3, _4, _5, _6, _7, _8)); + mixer->get_audio_mixer()->set_audio_level_callback(bind(&MainWindow::audio_level_callback, this, _1, _2, _3, _4, _5, _6, _7, _8)); struct sigaction act; memset(&act, 0, sizeof(act)); @@ -391,7 +391,7 @@ void MainWindow::mini_fader_changed(Ui::AudioMiniView *ui, int channel, int valu void MainWindow::reset_meters_button_clicked() { - global_mixer->reset_meters(); + global_mixer->get_audio_mixer()->reset_meters(); ui->peak_display->setText(QString::fromStdString(format_db(-HUGE_VAL, DB_WITH_SIGN | DB_BARE))); ui->peak_display->setStyleSheet(""); } diff --git a/mixer.cpp b/mixer.cpp index 38578f9..00f161a 100644 --- a/mixer.cpp +++ b/mixer.cpp @@ -105,8 +105,7 @@ Mixer::Mixer(const QSurfaceFormat &format, unsigned num_cards) num_cards(num_cards), mixer_surface(create_surface(format)), h264_encoder_surface(create_surface(format)), - audio_mixer(num_cards), - correlation(OUTPUT_FREQUENCY) + audio_mixer(num_cards) { CHECK(init_movit(MOVIT_SHADER_DIR, MOVIT_DEBUG_OFF)); check_error(); @@ -232,13 +231,6 @@ Mixer::Mixer(const QSurfaceFormat &format, unsigned num_cards) cbcr_position_attribute_index = glGetAttribLocation(cbcr_program_num, "position"); cbcr_texcoord_attribute_index = glGetAttribLocation(cbcr_program_num, "texcoord"); - r128.init(2, OUTPUT_FREQUENCY); - r128.integr_start(); - - // hlen=16 is pretty low quality, but we use quite a bit of CPU otherwise, - // and there's a limit to how important the peak meter is. - peak_resampler.setup(OUTPUT_FREQUENCY, OUTPUT_FREQUENCY * 4, /*num_channels=*/2, /*hlen=*/16, /*frel=*/1.0); - if (global_flags.enable_alsa_output) { alsa.reset(new ALSAOutput(OUTPUT_FREQUENCY, /*num_channels=*/2)); } @@ -304,30 +296,6 @@ int unwrap_timecode(uint16_t current_wrapped, int last) } } -float find_peak(const float *samples, size_t num_samples) -{ - float m = fabs(samples[0]); - for (size_t i = 1; i < num_samples; ++i) { - m = max(m, fabs(samples[i])); - } - return m; -} - -void deinterleave_samples(const vector &in, vector *out_l, vector *out_r) -{ - size_t num_samples = in.size() / 2; - out_l->resize(num_samples); - out_r->resize(num_samples); - - const float *inptr = in.data(); - float *lptr = &(*out_l)[0]; - float *rptr = &(*out_r)[0]; - for (size_t i = 0; i < num_samples; ++i) { - *lptr++ = *inptr++; - *rptr++ = *inptr++; - } -} - } // namespace void Mixer::bm_frame(unsigned card_index, uint16_t timecode, @@ -582,7 +550,6 @@ void Mixer::thread_func() get_one_frame_from_each_card(master_card_index, new_frames, has_new_frame, num_samples); schedule_audio_resampling_tasks(new_frames[master_card_index].dropped_frames, num_samples[master_card_index], new_frames[master_card_index].length); stats_dropped_frames += new_frames[master_card_index].dropped_frames; - send_audio_level_callback(); handle_hotplugged_cards(); @@ -872,25 +839,6 @@ void Mixer::render_one_frame(int64_t duration) } } -void Mixer::send_audio_level_callback() -{ - if (audio_level_callback == nullptr) { - return; - } - - unique_lock lock(audio_measure_mutex); - double loudness_s = r128.loudness_S(); - double loudness_i = r128.integrated(); - double loudness_range_low = r128.range_min(); - double loudness_range_high = r128.range_max(); - - audio_level_callback(loudness_s, to_db(peak), - loudness_i, loudness_range_low, loudness_range_high, - audio_mixer.get_gain_staging_db(), - audio_mixer.get_final_makeup_gain_db(), - correlation.get_correlation()); -} - void Mixer::audio_thread_func() { while (!should_quit) { @@ -908,51 +856,14 @@ void Mixer::audio_thread_func() ResamplingQueue::RateAdjustmentPolicy rate_adjustment_policy = task.adjust_rate ? ResamplingQueue::ADJUST_RATE : ResamplingQueue::DO_NOT_ADJUST_RATE; - process_audio_one_frame(task.pts_int, task.num_samples, rate_adjustment_policy); - } -} - -void Mixer::process_audio_one_frame(int64_t frame_pts_int, int num_samples, ResamplingQueue::RateAdjustmentPolicy rate_adjustment_policy) -{ - vector samples_out = audio_mixer.get_output(double(frame_pts_int) / TIMEBASE, num_samples, rate_adjustment_policy); + vector samples_out = audio_mixer.get_output(double(task.pts_int) / TIMEBASE, task.num_samples, rate_adjustment_policy); - // Upsample 4x to find interpolated peak. - peak_resampler.inp_data = samples_out.data(); - peak_resampler.inp_count = samples_out.size() / 2; - - vector interpolated_samples_out; - interpolated_samples_out.resize(samples_out.size()); - { - unique_lock lock(audio_measure_mutex); - - while (peak_resampler.inp_count > 0) { // About four iterations. - peak_resampler.out_data = &interpolated_samples_out[0]; - peak_resampler.out_count = interpolated_samples_out.size() / 2; - peak_resampler.process(); - size_t out_stereo_samples = interpolated_samples_out.size() / 2 - peak_resampler.out_count; - peak = max(peak, find_peak(interpolated_samples_out.data(), out_stereo_samples * 2)); - peak_resampler.out_data = nullptr; + // Send the samples to the sound card, then add them to the output. + if (alsa) { + alsa->write(samples_out); } + video_encoder->add_audio(task.pts_int, move(samples_out)); } - - // Find R128 levels and L/R correlation. - vector left, right; - deinterleave_samples(samples_out, &left, &right); - float *ptrs[] = { left.data(), right.data() }; - { - unique_lock lock(audio_measure_mutex); - r128.process(left.size(), ptrs); - audio_mixer.set_current_loudness(r128.loudness_M()); - correlation.process_samples(samples_out); - } - - // Send the samples to the sound card. - if (alsa) { - alsa->write(samples_out); - } - - // And finally add them to the output. - video_encoder->add_audio(frame_pts_int, move(samples_out)); } void Mixer::subsample_chroma(GLuint src_tex, GLuint dst_tex) @@ -1048,16 +959,6 @@ void Mixer::channel_clicked(int preview_num) theme->channel_clicked(preview_num); } -void Mixer::reset_meters() -{ - unique_lock lock(audio_measure_mutex); - peak_resampler.reset(); - peak = 0.0f; - r128.reset(); - r128.integr_start(); - correlation.reset(); -} - void Mixer::start_mode_scanning(unsigned card_index) { assert(card_index < num_cards); diff --git a/mixer.h b/mixer.h index 354735d..5c69edd 100644 --- a/mixer.h +++ b/mixer.h @@ -11,7 +11,6 @@ #include #include #include -#include #include #include @@ -29,9 +28,7 @@ #include "alsa_output.h" #include "audio_mixer.h" #include "bmusb/bmusb.h" -#include "correlation_measurer.h" #include "defs.h" -#include "ebu_r128_proc.h" #include "httpd.h" #include "input_state.h" #include "pbo_frame_allocator.h" @@ -172,15 +169,6 @@ public: output_channel[output].set_color_updated_callback(callback); } - typedef std::function audio_level_callback_t; - void set_audio_level_callback(audio_level_callback_t callback) - { - audio_level_callback = callback; - } - std::vector get_transition_names() { return theme->get_transition_names(pts()); @@ -254,8 +242,6 @@ public: should_cut = true; } - void reset_meters(); - unsigned get_num_cards() const { return num_cards; } std::string get_card_description(unsigned card_index) const { @@ -326,9 +312,7 @@ private: void handle_hotplugged_cards(); void schedule_audio_resampling_tasks(unsigned dropped_frames, int num_samples_per_frame, int length_per_frame); void render_one_frame(int64_t duration); - void send_audio_level_callback(); void audio_thread_func(); - void process_audio_one_frame(int64_t frame_pts_int, int num_samples, ResamplingQueue::RateAdjustmentPolicy rate_adjustment_policy); void subsample_chroma(GLuint src_tex, GLuint dst_dst); void release_display_frame(DisplayFrame *frame); double pts() { return double(pts_int) / TIMEBASE; } @@ -426,13 +410,6 @@ private: std::atomic should_quit{false}; std::atomic should_cut{false}; - audio_level_callback_t audio_level_callback = nullptr; - mutable std::mutex audio_measure_mutex; - Ebu_r128_proc r128; // Under audio_measure_mutex. - CorrelationMeasurer correlation; // Under audio_measure_mutex. - Resampler peak_resampler; // Under audio_measure_mutex. - std::atomic peak{0.0f}; - std::unique_ptr alsa; struct AudioTask {