X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=audio_mixer.cpp;h=468edd531859b75793750564bc6a29f433a1675b;hb=b9feb66845bf24465b7671ac9ff7a52b88f6032b;hp=506e78d50ede72820e08ba1acfecb6d6dbad8411;hpb=e5b9eff03d96d79c93a33d354b51c1afe2f44d59;p=nageru diff --git a/audio_mixer.cpp b/audio_mixer.cpp index 506e78d..468edd5 100644 --- a/audio_mixer.cpp +++ b/audio_mixer.cpp @@ -1,24 +1,32 @@ #include "audio_mixer.h" #include -#include #include -#include #include -#include -#include -#ifdef __SSE__ +#include +#ifdef __SSE2__ #include #endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include "db.h" #include "flags.h" -#include "mixer.h" +#include "metrics.h" #include "state.pb.h" #include "timebase.h" using namespace bmusb; using namespace std; +using namespace std::chrono; using namespace std::placeholders; namespace { @@ -164,8 +172,6 @@ AudioMixer::AudioMixer(unsigned num_cards) limiter(OUTPUT_FREQUENCY), correlation(OUTPUT_FREQUENCY) { - global_audio_mixer = this; - for (unsigned bus_index = 0; bus_index < MAX_BUSES; ++bus_index) { locut[bus_index].init(FILTER_HPF, 2); eq[bus_index][EQ_BAND_BASS].init(FILTER_LOW_SHELF, 1); @@ -178,9 +184,19 @@ AudioMixer::AudioMixer(unsigned num_cards) } set_limiter_enabled(global_flags.limiter_enabled); set_final_makeup_gain_auto(global_flags.final_makeup_gain_auto); + + r128.init(2, OUTPUT_FREQUENCY); + r128.integr_start(); + + // hlen=16 is pretty low quality, but we use quite a bit of CPU otherwise, + // and there's a limit to how important the peak meter is. + peak_resampler.setup(OUTPUT_FREQUENCY, OUTPUT_FREQUENCY * 4, /*num_channels=*/2, /*hlen=*/16, /*frel=*/1.0); + + global_audio_mixer = this; alsa_pool.init(); if (!global_flags.input_mapping_filename.empty()) { + // Must happen after ALSAPool is initialized, as it needs to know the card list. current_mapping_mode = MappingMode::MULTICHANNEL; InputMapping new_input_mapping; if (!load_input_mapping_from_file(get_devices(), @@ -198,12 +214,13 @@ AudioMixer::AudioMixer(unsigned num_cards) } } - r128.init(2, OUTPUT_FREQUENCY); - r128.integr_start(); - - // hlen=16 is pretty low quality, but we use quite a bit of CPU otherwise, - // and there's a limit to how important the peak meter is. - peak_resampler.setup(OUTPUT_FREQUENCY, OUTPUT_FREQUENCY * 4, /*num_channels=*/2, /*hlen=*/16, /*frel=*/1.0); + global_metrics.add("audio_loudness_short_lufs", &metric_audio_loudness_short_lufs, Metrics::TYPE_GAUGE); + global_metrics.add("audio_loudness_integrated_lufs", &metric_audio_loudness_integrated_lufs, Metrics::TYPE_GAUGE); + global_metrics.add("audio_loudness_range_low_lufs", &metric_audio_loudness_range_low_lufs, Metrics::TYPE_GAUGE); + global_metrics.add("audio_loudness_range_high_lufs", &metric_audio_loudness_range_high_lufs, Metrics::TYPE_GAUGE); + global_metrics.add("audio_peak_dbfs", &metric_audio_peak_dbfs, Metrics::TYPE_GAUGE); + global_metrics.add("audio_final_makeup_gain_db", &metric_audio_final_makeup_gain_db, Metrics::TYPE_GAUGE); + global_metrics.add("audio_correlation", &metric_audio_correlation, Metrics::TYPE_GAUGE); } void AudioMixer::reset_resampler(DeviceSpec device_spec) @@ -221,12 +238,13 @@ void AudioMixer::reset_resampler_mutex_held(DeviceSpec device_spec) } else { // TODO: ResamplingQueue should probably take the full device spec. // (It's only used for console output, though.) - device->resampling_queue.reset(new ResamplingQueue(device_spec.index, device->capture_frequency, OUTPUT_FREQUENCY, device->interesting_channels.size())); + device->resampling_queue.reset(new ResamplingQueue( + device_spec.index, device->capture_frequency, OUTPUT_FREQUENCY, device->interesting_channels.size(), + global_flags.audio_queue_length_ms * 0.001)); } - device->next_local_pts = 0; } -bool AudioMixer::add_audio(DeviceSpec device_spec, const uint8_t *data, unsigned num_samples, AudioFormat audio_format, int64_t frame_length) +bool AudioMixer::add_audio(DeviceSpec device_spec, const uint8_t *data, unsigned num_samples, AudioFormat audio_format, int64_t frame_length, steady_clock::time_point frame_time) { AudioDevice *device = find_audio_device(device_spec); @@ -265,10 +283,14 @@ bool AudioMixer::add_audio(DeviceSpec device_spec, const uint8_t *data, unsigned } } + // If we changed frequency since last frame, we'll need to reset the resampler. + if (audio_format.sample_rate != device->capture_frequency) { + device->capture_frequency = audio_format.sample_rate; + reset_resampler_mutex_held(device_spec); + } + // Now add it. - int64_t local_pts = device->next_local_pts; - device->resampling_queue->add_input_samples(local_pts / double(TIMEBASE), audio.get(), num_samples); - device->next_local_pts = local_pts + frame_length; + device->resampling_queue->add_input_samples(frame_time, audio.get(), num_samples, ResamplingQueue::ADJUST_RATE); return true; } @@ -290,11 +312,7 @@ bool AudioMixer::add_silence(DeviceSpec device_spec, unsigned samples_per_frame, vector silence(samples_per_frame * num_channels, 0.0f); for (unsigned i = 0; i < num_frames; ++i) { - device->resampling_queue->add_input_samples(device->next_local_pts / double(TIMEBASE), silence.data(), samples_per_frame); - // Note that if the format changed in the meantime, we have - // no way of detecting that; we just have to assume the frame length - // is always the same. - device->next_local_pts += frame_length; + device->resampling_queue->add_input_samples(steady_clock::now(), silence.data(), samples_per_frame, ResamplingQueue::DO_NOT_ADJUST_RATE); } return true; } @@ -319,6 +337,7 @@ AudioMixer::BusSettings AudioMixer::get_default_bus_settings() { BusSettings settings; settings.fader_volume_db = 0.0f; + settings.muted = false; settings.locut_enabled = global_flags.locut_enabled; for (unsigned band_index = 0; band_index < NUM_EQ_BANDS; ++band_index) { settings.eq_level_db[band_index] = 0.0f; @@ -335,6 +354,7 @@ AudioMixer::BusSettings AudioMixer::get_bus_settings(unsigned bus_index) const lock_guard lock(audio_mutex); BusSettings settings; settings.fader_volume_db = fader_volume_db[bus_index]; + settings.muted = mute[bus_index]; settings.locut_enabled = locut_enabled[bus_index]; for (unsigned band_index = 0; band_index < NUM_EQ_BANDS; ++band_index) { settings.eq_level_db[band_index] = eq_level_db[bus_index][band_index]; @@ -350,11 +370,13 @@ void AudioMixer::set_bus_settings(unsigned bus_index, const AudioMixer::BusSetti { lock_guard lock(audio_mutex); fader_volume_db[bus_index] = settings.fader_volume_db; + mute[bus_index] = settings.muted; locut_enabled[bus_index] = settings.locut_enabled; for (unsigned band_index = 0; band_index < NUM_EQ_BANDS; ++band_index) { eq_level_db[bus_index][band_index] = settings.eq_level_db[band_index]; } gain_staging_db[bus_index] = settings.gain_staging_db; + last_gain_staging_db[bus_index] = gain_staging_db[bus_index]; level_compressor_enabled[bus_index] = settings.level_compressor_enabled; compressor_threshold_dbfs[bus_index] = settings.compressor_threshold_dbfs; compressor_enabled[bus_index] = settings.compressor_enabled; @@ -402,7 +424,7 @@ void AudioMixer::find_sample_src_from_device(const map void AudioMixer::fill_audio_bus(const map> &samples_card, const InputMapping::Bus &bus, unsigned num_samples, float *output) { if (bus.device.type == InputSourceType::SILENCE) { - memset(output, 0, num_samples * sizeof(*output)); + memset(output, 0, num_samples * 2 * sizeof(*output)); } else { assert(bus.device.type == InputSourceType::CAPTURE_CARD || bus.device.type == InputSourceType::ALSA_INPUT); @@ -438,7 +460,32 @@ vector AudioMixer::get_active_devices() const return ret; } -vector AudioMixer::get_output(double pts, unsigned num_samples, ResamplingQueue::RateAdjustmentPolicy rate_adjustment_policy) +namespace { + +void apply_gain(float db, float last_db, vector *samples) +{ + if (fabs(db - last_db) < 1e-3) { + // Constant over this frame. + const float gain = from_db(db); + for (size_t i = 0; i < samples->size(); ++i) { + (*samples)[i] *= gain; + } + } else { + // We need to do a fade. + unsigned num_samples = samples->size() / 2; + float gain = from_db(last_db); + const float gain_inc = pow(from_db(db - last_db), 1.0 / num_samples); + for (size_t i = 0; i < num_samples; ++i) { + (*samples)[i * 2 + 0] *= gain; + (*samples)[i * 2 + 1] *= gain; + gain *= gain_inc; + } + } +} + +} // namespace + +vector AudioMixer::get_output(steady_clock::time_point ts, unsigned num_samples, ResamplingQueue::RateAdjustmentPolicy rate_adjustment_policy) { map> samples_card; vector samples_bus; @@ -453,7 +500,7 @@ vector AudioMixer::get_output(double pts, unsigned num_samples, Resamplin memset(&samples_card[device_spec][0], 0, samples_card[device_spec].size() * sizeof(float)); } else { device->resampling_queue->get_output_samples( - pts, + ts, &samples_card[device_spec][0], num_samples, rate_adjustment_policy); @@ -486,11 +533,11 @@ vector AudioMixer::get_output(double pts, unsigned num_samples, Resamplin gain_staging_db[bus_index] = to_db(level_compressor[bus_index]->get_attenuation() * makeup_gain); } else { // Just apply the gain we already had. - float g = from_db(gain_staging_db[bus_index]); - for (size_t i = 0; i < samples_bus.size(); ++i) { - samples_bus[i] *= g; - } + float db = gain_staging_db[bus_index]; + float last_db = last_gain_staging_db[bus_index]; + apply_gain(db, last_db, &samples_bus); } + last_gain_staging_db[bus_index] = gain_staging_db[bus_index]; #if 0 printf("level=%f (%+5.2f dBFS) attenuation=%f (%+5.2f dB) end_result=%+5.2f dB\n", @@ -548,13 +595,12 @@ vector AudioMixer::get_output(double pts, unsigned num_samples, Resamplin // (half-time of 30 seconds). double target_loudness_factor, alpha; double loudness_lu = r128.loudness_M() - ref_level_lufs; - double current_makeup_lu = to_db(final_makeup_gain); target_loudness_factor = final_makeup_gain * from_db(-loudness_lu); - // If we're outside +/- 5 LU uncorrected, we don't count it as + // If we're outside +/- 5 LU (after correction), we don't count it as // a normal signal (probably silence) and don't change the // correction factor; just apply what we already have. - if (fabs(loudness_lu - current_makeup_lu) >= 5.0 || !final_makeup_gain_auto) { + if (fabs(loudness_lu) >= 5.0 || !final_makeup_gain_auto) { alpha = 0.0; } else { // Formula adapted from @@ -642,22 +688,7 @@ void AudioMixer::apply_eq(unsigned bus_index, vector *samples_bus) assert(samples_bus->size() % 2 == 0); const unsigned num_samples = samples_bus->size() / 2; - if (fabs(mid_db - last_mid_db) < 1e-3) { - // Constant over this frame. - const float gain = from_db(mid_db); - for (size_t i = 0; i < samples_bus->size(); ++i) { - (*samples_bus)[i] *= gain; - } - } else { - // We need to do a fade. - float gain = from_db(last_mid_db); - const float gain_inc = pow(from_db(mid_db - last_mid_db), 1.0 / num_samples); - for (size_t i = 0; i < num_samples; ++i) { - (*samples_bus)[i * 2 + 0] *= gain; - (*samples_bus)[i * 2 + 1] *= gain; - gain *= gain_inc; - } - } + apply_gain(mid_db, last_mid_db, samples_bus); apply_filter_fade(&eq[bus_index][EQ_BAND_BASS], samples_bus->data(), num_samples, bass_freq_hz, bass_db - mid_db, last_bass_db - last_mid_db); apply_filter_fade(&eq[bus_index][EQ_BAND_TREBLE], samples_bus->data(), num_samples, treble_freq_hz, treble_db - mid_db, last_treble_db - last_mid_db); @@ -672,13 +703,14 @@ void AudioMixer::add_bus_to_master(unsigned bus_index, const vector &samp assert(samples_bus.size() == samples_out->size()); assert(samples_bus.size() % 2 == 0); unsigned num_samples = samples_bus.size() / 2; - if (fabs(fader_volume_db[bus_index] - last_fader_volume_db[bus_index]) > 1e-3) { + const float new_volume_db = mute[bus_index] ? -90.0f : fader_volume_db[bus_index].load(); + if (fabs(new_volume_db - last_fader_volume_db[bus_index]) > 1e-3) { // The volume has changed; do a fade over the course of this frame. // (We might have some numerical issues here, but it seems to sound OK.) // For the purpose of fading here, the silence floor is set to -90 dB // (the fader only goes to -84). float old_volume = from_db(max(last_fader_volume_db[bus_index], -90.0f)); - float volume = from_db(max(fader_volume_db[bus_index], -90.0f)); + float volume = from_db(max(new_volume_db, -90.0f)); float volume_inc = pow(volume / old_volume, 1.0 / num_samples); volume = old_volume; @@ -695,8 +727,8 @@ void AudioMixer::add_bus_to_master(unsigned bus_index, const vector &samp volume *= volume_inc; } } - } else { - float volume = from_db(fader_volume_db[bus_index]); + } else if (new_volume_db > -90.0f) { + float volume = from_db(new_volume_db); if (bus_index == 0) { for (unsigned i = 0; i < num_samples; ++i) { (*samples_out)[i * 2 + 0] = samples_bus[i * 2 + 0] * volume; @@ -710,13 +742,13 @@ void AudioMixer::add_bus_to_master(unsigned bus_index, const vector &samp } } - last_fader_volume_db[bus_index] = fader_volume_db[bus_index]; + last_fader_volume_db[bus_index] = new_volume_db; } void AudioMixer::measure_bus_levels(unsigned bus_index, const vector &left, const vector &right) { assert(left.size() == right.size()); - const float volume = from_db(fader_volume_db[bus_index]); + const float volume = mute[bus_index] ? 0.0f : from_db(fader_volume_db[bus_index]); const float peak_levels[2] = { find_peak(left.data(), left.size()) * volume, find_peak(right.data(), right.size()) * volume @@ -804,23 +836,35 @@ void AudioMixer::send_audio_level_callback() double loudness_range_low = r128.range_min(); double loudness_range_high = r128.range_max(); + metric_audio_loudness_short_lufs = loudness_s; + metric_audio_loudness_integrated_lufs = loudness_i; + metric_audio_loudness_range_low_lufs = loudness_range_low; + metric_audio_loudness_range_high_lufs = loudness_range_high; + metric_audio_peak_dbfs = to_db(peak); + metric_audio_final_makeup_gain_db = to_db(final_makeup_gain); + metric_audio_correlation = correlation.get_correlation(); + vector bus_levels; bus_levels.resize(input_mapping.buses.size()); { lock_guard lock(compressor_mutex); for (unsigned bus_index = 0; bus_index < bus_levels.size(); ++bus_index) { - bus_levels[bus_index].current_level_dbfs[0] = to_db(peak_history[bus_index][0].current_level); - bus_levels[bus_index].current_level_dbfs[1] = to_db(peak_history[bus_index][1].current_level); - bus_levels[bus_index].peak_level_dbfs[0] = to_db(peak_history[bus_index][0].current_peak); - bus_levels[bus_index].peak_level_dbfs[1] = to_db(peak_history[bus_index][1].current_peak); - bus_levels[bus_index].historic_peak_dbfs = to_db( + BusLevel &levels = bus_levels[bus_index]; + BusMetrics &metrics = bus_metrics[bus_index]; + + levels.current_level_dbfs[0] = metrics.current_level_dbfs[0] = to_db(peak_history[bus_index][0].current_level); + levels.current_level_dbfs[1] = metrics.current_level_dbfs[1] = to_db(peak_history[bus_index][1].current_level); + levels.peak_level_dbfs[0] = metrics.peak_level_dbfs[0] = to_db(peak_history[bus_index][0].current_peak); + levels.peak_level_dbfs[1] = metrics.peak_level_dbfs[1] = to_db(peak_history[bus_index][1].current_peak); + levels.historic_peak_dbfs = metrics.historic_peak_dbfs = to_db( max(peak_history[bus_index][0].historic_peak, peak_history[bus_index][1].historic_peak)); - bus_levels[bus_index].gain_staging_db = gain_staging_db[bus_index]; + levels.gain_staging_db = metrics.gain_staging_db = gain_staging_db[bus_index]; if (compressor_enabled[bus_index]) { - bus_levels[bus_index].compressor_attenuation_db = -to_db(compressor[bus_index]->get_attenuation()); + levels.compressor_attenuation_db = metrics.compressor_attenuation_db = -to_db(compressor[bus_index]->get_attenuation()); } else { - bus_levels[bus_index].compressor_attenuation_db = 0.0; + levels.compressor_attenuation_db = 0.0; + metrics.compressor_attenuation_db = 0.0 / 0.0; } } } @@ -943,6 +987,62 @@ void AudioMixer::set_input_mapping_lock_held(const InputMapping &new_input_mappi } } + // Kill all the old metrics, and set up new ones. + for (unsigned bus_index = 0; bus_index < input_mapping.buses.size(); ++bus_index) { + BusMetrics &metrics = bus_metrics[bus_index]; + + vector> labels_left = metrics.labels; + labels_left.emplace_back("channel", "left"); + vector> labels_right = metrics.labels; + labels_right.emplace_back("channel", "right"); + + global_metrics.remove("bus_current_level_dbfs", labels_left); + global_metrics.remove("bus_current_level_dbfs", labels_right); + global_metrics.remove("bus_peak_level_dbfs", labels_left); + global_metrics.remove("bus_peak_level_dbfs", labels_right); + global_metrics.remove("bus_historic_peak_dbfs", metrics.labels); + global_metrics.remove("bus_gain_staging_db", metrics.labels); + global_metrics.remove("bus_compressor_attenuation_db", metrics.labels); + } + bus_metrics.reset(new BusMetrics[new_input_mapping.buses.size()]); + for (unsigned bus_index = 0; bus_index < new_input_mapping.buses.size(); ++bus_index) { + const InputMapping::Bus &bus = new_input_mapping.buses[bus_index]; + BusMetrics &metrics = bus_metrics[bus_index]; + + char bus_index_str[16], source_index_str[16], source_channels_str[64]; + snprintf(bus_index_str, sizeof(bus_index_str), "%u", bus_index); + snprintf(source_index_str, sizeof(source_index_str), "%u", bus.device.index); + snprintf(source_channels_str, sizeof(source_channels_str), "%d:%d", bus.source_channel[0], bus.source_channel[1]); + + vector> labels; + metrics.labels.emplace_back("index", bus_index_str); + metrics.labels.emplace_back("name", bus.name); + if (bus.device.type == InputSourceType::SILENCE) { + metrics.labels.emplace_back("source_type", "silence"); + } else if (bus.device.type == InputSourceType::CAPTURE_CARD) { + metrics.labels.emplace_back("source_type", "capture_card"); + } else if (bus.device.type == InputSourceType::ALSA_INPUT) { + metrics.labels.emplace_back("source_type", "alsa_input"); + } else { + assert(false); + } + metrics.labels.emplace_back("source_index", source_index_str); + metrics.labels.emplace_back("source_channels", source_channels_str); + + vector> labels_left = metrics.labels; + labels_left.emplace_back("channel", "left"); + vector> labels_right = metrics.labels; + labels_right.emplace_back("channel", "right"); + + global_metrics.add("bus_current_level_dbfs", labels_left, &metrics.current_level_dbfs[0], Metrics::TYPE_GAUGE); + global_metrics.add("bus_current_level_dbfs", labels_right, &metrics.current_level_dbfs[1], Metrics::TYPE_GAUGE); + global_metrics.add("bus_peak_level_dbfs", labels_left, &metrics.peak_level_dbfs[0], Metrics::TYPE_GAUGE); + global_metrics.add("bus_peak_level_dbfs", labels_right, &metrics.peak_level_dbfs[1], Metrics::TYPE_GAUGE); + global_metrics.add("bus_historic_peak_dbfs", metrics.labels, &metrics.historic_peak_dbfs, Metrics::TYPE_GAUGE); + global_metrics.add("bus_gain_staging_db", metrics.labels, &metrics.gain_staging_db, Metrics::TYPE_GAUGE); + global_metrics.add("bus_compressor_attenuation_db", metrics.labels, &metrics.compressor_attenuation_db, Metrics::TYPE_GAUGE); + } + // Reset resamplers for all cards that don't have the exact same state as before. for (unsigned card_index = 0; card_index < MAX_VIDEO_CARDS; ++card_index) { const DeviceSpec device_spec{InputSourceType::CAPTURE_CARD, card_index}; @@ -976,6 +1076,12 @@ InputMapping AudioMixer::get_input_mapping() const return input_mapping; } +unsigned AudioMixer::num_buses() const +{ + lock_guard lock(audio_mutex); + return input_mapping.buses.size(); +} + void AudioMixer::reset_peak(unsigned bus_index) { lock_guard lock(audio_mutex);