From: Steinar H. Gunderson Date: Sun, 11 Aug 2019 08:50:37 +0000 (+0200) Subject: Support delaying audio sources selectively. X-Git-Url: https://git.sesse.net/?p=nageru;a=commitdiff_plain;h=ff13a6a1cc56e97260c2be1ba0d29a435d46c1c3 Support delaying audio sources selectively. This can be useful if you have video sources with significant delay and wish to delay audio from other sources (e.g. ALSA) correspondingly. It works by simply changing the length of the resampling queue, from the default 100 ms. Inspired by a patch by Yoann Dubreuil in the Breizhcamp repository. --- diff --git a/nageru/audio_mixer.cpp b/nageru/audio_mixer.cpp index 360689b..a095c17 100644 --- a/nageru/audio_mixer.cpp +++ b/nageru/audio_mixer.cpp @@ -300,9 +300,13 @@ void AudioMixer::reset_resampler_mutex_held(DeviceSpec device_spec) if (device->interesting_channels.empty()) { device->resampling_queue.reset(); } else { + // Make sure we never get negative delay. Even 1 ms is probably way less than we + // could ever hope to actually have; this is just a failsafe. + double delay_ms = max(global_flags.audio_queue_length_ms + device->extra_delay_ms, 1.0); + device->resampling_queue.reset(new ResamplingQueue( device_spec, device->capture_frequency, OUTPUT_FREQUENCY, device->interesting_channels.size(), - global_flags.audio_queue_length_ms * 0.001)); + delay_ms * 0.001)); } } @@ -1213,11 +1217,14 @@ void AudioMixer::set_input_mapping_lock_held(const InputMapping &new_input_mappi } // Reset resamplers for all cards that don't have the exact same state as before. + map new_extra_delay_ms = new_input_mapping.extra_delay_ms; // Convenience so we can use []. for (unsigned card_index = 0; card_index < MAX_VIDEO_CARDS; ++card_index) { const DeviceSpec device_spec{InputSourceType::CAPTURE_CARD, card_index}; AudioDevice *device = find_audio_device(device_spec); - if (device->interesting_channels != interesting_channels[device_spec]) { + if (device->interesting_channels != interesting_channels[device_spec] || + device->extra_delay_ms != new_extra_delay_ms[device_spec]) { device->interesting_channels = interesting_channels[device_spec]; + device->extra_delay_ms = new_extra_delay_ms[device_spec]; reset_resampler_mutex_held(device_spec); } } @@ -1229,8 +1236,10 @@ void AudioMixer::set_input_mapping_lock_held(const InputMapping &new_input_mappi } else { alsa_pool.hold_device(card_index); } - if (device->interesting_channels != interesting_channels[device_spec]) { + if (device->interesting_channels != interesting_channels[device_spec] || + device->extra_delay_ms != new_extra_delay_ms[device_spec]) { device->interesting_channels = interesting_channels[device_spec]; + device->extra_delay_ms = new_extra_delay_ms[device_spec]; alsa_pool.reset_device(device_spec.index); reset_resampler_mutex_held(device_spec); } @@ -1238,8 +1247,10 @@ void AudioMixer::set_input_mapping_lock_held(const InputMapping &new_input_mappi for (unsigned card_index = 0; card_index < num_ffmpeg_inputs; ++card_index) { const DeviceSpec device_spec{InputSourceType::FFMPEG_VIDEO_INPUT, card_index}; AudioDevice *device = find_audio_device(device_spec); - if (device->interesting_channels != interesting_channels[device_spec]) { + if (device->interesting_channels != interesting_channels[device_spec] || + device->extra_delay_ms != new_extra_delay_ms[device_spec]) { device->interesting_channels = interesting_channels[device_spec]; + device->extra_delay_ms = new_extra_delay_ms[device_spec]; reset_resampler_mutex_held(device_spec); } } diff --git a/nageru/audio_mixer.h b/nageru/audio_mixer.h index 14e7e85..de6ff54 100644 --- a/nageru/audio_mixer.h +++ b/nageru/audio_mixer.h @@ -324,6 +324,10 @@ private: // Which channels we consider interesting (ie., are part of some input_mapping). std::set interesting_channels; bool silenced = false; + + // Positive means the audio is delayed, negative means we try to have it earlier + // (although we can't time-travel!). Stored together with the input mapping. + double extra_delay_ms = 0.0; }; const AudioDevice *find_audio_device(DeviceSpec device_spec) const diff --git a/nageru/input_mapping.cpp b/nageru/input_mapping.cpp index dfcc97c..e41f995 100644 --- a/nageru/input_mapping.cpp +++ b/nageru/input_mapping.cpp @@ -44,7 +44,13 @@ bool save_input_mapping_to_file(const map &devices, cons for (const InputMapping::Bus &bus : input_mapping.buses) { if (!used_devices.count(bus.device)) { used_devices.emplace(bus.device, used_devices.size()); - global_audio_mixer->serialize_device(bus.device, mapping_proto.add_device()); + DeviceSpecProto *device_proto = mapping_proto.add_device(); + global_audio_mixer->serialize_device(bus.device, device_proto); + + const auto delay_it = input_mapping.extra_delay_ms.find(bus.device); + if (delay_it != input_mapping.extra_delay_ms.end()) { + device_proto->set_extra_delay_ms(delay_it->second); + } } BusProto *bus_proto = mapping_proto.add_bus(); @@ -175,6 +181,7 @@ found_alsa_input: default: assert(false); } + new_mapping->extra_delay_ms.emplace(device_mapping.back(), device_proto.extra_delay_ms()); } for (const BusProto &bus_proto : mapping_proto.bus()) { diff --git a/nageru/input_mapping.h b/nageru/input_mapping.h index 67af0f4..6f5a42a 100644 --- a/nageru/input_mapping.h +++ b/nageru/input_mapping.h @@ -45,6 +45,7 @@ struct InputMapping { }; std::vector buses; + std::map extra_delay_ms; }; // This is perhaps not the most user-friendly output, but it's at least better diff --git a/nageru/input_mapping.ui b/nageru/input_mapping.ui index 4487b94..bec59de 100644 --- a/nageru/input_mapping.ui +++ b/nageru/input_mapping.ui @@ -36,6 +36,11 @@ Right input + + + Extra delay to add (ms) + + diff --git a/nageru/input_mapping_dialog.cpp b/nageru/input_mapping_dialog.cpp index b456515..529b2e5 100644 --- a/nageru/input_mapping_dialog.cpp +++ b/nageru/input_mapping_dialog.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -41,6 +42,7 @@ InputMappingDialog::InputMappingDialog() ui->setupUi(this); ui->table->setSelectionBehavior(QAbstractItemView::SelectRows); ui->table->setSelectionMode(QAbstractItemView::SingleSelection); // Makes implementing moving easier for now. + ui->table->setColumnWidth(4, 200); fill_ui_from_mapping(mapping); connect(ui->table, &QTableWidget::cellChanged, this, &InputMappingDialog::cell_changed); @@ -61,7 +63,7 @@ InputMappingDialog::InputMappingDialog() post_to_main_thread([this]{ devices = global_audio_mixer->get_devices(); for (unsigned row = 0; row < mapping.buses.size(); ++row) { - fill_row_from_bus(row, mapping.buses[row]); + fill_row_from_bus(row, mapping.buses[row], mapping); } }); }); @@ -82,11 +84,11 @@ void InputMappingDialog::fill_ui_from_mapping(const InputMapping &mapping) ui->table->setRowCount(mapping.buses.size()); for (unsigned row = 0; row < mapping.buses.size(); ++row) { - fill_row_from_bus(row, mapping.buses[row]); + fill_row_from_bus(row, mapping.buses[row], mapping); } } -void InputMappingDialog::fill_row_from_bus(unsigned row, const InputMapping::Bus &bus) +void InputMappingDialog::fill_row_from_bus(unsigned row, const InputMapping::Bus &bus, const InputMapping &mapping) { QString name(QString::fromStdString(bus.name)); ui->table->setItem(row, 0, new QTableWidgetItem(name)); @@ -136,6 +138,7 @@ void InputMappingDialog::fill_row_from_bus(unsigned row, const InputMapping::Bus bind(&InputMappingDialog::card_selected, this, card_combo, row, _1)); ui->table->setCellWidget(row, 1, card_combo); + fill_delay_from_bus(row, bus, mapping); setup_channel_choices_from_bus(row, bus); } @@ -169,6 +172,34 @@ void InputMappingDialog::setup_channel_choices_from_bus(unsigned row, const Inpu } } +void InputMappingDialog::fill_delay_from_bus(unsigned row, const InputMapping::Bus &bus, const InputMapping &mapping) +{ + QDoubleSpinBox *delay_spinner = static_cast(ui->table->cellWidget(row, 4)); + if (delay_spinner == nullptr) { + delay_spinner = new QDoubleSpinBox; + delay_spinner->setDecimals(1); + delay_spinner->setMinimum(-50.0); + delay_spinner->setMaximum(500.0); + connect(delay_spinner, static_cast(&QDoubleSpinBox::valueChanged), + bind(&InputMappingDialog::delay_changed, this, row, _1)); + } + delay_spinner->blockSignals(true); + if (bus.device.type == InputSourceType::SILENCE) { + delay_spinner->setEnabled(false); + delay_spinner->setValue(0.0); + } else { + delay_spinner->setEnabled(true); + const auto delay_it = mapping.extra_delay_ms.find(bus.device); + if (delay_it != mapping.extra_delay_ms.end()) { + delay_spinner->setValue(delay_it->second); + } else { + delay_spinner->setValue(0.0); + } + } + delay_spinner->blockSignals(false); + ui->table->setCellWidget(row, 4, delay_spinner); +} + void InputMappingDialog::ok_clicked() { global_audio_mixer->set_state_changed_callback(saved_callback); @@ -200,6 +231,7 @@ void InputMappingDialog::card_selected(QComboBox *card_combo, unsigned row, int { uint64_t key = card_combo->itemData(index).toULongLong(); mapping.buses[row].device = key_to_DeviceSpec(key); + fill_delay_from_bus(row, mapping.buses[row], mapping); setup_channel_choices_from_bus(row, mapping.buses[row]); } @@ -208,6 +240,17 @@ void InputMappingDialog::channel_selected(unsigned row, unsigned channel, int in mapping.buses[row].source_channel[channel] = index - 1; } +void InputMappingDialog::delay_changed(unsigned row, double value) +{ + mapping.extra_delay_ms[mapping.buses[row].device] = value; + + for (unsigned other_row = 0; other_row < mapping.buses.size(); ++other_row) { + if (row != other_row && mapping.buses[row].device == mapping.buses[other_row].device) { + fill_delay_from_bus(other_row, mapping.buses[other_row], mapping); + } + } +} + void InputMappingDialog::add_clicked() { QTableWidgetSelectionRange all(0, 0, ui->table->rowCount() - 1, ui->table->columnCount() - 1); @@ -221,7 +264,7 @@ void InputMappingDialog::add_clicked() ui->table->setRowCount(mapping.buses.size()); unsigned row = mapping.buses.size() - 1; - fill_row_from_bus(row, new_bus); + fill_row_from_bus(row, new_bus, mapping); ui->table->editItem(ui->table->item(row, 0)); // Start editing the name. update_button_state(); } @@ -257,8 +300,8 @@ void InputMappingDialog::updown_clicked(int direction) swap(mapping.buses[a_row], mapping.buses[b_row]); swap(bus_settings[a_row], bus_settings[b_row]); - fill_row_from_bus(a_row, mapping.buses[a_row]); - fill_row_from_bus(b_row, mapping.buses[b_row]); + fill_row_from_bus(a_row, mapping.buses[a_row], mapping); + fill_row_from_bus(b_row, mapping.buses[b_row], mapping); QTableWidgetSelectionRange a_sel(a_row, 0, a_row, ui->table->columnCount() - 1); QTableWidgetSelectionRange b_sel(b_row, 0, b_row, ui->table->columnCount() - 1); diff --git a/nageru/input_mapping_dialog.h b/nageru/input_mapping_dialog.h index 640644e..8873850 100644 --- a/nageru/input_mapping_dialog.h +++ b/nageru/input_mapping_dialog.h @@ -27,11 +27,13 @@ public: private: void fill_ui_from_mapping(const InputMapping &mapping); - void fill_row_from_bus(unsigned row, const InputMapping::Bus &bus); + void fill_row_from_bus(unsigned row, const InputMapping::Bus &bus, const InputMapping &mapping); + void fill_delay_from_bus(unsigned row, const InputMapping::Bus &bus, const InputMapping &mapping); void setup_channel_choices_from_bus(unsigned row, const InputMapping::Bus &bus); void cell_changed(int row, int column); void card_selected(QComboBox *card_combo, unsigned row, int index); void channel_selected(unsigned row, unsigned channel, int index); + void delay_changed(unsigned row, double value); void ok_clicked(); void cancel_clicked(); void add_clicked(); diff --git a/nageru/state.proto b/nageru/state.proto index 6372e61..a7df37c 100644 --- a/nageru/state.proto +++ b/nageru/state.proto @@ -18,6 +18,11 @@ message DeviceSpecProto { optional string alsa_info = 5; // Only for ALSA devices. optional int32 num_channels = 6; // Only for ALSA devices. optional string address = 7; // Only for ALSA devices. + + // This isn't strictly part of the _mapping_, but it fits in fairly naturally. + // Positive means the audio is delayed, negative means we try to have it earlier + // (although we can't time-travel!). + optional float extra_delay_ms = 8; } // Corresponds to InputMapping::Bus.