Support delaying audio sources selectively.

This can be useful if you have video sources with significant delay
and wish to delay audio from other sources (e.g. ALSA) correspondingly.
It works by simply changing the length of the resampling queue,
from the default 100 ms.

Inspired by a patch by Yoann Dubreuil in the Breizhcamp repository.

diff --git a/nageru/audio_mixer.cpp b/nageru/audio_mixer.cpp
index 360689b83406b04e6737cb6b7815d4092aca0a90..a095c177d6265467922f2bc58c4033c5da9cd71e 100644 (file)
--- a/nageru/audio_mixer.cpp
+++ b/nageru/audio_mixer.cpp
@@ -300,9 +300,13 @@ void AudioMixer::reset_resampler_mutex_held(DeviceSpec device_spec)
        if (device->interesting_channels.empty()) {
                device->resampling_queue.reset();
        } else {
+               // Make sure we never get negative delay. Even 1 ms is probably way less than we
+               // could ever hope to actually have; this is just a failsafe.
+               double delay_ms = max(global_flags.audio_queue_length_ms + device->extra_delay_ms, 1.0);
+
                device->resampling_queue.reset(new ResamplingQueue(
                        device_spec, device->capture_frequency, OUTPUT_FREQUENCY, device->interesting_channels.size(),
-                       global_flags.audio_queue_length_ms * 0.001));
+                       delay_ms * 0.001));
        }
 }
 
@@ -1213,11 +1217,14 @@ void AudioMixer::set_input_mapping_lock_held(const InputMapping &new_input_mappi
        }
 
        // Reset resamplers for all cards that don't have the exact same state as before.
+       map<DeviceSpec, double> new_extra_delay_ms = new_input_mapping.extra_delay_ms;  // Convenience so we can use [].
        for (unsigned card_index = 0; card_index < MAX_VIDEO_CARDS; ++card_index) {
                const DeviceSpec device_spec{InputSourceType::CAPTURE_CARD, card_index};
                AudioDevice *device = find_audio_device(device_spec);
-               if (device->interesting_channels != interesting_channels[device_spec]) {
+               if (device->interesting_channels != interesting_channels[device_spec] ||
+                   device->extra_delay_ms != new_extra_delay_ms[device_spec]) {
                        device->interesting_channels = interesting_channels[device_spec];
+                       device->extra_delay_ms = new_extra_delay_ms[device_spec];
                        reset_resampler_mutex_held(device_spec);
                }
        }
@@ -1229,8 +1236,10 @@ void AudioMixer::set_input_mapping_lock_held(const InputMapping &new_input_mappi
                } else {
                        alsa_pool.hold_device(card_index);
                }
-               if (device->interesting_channels != interesting_channels[device_spec]) {
+               if (device->interesting_channels != interesting_channels[device_spec] ||
+                   device->extra_delay_ms != new_extra_delay_ms[device_spec]) {
                        device->interesting_channels = interesting_channels[device_spec];
+                       device->extra_delay_ms = new_extra_delay_ms[device_spec];
                        alsa_pool.reset_device(device_spec.index);
                        reset_resampler_mutex_held(device_spec);
                }
@@ -1238,8 +1247,10 @@ void AudioMixer::set_input_mapping_lock_held(const InputMapping &new_input_mappi
        for (unsigned card_index = 0; card_index < num_ffmpeg_inputs; ++card_index) {
                const DeviceSpec device_spec{InputSourceType::FFMPEG_VIDEO_INPUT, card_index};
                AudioDevice *device = find_audio_device(device_spec);
-               if (device->interesting_channels != interesting_channels[device_spec]) {
+               if (device->interesting_channels != interesting_channels[device_spec] ||
+                   device->extra_delay_ms != new_extra_delay_ms[device_spec]) {
                        device->interesting_channels = interesting_channels[device_spec];
+                       device->extra_delay_ms = new_extra_delay_ms[device_spec];
                        reset_resampler_mutex_held(device_spec);
                }
        }

diff --git a/nageru/audio_mixer.h b/nageru/audio_mixer.h
index 14e7e85d098065aabc5cd39d70ed5b31a8713041..de6ff5414032a7f414d82b7195156c2921400a7c 100644 (file)
--- a/nageru/audio_mixer.h
+++ b/nageru/audio_mixer.h
@@ -324,6 +324,10 @@ private:
                // Which channels we consider interesting (ie., are part of some input_mapping).
                std::set<unsigned> interesting_channels;
                bool silenced = false;
+
+               // Positive means the audio is delayed, negative means we try to have it earlier
+               // (although we can't time-travel!). Stored together with the input mapping.
+               double extra_delay_ms = 0.0;
        };
 
        const AudioDevice *find_audio_device(DeviceSpec device_spec) const

diff --git a/nageru/input_mapping.cpp b/nageru/input_mapping.cpp
index dfcc97ce5e6613cd8c51f12d55a1be49433daf5f..e41f9950332244d83a9e4d99c8e200541b9e89eb 100644 (file)
--- a/nageru/input_mapping.cpp
+++ b/nageru/input_mapping.cpp
@@ -44,7 +44,13 @@ bool save_input_mapping_to_file(const map<DeviceSpec, DeviceInfo> &devices, cons
                for (const InputMapping::Bus &bus : input_mapping.buses) {
                        if (!used_devices.count(bus.device)) {
                                used_devices.emplace(bus.device, used_devices.size());
-                               global_audio_mixer->serialize_device(bus.device, mapping_proto.add_device());
+                               DeviceSpecProto *device_proto = mapping_proto.add_device();
+                               global_audio_mixer->serialize_device(bus.device, device_proto);
+
+                               const auto delay_it = input_mapping.extra_delay_ms.find(bus.device);
+                               if (delay_it != input_mapping.extra_delay_ms.end()) {
+                                       device_proto->set_extra_delay_ms(delay_it->second);
+                               }
                        }
 
                        BusProto *bus_proto = mapping_proto.add_bus();
@@ -175,6 +181,7 @@ found_alsa_input:
                default:
                        assert(false);
                }
+               new_mapping->extra_delay_ms.emplace(device_mapping.back(), device_proto.extra_delay_ms());
        }
 
        for (const BusProto &bus_proto : mapping_proto.bus()) {

diff --git a/nageru/input_mapping.h b/nageru/input_mapping.h
index 67af0f489ee3aaf465b8c6c32aa554796c788cf1..6f5a42a5fc254c06872b41c6c4ef2cd7f3c9fc9e 100644 (file)
--- a/nageru/input_mapping.h
+++ b/nageru/input_mapping.h
@@ -45,6 +45,7 @@ struct InputMapping {
        };
 
        std::vector<Bus> buses;
+       std::map<DeviceSpec, double> extra_delay_ms;
 };
 
 // This is perhaps not the most user-friendly output, but it's at least better

diff --git a/nageru/input_mapping.ui b/nageru/input_mapping.ui
index 4487b9419506fa2618a2f833063c2b4bcfd8c00a..bec59de1665e48c25f7bcc57d9fd3928d0d88566 100644 (file)
--- a/nageru/input_mapping.ui
+++ b/nageru/input_mapping.ui
@@ -36,6 +36,11 @@
        <string>Right input</string>
       </property>
      </column>
+     <column>
+      <property name="text">
+       <string>Extra delay to add (ms)</string>
+      </property>
+     </column>
     </widget>
    </item>
    <item>

diff --git a/nageru/input_mapping_dialog.cpp b/nageru/input_mapping_dialog.cpp
index b4565152a4af01b290941025633e7224b8c4d4dd..529b2e5aa93b41bc59d08bca9302289801ff089e 100644 (file)
--- a/nageru/input_mapping_dialog.cpp
+++ b/nageru/input_mapping_dialog.cpp
@@ -12,6 +12,7 @@
 #include <QList>
 #include <QMessageBox>
 #include <QPushButton>
+#include <QSpinBox>
 #include <QTableWidget>
 #include <QVariant>
 #include <functional>
@@ -41,6 +42,7 @@ InputMappingDialog::InputMappingDialog()
        ui->setupUi(this);
        ui->table->setSelectionBehavior(QAbstractItemView::SelectRows);
        ui->table->setSelectionMode(QAbstractItemView::SingleSelection);  // Makes implementing moving easier for now.
+       ui->table->setColumnWidth(4, 200);
 
        fill_ui_from_mapping(mapping);
        connect(ui->table, &QTableWidget::cellChanged, this, &InputMappingDialog::cell_changed);
@@ -61,7 +63,7 @@ InputMappingDialog::InputMappingDialog()
                post_to_main_thread([this]{
                        devices = global_audio_mixer->get_devices();
                        for (unsigned row = 0; row < mapping.buses.size(); ++row) {
-                               fill_row_from_bus(row, mapping.buses[row]);
+                               fill_row_from_bus(row, mapping.buses[row], mapping);
                        }
                });
        });
@@ -82,11 +84,11 @@ void InputMappingDialog::fill_ui_from_mapping(const InputMapping &mapping)
 
        ui->table->setRowCount(mapping.buses.size());
        for (unsigned row = 0; row < mapping.buses.size(); ++row) {
-               fill_row_from_bus(row, mapping.buses[row]);
+               fill_row_from_bus(row, mapping.buses[row], mapping);
        }
 }
 
-void InputMappingDialog::fill_row_from_bus(unsigned row, const InputMapping::Bus &bus)
+void InputMappingDialog::fill_row_from_bus(unsigned row, const InputMapping::Bus &bus, const InputMapping &mapping)
 {
        QString name(QString::fromStdString(bus.name));
        ui->table->setItem(row, 0, new QTableWidgetItem(name));
@@ -136,6 +138,7 @@ void InputMappingDialog::fill_row_from_bus(unsigned row, const InputMapping::Bus
                bind(&InputMappingDialog::card_selected, this, card_combo, row, _1));
        ui->table->setCellWidget(row, 1, card_combo);
 
+       fill_delay_from_bus(row, bus, mapping);
        setup_channel_choices_from_bus(row, bus);
 }
 
@@ -169,6 +172,34 @@ void InputMappingDialog::setup_channel_choices_from_bus(unsigned row, const Inpu
        }
 }
 
+void InputMappingDialog::fill_delay_from_bus(unsigned row, const InputMapping::Bus &bus, const InputMapping &mapping)
+{
+       QDoubleSpinBox *delay_spinner = static_cast<QDoubleSpinBox *>(ui->table->cellWidget(row, 4));
+       if (delay_spinner == nullptr) {
+               delay_spinner = new QDoubleSpinBox;
+               delay_spinner->setDecimals(1);
+               delay_spinner->setMinimum(-50.0);
+               delay_spinner->setMaximum(500.0);
+               connect(delay_spinner, static_cast<void(QDoubleSpinBox::*)(double)>(&QDoubleSpinBox::valueChanged),
+                       bind(&InputMappingDialog::delay_changed, this, row, _1));
+       }
+       delay_spinner->blockSignals(true);
+       if (bus.device.type == InputSourceType::SILENCE) {
+               delay_spinner->setEnabled(false);
+               delay_spinner->setValue(0.0);
+       } else {
+               delay_spinner->setEnabled(true);
+               const auto delay_it = mapping.extra_delay_ms.find(bus.device);
+               if (delay_it != mapping.extra_delay_ms.end()) {
+                       delay_spinner->setValue(delay_it->second);
+               } else {
+                       delay_spinner->setValue(0.0);
+               }
+       }
+       delay_spinner->blockSignals(false);
+       ui->table->setCellWidget(row, 4, delay_spinner);
+}
+
 void InputMappingDialog::ok_clicked()
 {
        global_audio_mixer->set_state_changed_callback(saved_callback);
@@ -200,6 +231,7 @@ void InputMappingDialog::card_selected(QComboBox *card_combo, unsigned row, int
 {
        uint64_t key = card_combo->itemData(index).toULongLong();
        mapping.buses[row].device = key_to_DeviceSpec(key);
+       fill_delay_from_bus(row, mapping.buses[row], mapping);
        setup_channel_choices_from_bus(row, mapping.buses[row]);
 }
 
@@ -208,6 +240,17 @@ void InputMappingDialog::channel_selected(unsigned row, unsigned channel, int in
        mapping.buses[row].source_channel[channel] = index - 1;
 }
 
+void InputMappingDialog::delay_changed(unsigned row, double value)
+{
+       mapping.extra_delay_ms[mapping.buses[row].device] = value;
+
+       for (unsigned other_row = 0; other_row < mapping.buses.size(); ++other_row) {
+               if (row != other_row && mapping.buses[row].device == mapping.buses[other_row].device) {
+                       fill_delay_from_bus(other_row, mapping.buses[other_row], mapping);
+               }
+       }
+}
+
 void InputMappingDialog::add_clicked()
 {
        QTableWidgetSelectionRange all(0, 0, ui->table->rowCount() - 1, ui->table->columnCount() - 1);
@@ -221,7 +264,7 @@ void InputMappingDialog::add_clicked()
        ui->table->setRowCount(mapping.buses.size());
 
        unsigned row = mapping.buses.size() - 1;
-       fill_row_from_bus(row, new_bus);
+       fill_row_from_bus(row, new_bus, mapping);
        ui->table->editItem(ui->table->item(row, 0));  // Start editing the name.
        update_button_state();
 }
@@ -257,8 +300,8 @@ void InputMappingDialog::updown_clicked(int direction)
 
        swap(mapping.buses[a_row], mapping.buses[b_row]);
        swap(bus_settings[a_row], bus_settings[b_row]);
-       fill_row_from_bus(a_row, mapping.buses[a_row]);
-       fill_row_from_bus(b_row, mapping.buses[b_row]);
+       fill_row_from_bus(a_row, mapping.buses[a_row], mapping);
+       fill_row_from_bus(b_row, mapping.buses[b_row], mapping);
 
        QTableWidgetSelectionRange a_sel(a_row, 0, a_row, ui->table->columnCount() - 1);
        QTableWidgetSelectionRange b_sel(b_row, 0, b_row, ui->table->columnCount() - 1);

diff --git a/nageru/input_mapping_dialog.h b/nageru/input_mapping_dialog.h
index 640644e3898ce9cca25fc950bf9e350477c1b17b..8873850b8798f7f102e6cfa3704735cf2ee16d67 100644 (file)
--- a/nageru/input_mapping_dialog.h
+++ b/nageru/input_mapping_dialog.h
@@ -27,11 +27,13 @@ public:
 
 private:
        void fill_ui_from_mapping(const InputMapping &mapping);
-       void fill_row_from_bus(unsigned row, const InputMapping::Bus &bus);
+       void fill_row_from_bus(unsigned row, const InputMapping::Bus &bus, const InputMapping &mapping);
+       void fill_delay_from_bus(unsigned row, const InputMapping::Bus &bus, const InputMapping &mapping);
        void setup_channel_choices_from_bus(unsigned row, const InputMapping::Bus &bus);
        void cell_changed(int row, int column);
        void card_selected(QComboBox *card_combo, unsigned row, int index);
        void channel_selected(unsigned row, unsigned channel, int index);
+       void delay_changed(unsigned row, double value);
        void ok_clicked();
        void cancel_clicked();
        void add_clicked();

diff --git a/nageru/state.proto b/nageru/state.proto
index 6372e61b5cc4b22575888c47c3f38cc91324649e..a7df37c7bf8767c4766e6f43fbdc0ce7cd30ad9a 100644 (file)
--- a/nageru/state.proto
+++ b/nageru/state.proto
@@ -18,6 +18,11 @@ message DeviceSpecProto {
        optional string alsa_info = 5;  // Only for ALSA devices.
        optional int32 num_channels = 6;  // Only for ALSA devices.
        optional string address = 7;  // Only for ALSA devices.
+
+       // This isn't strictly part of the _mapping_, but it fits in fairly naturally.
+       // Positive means the audio is delayed, negative means we try to have it earlier
+       // (although we can't time-travel!).
+       optional float extra_delay_ms = 8;
 }
 
 // Corresponds to InputMapping::Bus.