#include <endian.h>
#include <bmusb/bmusb.h>
#include <stdio.h>
+#include <endian.h>
#include <cmath>
#include "db.h"
using namespace bmusb;
using namespace std;
+using namespace std::placeholders;
namespace {
-void convert_fixed24_to_fp32(float *dst, size_t out_channels, const uint8_t *src, size_t in_channels, size_t num_samples)
+// TODO: If these prove to be a bottleneck, they can be SSSE3-optimized
+// (usually including multiple channels at a time).
+
+void convert_fixed16_to_fp32(float *dst, size_t out_channel, size_t out_num_channels,
+ const uint8_t *src, size_t in_channel, size_t in_num_channels,
+ size_t num_samples)
{
- assert(in_channels >= out_channels);
+ assert(in_channel < in_num_channels);
+ assert(out_channel < out_num_channels);
+ src += in_channel * 2;
+ dst += out_channel;
+
for (size_t i = 0; i < num_samples; ++i) {
- for (size_t j = 0; j < out_channels; ++j) {
- uint32_t s1 = *src++;
- uint32_t s2 = *src++;
- uint32_t s3 = *src++;
- uint32_t s = s1 | (s1 << 8) | (s2 << 16) | (s3 << 24);
- dst[i * out_channels + j] = int(s) * (1.0f / 2147483648.0f);
- }
- src += 3 * (in_channels - out_channels);
+ int16_t s = le16toh(*(int16_t *)src);
+ *dst = s * (1.0f / 32768.0f);
+
+ src += 2 * in_num_channels;
+ dst += out_num_channels;
}
}
-void convert_fixed32_to_fp32(float *dst, size_t out_channels, const uint8_t *src, size_t in_channels, size_t num_samples)
+void convert_fixed24_to_fp32(float *dst, size_t out_channel, size_t out_num_channels,
+ const uint8_t *src, size_t in_channel, size_t in_num_channels,
+ size_t num_samples)
{
- assert(in_channels >= out_channels);
+ assert(in_channel < in_num_channels);
+ assert(out_channel < out_num_channels);
+ src += in_channel * 3;
+ dst += out_channel;
+
for (size_t i = 0; i < num_samples; ++i) {
- for (size_t j = 0; j < out_channels; ++j) {
- int32_t s = le32toh(*(int32_t *)src);
- dst[i * out_channels + j] = s * (1.0f / 2147483648.0f);
- src += 4;
- }
- src += 4 * (in_channels - out_channels);
+ uint32_t s1 = src[0];
+ uint32_t s2 = src[1];
+ uint32_t s3 = src[2];
+ uint32_t s = s1 | (s1 << 8) | (s2 << 16) | (s3 << 24);
+ *dst = int(s) * (1.0f / 2147483648.0f);
+
+ src += 3 * in_num_channels;
+ dst += out_num_channels;
+ }
+}
+
+void convert_fixed32_to_fp32(float *dst, size_t out_channel, size_t out_num_channels,
+ const uint8_t *src, size_t in_channel, size_t in_num_channels,
+ size_t num_samples)
+{
+ assert(in_channel < in_num_channels);
+ assert(out_channel < out_num_channels);
+ src += in_channel * 4;
+ dst += out_channel;
+
+ for (size_t i = 0; i < num_samples; ++i) {
+ int32_t s = le32toh(*(int32_t *)src);
+ *dst = s * (1.0f / 2147483648.0f);
+
+ src += 4 * in_num_channels;
+ dst += out_num_channels;
+ }
+}
+
+float find_peak(const float *samples, size_t num_samples)
+{
+ float m = fabs(samples[0]);
+ for (size_t i = 1; i < num_samples; ++i) {
+ m = max(m, fabs(samples[i]));
+ }
+ return m;
+}
+
+void deinterleave_samples(const vector<float> &in, vector<float> *out_l, vector<float> *out_r)
+{
+ size_t num_samples = in.size() / 2;
+ out_l->resize(num_samples);
+ out_r->resize(num_samples);
+
+ const float *inptr = in.data();
+ float *lptr = &(*out_l)[0];
+ float *rptr = &(*out_r)[0];
+ for (size_t i = 0; i < num_samples; ++i) {
+ *lptr++ = *inptr++;
+ *rptr++ = *inptr++;
}
}
: num_cards(num_cards),
level_compressor(OUTPUT_FREQUENCY),
limiter(OUTPUT_FREQUENCY),
- compressor(OUTPUT_FREQUENCY)
+ compressor(OUTPUT_FREQUENCY),
+ correlation(OUTPUT_FREQUENCY)
{
locut.init(FILTER_HPF, 2);
set_compressor_enabled(global_flags.compressor_enabled);
set_limiter_enabled(global_flags.limiter_enabled);
set_final_makeup_gain_auto(global_flags.final_makeup_gain_auto);
+
+ // Generate a very simple, default input mapping.
+ InputMapping::Bus input;
+ input.name = "Main";
+ input.device.type = InputSourceType::CAPTURE_CARD;
+ input.device.index = 0;
+ input.source_channel[0] = 0;
+ input.source_channel[1] = 1;
+
+ InputMapping new_input_mapping;
+ new_input_mapping.buses.push_back(input);
+ set_input_mapping(new_input_mapping);
+
+ // Look for ALSA cards.
+ available_alsa_cards = ALSAInput::enumerate_devices();
+
+ r128.init(2, OUTPUT_FREQUENCY);
+ r128.integr_start();
+
+ // hlen=16 is pretty low quality, but we use quite a bit of CPU otherwise,
+ // and there's a limit to how important the peak meter is.
+ peak_resampler.setup(OUTPUT_FREQUENCY, OUTPUT_FREQUENCY * 4, /*num_channels=*/2, /*hlen=*/16, /*frel=*/1.0);
+}
+
+AudioMixer::~AudioMixer()
+{
+ for (unsigned card_index = 0; card_index < available_alsa_cards.size(); ++card_index) {
+ const AudioDevice &device = alsa_inputs[card_index];
+ if (device.alsa_device != nullptr) {
+ device.alsa_device->stop_capture_thread();
+ }
+ }
}
-void AudioMixer::reset_card(unsigned card_index)
+
+void AudioMixer::reset_resampler(DeviceSpec device_spec)
{
- CaptureCard *card = &cards[card_index];
+ lock_guard<timed_mutex> lock(audio_mutex);
+ reset_resampler_mutex_held(device_spec);
+}
- unique_lock<mutex> lock(card->audio_mutex);
- card->resampling_queue.reset(new ResamplingQueue(card_index, OUTPUT_FREQUENCY, OUTPUT_FREQUENCY, 2));
- card->next_local_pts = 0;
+void AudioMixer::reset_resampler_mutex_held(DeviceSpec device_spec)
+{
+ AudioDevice *device = find_audio_device(device_spec);
+
+ if (device->interesting_channels.empty()) {
+ device->resampling_queue.reset();
+ } else {
+ // TODO: ResamplingQueue should probably take the full device spec.
+ // (It's only used for console output, though.)
+ device->resampling_queue.reset(new ResamplingQueue(device_spec.index, device->capture_frequency, OUTPUT_FREQUENCY, device->interesting_channels.size()));
+ }
+ device->next_local_pts = 0;
}
-void AudioMixer::add_audio(unsigned card_index, const uint8_t *data, unsigned num_samples, AudioFormat audio_format, int64_t frame_length)
+void AudioMixer::reset_alsa_mutex_held(DeviceSpec device_spec)
{
- CaptureCard *card = &cards[card_index];
+ assert(device_spec.type == InputSourceType::ALSA_INPUT);
+ unsigned card_index = device_spec.index;
+ AudioDevice *device = find_audio_device(device_spec);
- // Convert the audio to stereo fp32.
- vector<float> audio;
- audio.resize(num_samples * 2);
- switch (audio_format.bits_per_sample) {
- case 0:
- assert(num_samples == 0);
- break;
- case 24:
- convert_fixed24_to_fp32(&audio[0], 2, data, audio_format.num_channels, num_samples);
- break;
- case 32:
- convert_fixed32_to_fp32(&audio[0], 2, data, audio_format.num_channels, num_samples);
- break;
- default:
- fprintf(stderr, "Cannot handle audio with %u bits per sample\n", audio_format.bits_per_sample);
- assert(false);
+ if (device->alsa_device != nullptr) {
+ device->alsa_device->stop_capture_thread();
}
+ if (device->interesting_channels.empty()) {
+ device->alsa_device.reset();
+ } else {
+ device->alsa_device.reset(new ALSAInput(available_alsa_cards[card_index].address.c_str(), OUTPUT_FREQUENCY, 2, bind(&AudioMixer::add_audio, this, device_spec, _1, _2, _3, _4)));
+ device->capture_frequency = device->alsa_device->get_sample_rate();
+ device->alsa_device->start_capture_thread();
+ }
+}
- // Now add it.
- {
- unique_lock<mutex> lock(card->audio_mutex);
+bool AudioMixer::add_audio(DeviceSpec device_spec, const uint8_t *data, unsigned num_samples, AudioFormat audio_format, int64_t frame_length)
+{
+ AudioDevice *device = find_audio_device(device_spec);
+
+ unique_lock<timed_mutex> lock(audio_mutex, defer_lock);
+ if (!lock.try_lock_for(chrono::milliseconds(10))) {
+ return false;
+ }
+ if (device->resampling_queue == nullptr) {
+ // No buses use this device; throw it away.
+ return true;
+ }
+
+ unsigned num_channels = device->interesting_channels.size();
+ assert(num_channels > 0);
- int64_t local_pts = card->next_local_pts;
- card->resampling_queue->add_input_samples(local_pts / double(TIMEBASE), audio.data(), num_samples);
- card->next_local_pts = local_pts + frame_length;
+ // Convert the audio to fp32.
+ vector<float> audio;
+ audio.resize(num_samples * num_channels);
+ unsigned channel_index = 0;
+ for (auto channel_it = device->interesting_channels.cbegin(); channel_it != device->interesting_channels.end(); ++channel_it, ++channel_index) {
+ switch (audio_format.bits_per_sample) {
+ case 0:
+ assert(num_samples == 0);
+ break;
+ case 16:
+ convert_fixed16_to_fp32(&audio[0], channel_index, num_channels, data, *channel_it, audio_format.num_channels, num_samples);
+ break;
+ case 24:
+ convert_fixed24_to_fp32(&audio[0], channel_index, num_channels, data, *channel_it, audio_format.num_channels, num_samples);
+ break;
+ case 32:
+ convert_fixed32_to_fp32(&audio[0], channel_index, num_channels, data, *channel_it, audio_format.num_channels, num_samples);
+ break;
+ default:
+ fprintf(stderr, "Cannot handle audio with %u bits per sample\n", audio_format.bits_per_sample);
+ assert(false);
+ }
}
+
+ // Now add it.
+ int64_t local_pts = device->next_local_pts;
+ device->resampling_queue->add_input_samples(local_pts / double(TIMEBASE), audio.data(), num_samples);
+ device->next_local_pts = local_pts + frame_length;
+ return true;
}
-void AudioMixer::add_silence(unsigned card_index, unsigned samples_per_frame, unsigned num_frames, int64_t frame_length)
+bool AudioMixer::add_silence(DeviceSpec device_spec, unsigned samples_per_frame, unsigned num_frames, int64_t frame_length)
{
- CaptureCard *card = &cards[card_index];
- unique_lock<mutex> lock(card->audio_mutex);
+ AudioDevice *device = find_audio_device(device_spec);
+
+ unique_lock<timed_mutex> lock(audio_mutex, defer_lock);
+ if (!lock.try_lock_for(chrono::milliseconds(10))) {
+ return false;
+ }
+ if (device->resampling_queue == nullptr) {
+ // No buses use this device; throw it away.
+ return true;
+ }
- vector<float> silence(samples_per_frame * 2, 0.0f);
+ unsigned num_channels = device->interesting_channels.size();
+ assert(num_channels > 0);
+
+ vector<float> silence(samples_per_frame * num_channels, 0.0f);
for (unsigned i = 0; i < num_frames; ++i) {
- card->resampling_queue->add_input_samples(card->next_local_pts / double(TIMEBASE), silence.data(), samples_per_frame);
+ device->resampling_queue->add_input_samples(device->next_local_pts / double(TIMEBASE), silence.data(), samples_per_frame);
// Note that if the format changed in the meantime, we have
// no way of detecting that; we just have to assume the frame length
// is always the same.
- card->next_local_pts += frame_length;
+ device->next_local_pts += frame_length;
}
+ return true;
}
-vector<float> AudioMixer::get_output(double pts, unsigned num_samples, ResamplingQueue::RateAdjustmentPolicy rate_adjustment_policy)
+AudioMixer::AudioDevice *AudioMixer::find_audio_device(DeviceSpec device)
{
- vector<float> samples_card;
- vector<float> samples_out;
- samples_out.resize(num_samples * 2);
+ switch (device.type) {
+ case InputSourceType::CAPTURE_CARD:
+ return &video_cards[device.index];
+ case InputSourceType::ALSA_INPUT:
+ return &alsa_inputs[device.index];
+ case InputSourceType::SILENCE:
+ default:
+ assert(false);
+ }
+ return nullptr;
+}
- // TODO: Allow more flexible input mapping.
- for (unsigned card_index = 0; card_index < num_cards; ++card_index) {
- samples_card.resize(num_samples * 2);
- {
- unique_lock<mutex> lock(cards[card_index].audio_mutex);
- cards[card_index].resampling_queue->get_output_samples(
+// Get a pointer to the given channel from the given device.
+// The channel must be picked out earlier and resampled.
+void AudioMixer::find_sample_src_from_device(const map<DeviceSpec, vector<float>> &samples_card, DeviceSpec device_spec, int source_channel, const float **srcptr, unsigned *stride)
+{
+ static float zero = 0.0f;
+ if (source_channel == -1 || device_spec.type == InputSourceType::SILENCE) {
+ *srcptr = &zero;
+ *stride = 0;
+ return;
+ }
+ AudioDevice *device = find_audio_device(device_spec);
+ assert(device->interesting_channels.count(source_channel) != 0);
+ unsigned channel_index = 0;
+ for (int channel : device->interesting_channels) {
+ if (channel == source_channel) break;
+ ++channel_index;
+ }
+ assert(channel_index < device->interesting_channels.size());
+ const auto it = samples_card.find(device_spec);
+ assert(it != samples_card.end());
+ *srcptr = &(it->second)[channel_index];
+ *stride = device->interesting_channels.size();
+}
+
+// TODO: Can be SSSE3-optimized if need be.
+void AudioMixer::fill_audio_bus(const map<DeviceSpec, vector<float>> &samples_card, const InputMapping::Bus &bus, unsigned num_samples, float *output)
+{
+ if (bus.device.type == InputSourceType::SILENCE) {
+ memset(output, 0, num_samples * sizeof(*output));
+ } else {
+ assert(bus.device.type == InputSourceType::CAPTURE_CARD ||
+ bus.device.type == InputSourceType::ALSA_INPUT);
+ const float *lsrc, *rsrc;
+ unsigned lstride, rstride;
+ float *dptr = output;
+ find_sample_src_from_device(samples_card, bus.device, bus.source_channel[0], &lsrc, &lstride);
+ find_sample_src_from_device(samples_card, bus.device, bus.source_channel[1], &rsrc, &rstride);
+ for (unsigned i = 0; i < num_samples; ++i) {
+ *dptr++ = *lsrc;
+ *dptr++ = *rsrc;
+ lsrc += lstride;
+ rsrc += rstride;
+ }
+ }
+}
+
+vector<float> AudioMixer::get_output(double pts, unsigned num_samples, ResamplingQueue::RateAdjustmentPolicy rate_adjustment_policy)
+{
+ map<DeviceSpec, vector<float>> samples_card;
+ vector<float> samples_bus;
+
+ lock_guard<timed_mutex> lock(audio_mutex);
+
+ // Pick out all the interesting channels from all the cards.
+ // TODO: If the card has been hotswapped, the number of channels
+ // might have changed; if so, we need to do some sort of remapping
+ // to silence.
+ for (const auto &spec_and_info : get_devices_mutex_held()) {
+ const DeviceSpec &device_spec = spec_and_info.first;
+ AudioDevice *device = find_audio_device(device_spec);
+ if (!device->interesting_channels.empty()) {
+ samples_card[device_spec].resize(num_samples * device->interesting_channels.size());
+ device->resampling_queue->get_output_samples(
pts,
- &samples_card[0],
+ &samples_card[device_spec][0],
num_samples,
rate_adjustment_policy);
}
+ }
+
+ // TODO: Move lo-cut etc. into each bus.
+ vector<float> samples_out;
+ samples_out.resize(num_samples * 2);
+ samples_bus.resize(num_samples * 2);
+ for (unsigned bus_index = 0; bus_index < input_mapping.buses.size(); ++bus_index) {
+ fill_audio_bus(samples_card, input_mapping.buses[bus_index], num_samples, &samples_bus[0]);
- float volume = from_db(cards[card_index].fader_volume_db);
- if (card_index == 0) {
+ float volume = from_db(fader_volume_db[bus_index]);
+ if (bus_index == 0) {
for (unsigned i = 0; i < num_samples * 2; ++i) {
- samples_out[i] = samples_card[i] * volume;
+ samples_out[i] = samples_bus[i] * volume;
}
} else {
for (unsigned i = 0; i < num_samples * 2; ++i) {
- samples_out[i] += samples_card[i] * volume;
+ samples_out[i] += samples_bus[i] * volume;
}
}
}
}
{
- unique_lock<mutex> lock(compressor_mutex);
+ lock_guard<mutex> lock(compressor_mutex);
// Apply a level compressor to get the general level right.
// Basically, if it's over about -40 dBFS, we squeeze it down to that level
// something we get out per-sample.
//
// Note that there's a feedback loop here, so we choose a very slow filter
- // (half-time of 100 seconds).
+ // (half-time of 30 seconds).
double target_loudness_factor, alpha;
- double loudness_lu = loudness_lufs - ref_level_lufs;
+ double loudness_lu = r128.loudness_M() - ref_level_lufs;
double current_makeup_lu = to_db(final_makeup_gain);
- target_loudness_factor = from_db(-loudness_lu);
+ target_loudness_factor = final_makeup_gain * from_db(-loudness_lu);
// If we're outside +/- 5 LU uncorrected, we don't count it as
// a normal signal (probably silence) and don't change the
} else {
// Formula adapted from
// https://en.wikipedia.org/wiki/Low-pass_filter#Simple_infinite_impulse_response_filter.
- const double half_time_s = 100.0;
+ const double half_time_s = 30.0;
const double fc_mul_2pi_delta_t = 1.0 / (half_time_s * OUTPUT_FREQUENCY);
alpha = fc_mul_2pi_delta_t / (fc_mul_2pi_delta_t + 1.0);
}
{
- unique_lock<mutex> lock(compressor_mutex);
+ lock_guard<mutex> lock(compressor_mutex);
double m = final_makeup_gain;
for (size_t i = 0; i < samples_out.size(); i += 2) {
samples_out[i + 0] *= m;
final_makeup_gain = m;
}
+ update_meters(samples_out);
+
return samples_out;
}
+
+void AudioMixer::update_meters(const vector<float> &samples)
+{
+ // Upsample 4x to find interpolated peak.
+ peak_resampler.inp_data = const_cast<float *>(samples.data());
+ peak_resampler.inp_count = samples.size() / 2;
+
+ vector<float> interpolated_samples;
+ interpolated_samples.resize(samples.size());
+ {
+ unique_lock<mutex> lock(audio_measure_mutex);
+
+ while (peak_resampler.inp_count > 0) { // About four iterations.
+ peak_resampler.out_data = &interpolated_samples[0];
+ peak_resampler.out_count = interpolated_samples.size() / 2;
+ peak_resampler.process();
+ size_t out_stereo_samples = interpolated_samples.size() / 2 - peak_resampler.out_count;
+ peak = max<float>(peak, find_peak(interpolated_samples.data(), out_stereo_samples * 2));
+ peak_resampler.out_data = nullptr;
+ }
+ }
+
+ // Find R128 levels and L/R correlation.
+ vector<float> left, right;
+ deinterleave_samples(samples, &left, &right);
+ float *ptrs[] = { left.data(), right.data() };
+ {
+ unique_lock<mutex> lock(audio_measure_mutex);
+ r128.process(left.size(), ptrs);
+ correlation.process_samples(samples);
+ }
+
+ send_audio_level_callback();
+}
+
+void AudioMixer::reset_meters()
+{
+ unique_lock<mutex> lock(audio_measure_mutex);
+ peak_resampler.reset();
+ peak = 0.0f;
+ r128.reset();
+ r128.integr_start();
+ correlation.reset();
+}
+
+void AudioMixer::send_audio_level_callback()
+{
+ if (audio_level_callback == nullptr) {
+ return;
+ }
+
+ unique_lock<mutex> lock(audio_measure_mutex);
+ double loudness_s = r128.loudness_S();
+ double loudness_i = r128.integrated();
+ double loudness_range_low = r128.range_min();
+ double loudness_range_high = r128.range_max();
+
+ audio_level_callback(loudness_s, to_db(peak),
+ loudness_i, loudness_range_low, loudness_range_high,
+ gain_staging_db,
+ to_db(final_makeup_gain),
+ correlation.get_correlation());
+}
+
+map<DeviceSpec, DeviceInfo> AudioMixer::get_devices() const
+{
+ lock_guard<timed_mutex> lock(audio_mutex);
+ return get_devices_mutex_held();
+}
+
+map<DeviceSpec, DeviceInfo> AudioMixer::get_devices_mutex_held() const
+{
+ map<DeviceSpec, DeviceInfo> devices;
+ for (unsigned card_index = 0; card_index < num_cards; ++card_index) {
+ const DeviceSpec spec{ InputSourceType::CAPTURE_CARD, card_index };
+ const AudioDevice *device = &video_cards[card_index];
+ DeviceInfo info;
+ info.name = device->name;
+ info.num_channels = 8; // FIXME: This is wrong for fake cards.
+ devices.insert(make_pair(spec, info));
+ }
+ for (unsigned card_index = 0; card_index < available_alsa_cards.size(); ++card_index) {
+ const DeviceSpec spec{ InputSourceType::ALSA_INPUT, card_index };
+ const ALSAInput::Device &device = available_alsa_cards[card_index];
+ DeviceInfo info;
+ info.name = device.name + " (" + device.info + ")";
+ info.num_channels = device.num_channels;
+ devices.insert(make_pair(spec, info));
+ }
+ return devices;
+}
+
+void AudioMixer::set_name(DeviceSpec device_spec, const string &name)
+{
+ AudioDevice *device = find_audio_device(device_spec);
+
+ lock_guard<timed_mutex> lock(audio_mutex);
+ device->name = name;
+}
+
+void AudioMixer::set_input_mapping(const InputMapping &new_input_mapping)
+{
+ lock_guard<timed_mutex> lock(audio_mutex);
+
+ map<DeviceSpec, set<unsigned>> interesting_channels;
+ for (const InputMapping::Bus &bus : new_input_mapping.buses) {
+ if (bus.device.type == InputSourceType::CAPTURE_CARD ||
+ bus.device.type == InputSourceType::ALSA_INPUT) {
+ for (unsigned channel = 0; channel < 2; ++channel) {
+ if (bus.source_channel[channel] != -1) {
+ interesting_channels[bus.device].insert(bus.source_channel[channel]);
+ }
+ }
+ }
+ }
+
+ // Reset resamplers for all cards that don't have the exact same state as before.
+ for (const auto &spec_and_info : get_devices_mutex_held()) {
+ const DeviceSpec &device_spec = spec_and_info.first;
+ AudioDevice *device = find_audio_device(device_spec);
+ if (device->interesting_channels != interesting_channels[device_spec]) {
+ device->interesting_channels = interesting_channels[device_spec];
+ if (device_spec.type == InputSourceType::ALSA_INPUT) {
+ reset_alsa_mutex_held(device_spec);
+ }
+ reset_resampler_mutex_held(device_spec);
+ }
+ }
+
+ input_mapping = new_input_mapping;
+}
+
+InputMapping AudioMixer::get_input_mapping() const
+{
+ lock_guard<timed_mutex> lock(audio_mutex);
+ return input_mapping;
+}