git.sesse.net Git - nageru/blob - audio_mixer.cpp

   1 #include "audio_mixer.h"
   2
   3 #include <assert.h>
   4 #include <endian.h>
   5 #include <bmusb/bmusb.h>
   6 #include <stdio.h>
   7 #include <cmath>
   8
   9 #include "flags.h"
  10 #include "timebase.h"
  11
  12 using namespace bmusb;
  13 using namespace std;
  14
  15 namespace {
  16
  17 void convert_fixed24_to_fp32(float *dst, size_t out_channels, const uint8_t *src, size_t in_channels, size_t num_samples)
  18 {
  19         assert(in_channels >= out_channels);
  20         for (size_t i = 0; i < num_samples; ++i) {
  21                 for (size_t j = 0; j < out_channels; ++j) {
  22                         uint32_t s1 = *src++;
  23                         uint32_t s2 = *src++;
  24                         uint32_t s3 = *src++;
  25                         uint32_t s = s1 | (s1 << 8) | (s2 << 16) | (s3 << 24);
  26                         dst[i * out_channels + j] = int(s) * (1.0f / 2147483648.0f);
  27                 }
  28                 src += 3 * (in_channels - out_channels);
  29         }
  30 }
  31
  32 void convert_fixed32_to_fp32(float *dst, size_t out_channels, const uint8_t *src, size_t in_channels, size_t num_samples)
  33 {
  34         assert(in_channels >= out_channels);
  35         for (size_t i = 0; i < num_samples; ++i) {
  36                 for (size_t j = 0; j < out_channels; ++j) {
  37                         int32_t s = le32toh(*(int32_t *)src);
  38                         dst[i * out_channels + j] = s * (1.0f / 2147483648.0f);
  39                         src += 4;
  40                 }
  41                 src += 4 * (in_channels - out_channels);
  42         }
  43 }
  44
  45 }  // namespace
  46
  47 AudioMixer::AudioMixer(unsigned num_cards)
  48         : num_cards(num_cards),
  49           level_compressor(OUTPUT_FREQUENCY),
  50           limiter(OUTPUT_FREQUENCY),
  51           compressor(OUTPUT_FREQUENCY)
  52 {
  53         locut.init(FILTER_HPF, 2);
  54
  55         set_locut_enabled(global_flags.locut_enabled);
  56         set_gain_staging_db(global_flags.initial_gain_staging_db);
  57         set_gain_staging_auto(global_flags.gain_staging_auto);
  58         set_compressor_enabled(global_flags.compressor_enabled);
  59         set_limiter_enabled(global_flags.limiter_enabled);
  60         set_final_makeup_gain_auto(global_flags.final_makeup_gain_auto);
  61 }
  62
  63 void AudioMixer::reset_card(unsigned card_index)
  64 {
  65         CaptureCard *card = &cards[card_index];
  66
  67         unique_lock<mutex> lock(card->audio_mutex);
  68         card->resampling_queue.reset(new ResamplingQueue(card_index, OUTPUT_FREQUENCY, OUTPUT_FREQUENCY, 2));
  69         card->next_local_pts = 0;
  70 }
  71
  72 void AudioMixer::add_audio(unsigned card_index, const uint8_t *data, unsigned num_samples, AudioFormat audio_format, int64_t frame_length)
  73 {
  74         CaptureCard *card = &cards[card_index];
  75
  76         // Convert the audio to stereo fp32.
  77         vector<float> audio;
  78         audio.resize(num_samples * 2);
  79         switch (audio_format.bits_per_sample) {
  80         case 0:
  81                 assert(num_samples == 0);
  82                 break;
  83         case 24:
  84                 convert_fixed24_to_fp32(&audio[0], 2, data, audio_format.num_channels, num_samples);
  85                 break;
  86         case 32:
  87                 convert_fixed32_to_fp32(&audio[0], 2, data, audio_format.num_channels, num_samples);
  88                 break;
  89         default:
  90                 fprintf(stderr, "Cannot handle audio with %u bits per sample\n", audio_format.bits_per_sample);
  91                 assert(false);
  92         }
  93
  94         // Now add it.
  95         {
  96                 unique_lock<mutex> lock(card->audio_mutex);
  97
  98                 int64_t local_pts = card->next_local_pts;
  99                 card->resampling_queue->add_input_samples(local_pts / double(TIMEBASE), audio.data(), num_samples);
 100                 card->next_local_pts = local_pts + frame_length;
 101         }
 102 }
 103
 104 void AudioMixer::add_silence(unsigned card_index, unsigned samples_per_frame, unsigned num_frames, int64_t frame_length)
 105 {
 106         CaptureCard *card = &cards[card_index];
 107         unique_lock<mutex> lock(card->audio_mutex);
 108
 109         vector<float> silence(samples_per_frame * 2, 0.0f);
 110         for (unsigned i = 0; i < num_frames; ++i) {
 111                 card->resampling_queue->add_input_samples(card->next_local_pts / double(TIMEBASE), silence.data(), samples_per_frame);
 112                 // Note that if the format changed in the meantime, we have
 113                 // no way of detecting that; we just have to assume the frame length
 114                 // is always the same.
 115                 card->next_local_pts += frame_length;
 116         }
 117 }
 118
 119 vector<float> AudioMixer::get_output(double pts, unsigned num_samples, ResamplingQueue::RateAdjustmentPolicy rate_adjustment_policy)
 120 {
 121         vector<float> samples_card;
 122         vector<float> samples_out;
 123         samples_out.resize(num_samples * 2);
 124
 125         // TODO: Allow more flexible input mapping.
 126         for (unsigned card_index = 0; card_index < num_cards; ++card_index) {
 127                 samples_card.resize(num_samples * 2);
 128                 {
 129                         unique_lock<mutex> lock(cards[card_index].audio_mutex);
 130                         cards[card_index].resampling_queue->get_output_samples(
 131                                 pts,
 132                                 &samples_card[0],
 133                                 num_samples,
 134                                 rate_adjustment_policy);
 135                 }
 136                 if (card_index == 0) {
 137                         for (unsigned i = 0; i < num_samples * 2; ++i) {
 138                                 samples_out[i] = samples_card[i];
 139                         }
 140                 } else {
 141                         for (unsigned i = 0; i < num_samples * 2; ++i) {
 142                                 samples_out[i] += samples_card[i];
 143                         }
 144                 }
 145         }
 146
 147         // Cut away everything under 120 Hz (or whatever the cutoff is);
 148         // we don't need it for voice, and it will reduce headroom
 149         // and confuse the compressor. (In particular, any hums at 50 or 60 Hz
 150         // should be dampened.)
 151         if (locut_enabled) {
 152                 locut.render(samples_out.data(), samples_out.size() / 2, locut_cutoff_hz * 2.0 * M_PI / OUTPUT_FREQUENCY, 0.5f);
 153         }
 154
 155         {
 156                 unique_lock<mutex> lock(compressor_mutex);
 157
 158                 // Apply a level compressor to get the general level right.
 159                 // Basically, if it's over about -40 dBFS, we squeeze it down to that level
 160                 // (or more precisely, near it, since we don't use infinite ratio),
 161                 // then apply a makeup gain to get it to -14 dBFS. -14 dBFS is, of course,
 162                 // entirely arbitrary, but from practical tests with speech, it seems to
 163                 // put ut around -23 LUFS, so it's a reasonable starting point for later use.
 164                 {
 165                         if (level_compressor_enabled) {
 166                                 float threshold = 0.01f;   // -40 dBFS.
 167                                 float ratio = 20.0f;
 168                                 float attack_time = 0.5f;
 169                                 float release_time = 20.0f;
 170                                 float makeup_gain = pow(10.0f, (ref_level_dbfs - (-40.0f)) / 20.0f);  // +26 dB.
 171                                 level_compressor.process(samples_out.data(), samples_out.size() / 2, threshold, ratio, attack_time, release_time, makeup_gain);
 172                                 gain_staging_db = 20.0 * log10(level_compressor.get_attenuation() * makeup_gain);
 173                         } else {
 174                                 // Just apply the gain we already had.
 175                                 float g = pow(10.0f, gain_staging_db / 20.0f);
 176                                 for (size_t i = 0; i < samples_out.size(); ++i) {
 177                                         samples_out[i] *= g;
 178                                 }
 179                         }
 180                 }
 181
 182         #if 0
 183                 printf("level=%f (%+5.2f dBFS) attenuation=%f (%+5.2f dB) end_result=%+5.2f dB\n",
 184                         level_compressor.get_level(), 20.0 * log10(level_compressor.get_level()),
 185                         level_compressor.get_attenuation(), 20.0 * log10(level_compressor.get_attenuation()),
 186                         20.0 * log10(level_compressor.get_level() * level_compressor.get_attenuation() * makeup_gain));
 187         #endif
 188
 189         //      float limiter_att, compressor_att;
 190
 191                 // The real compressor.
 192                 if (compressor_enabled) {
 193                         float threshold = pow(10.0f, compressor_threshold_dbfs / 20.0f);
 194                         float ratio = 20.0f;
 195                         float attack_time = 0.005f;
 196                         float release_time = 0.040f;
 197                         float makeup_gain = 2.0f;  // +6 dB.
 198                         compressor.process(samples_out.data(), samples_out.size() / 2, threshold, ratio, attack_time, release_time, makeup_gain);
 199         //              compressor_att = compressor.get_attenuation();
 200                 }
 201
 202                 // Finally a limiter at -4 dB (so, -10 dBFS) to take out the worst peaks only.
 203                 // Note that since ratio is not infinite, we could go slightly higher than this.
 204                 if (limiter_enabled) {
 205                         float threshold = pow(10.0f, limiter_threshold_dbfs / 20.0f);
 206                         float ratio = 30.0f;
 207                         float attack_time = 0.0f;  // Instant.
 208                         float release_time = 0.020f;
 209                         float makeup_gain = 1.0f;  // 0 dB.
 210                         limiter.process(samples_out.data(), samples_out.size() / 2, threshold, ratio, attack_time, release_time, makeup_gain);
 211         //              limiter_att = limiter.get_attenuation();
 212                 }
 213
 214         //      printf("limiter=%+5.1f  compressor=%+5.1f\n", 20.0*log10(limiter_att), 20.0*log10(compressor_att));
 215         }
 216
 217         // At this point, we are most likely close to +0 LU, but all of our
 218         // measurements have been on raw sample values, not R128 values.
 219         // So we have a final makeup gain to get us to +0 LU; the gain
 220         // adjustments required should be relatively small, and also, the
 221         // offset shouldn't change much (only if the type of audio changes
 222         // significantly). Thus, we shoot for updating this value basically
 223         // “whenever we process buffers”, since the R128 calculation isn't exactly
 224         // something we get out per-sample.
 225         //
 226         // Note that there's a feedback loop here, so we choose a very slow filter
 227         // (half-time of 100 seconds).
 228         double target_loudness_factor, alpha;
 229         double loudness_lu = loudness_lufs - ref_level_lufs;
 230         double current_makeup_lu = 20.0f * log10(final_makeup_gain);
 231         target_loudness_factor = pow(10.0f, -loudness_lu / 20.0f);
 232
 233         // If we're outside +/- 5 LU uncorrected, we don't count it as
 234         // a normal signal (probably silence) and don't change the
 235         // correction factor; just apply what we already have.
 236         if (fabs(loudness_lu - current_makeup_lu) >= 5.0 || !final_makeup_gain_auto) {
 237                 alpha = 0.0;
 238         } else {
 239                 // Formula adapted from
 240                 // https://en.wikipedia.org/wiki/Low-pass_filter#Simple_infinite_impulse_response_filter.
 241                 const double half_time_s = 100.0;
 242                 const double fc_mul_2pi_delta_t = 1.0 / (half_time_s * OUTPUT_FREQUENCY);
 243                 alpha = fc_mul_2pi_delta_t / (fc_mul_2pi_delta_t + 1.0);
 244         }
 245
 246         {
 247                 unique_lock<mutex> lock(compressor_mutex);
 248                 double m = final_makeup_gain;
 249                 for (size_t i = 0; i < samples_out.size(); i += 2) {
 250                         samples_out[i + 0] *= m;
 251                         samples_out[i + 1] *= m;
 252                         m += (target_loudness_factor - m) * alpha;
 253                 }
 254                 final_makeup_gain = m;
 255         }
 256
 257         return samples_out;
 258 }