X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=mixer.h;h=39b259b45938b4a4dbbf9ffedbf36dcb9c4b5ed3;hb=177725e4f259a75dcdbd4260ac57e5dd9c01fd57;hp=821ceb2f5f606ad48133f7d8475f1fe7d40af12a;hpb=7f552730c721332e9f22286e5f82de5fd9492249;p=nageru diff --git a/mixer.h b/mixer.h index 821ceb2..39b259b 100644 --- a/mixer.h +++ b/mixer.h @@ -3,29 +3,34 @@ // The actual video mixer, running in its own separate background thread. +#include #include #undef Success -#include -#include #include #include -#include +#include +#include + #include +#include #include #include #include +#include #include #include +#include #include #include #include -#include "bmusb/bmusb.h" #include "alsa_output.h" -#include "ebu_r128_proc.h" -#include "h264encode.h" +#include "audio_mixer.h" +#include "bmusb/bmusb.h" +#include "defs.h" #include "httpd.h" +#include "input_state.h" #include "pbo_frame_allocator.h" #include "ref_counted_frame.h" #include "ref_counted_gl_sync.h" @@ -33,12 +38,11 @@ #include "theme.h" #include "timebase.h" #include "stereocompressor.h" -#include "filter.h" -#include "input_state.h" -#include "correlation_measurer.h" +#include "video_encoder.h" -class H264Encoder; +class ALSAOutput; class QSurface; +class QuickSyncEncoder; namespace movit { class Effect; class EffectChain; @@ -49,9 +53,53 @@ class ResourcePool; namespace movit { class YCbCrInput; } -class QOpenGLContext; class QSurfaceFormat; +// For any card that's not the master (where we pick out the frames as they +// come, as fast as we can process), there's going to be a queue. The question +// is when we should drop frames from that queue (apart from the obvious +// dropping if the 16-frame queue should become full), especially given that +// the frame rate could be lower or higher than the master (either subtly or +// dramatically). We have two (conflicting) demands: +// +// 1. We want to avoid starving the queue. +// 2. We don't want to add more delay than is needed. +// +// Our general strategy is to drop as many frames as we can (helping for #2) +// that we think is safe for #1 given jitter. To this end, we set a lower floor N, +// where we assume that if we have N frames in the queue, we're always safe from +// starvation. (Typically, N will be 0 or 1. It starts off at 0.) If we have +// more than N frames in the queue after reading out the one we need, we head-drop +// them to reduce the queue. +// +// N is reduced as follows: If the queue has had at least one spare frame for +// at least 50 (master) frames (ie., it's been too conservative for a second), +// we reduce N by 1 and reset the timers. TODO: Only do this if N ever actually +// touched the limit. +// +// Whenever the queue is starved (we needed a frame but there was none), +// and we've been at N since the last starvation, N was obviously too low, +// so we increment it. We will never set N above 5, though. +class QueueLengthPolicy { +public: + QueueLengthPolicy() {} + void reset(unsigned card_index) { + this->card_index = card_index; + safe_queue_length = 0; + frames_with_at_least_one = 0; + been_at_safe_point_since_last_starvation = false; + } + + void update_policy(int queue_length); // Give in -1 for starvation. + unsigned get_safe_queue_length() const { return safe_queue_length; } + +private: + unsigned card_index; // For debugging only. + unsigned safe_queue_length = 0; // Called N in the comments. + unsigned frames_with_at_least_one = 0; + bool been_at_safe_point_since_last_starvation = false; +}; + class Mixer { public: // The surface format is used for offscreen destinations for OpenGL contexts we need. @@ -102,13 +150,23 @@ public: output_channel[output].set_frame_ready_callback(callback); } - typedef std::function audio_level_callback_t; - void set_audio_level_callback(audio_level_callback_t callback) + // TODO: Should this really be per-channel? Shouldn't it just be called for e.g. the live output? + typedef std::function &)> transition_names_updated_callback_t; + void set_transition_names_updated_callback(Output output, transition_names_updated_callback_t callback) + { + output_channel[output].set_transition_names_updated_callback(callback); + } + + typedef std::function name_updated_callback_t; + void set_name_updated_callback(Output output, name_updated_callback_t callback) { - audio_level_callback = callback; + output_channel[output].set_name_updated_callback(callback); + } + + typedef std::function color_updated_callback_t; + void set_color_updated_callback(Output output, color_updated_callback_t callback) + { + output_channel[output].set_color_updated_callback(callback); } std::vector get_transition_names() @@ -126,6 +184,11 @@ public: return theme->get_channel_name(channel); } + std::string get_channel_color(unsigned channel) const + { + return theme->get_channel_color(channel); + } + int get_channel_signal(unsigned channel) const { return theme->get_channel_signal(channel); @@ -136,14 +199,14 @@ public: return theme->map_signal(channel); } - unsigned get_audio_source() const + unsigned get_master_clock() const { - return audio_source_channel; + return master_clock_channel; } - void set_audio_source(unsigned channel) + void set_master_clock(unsigned channel) { - audio_source_channel = channel; + master_clock_channel = channel; } void set_signal_mapping(int signal, int card) @@ -161,94 +224,86 @@ public: theme->set_wb(channel, r, g, b); } - void set_locut_cutoff(float cutoff_hz) - { - locut_cutoff_hz = cutoff_hz; - } + // Note: You can also get this through the global variable global_audio_mixer. + AudioMixer *get_audio_mixer() { return &audio_mixer; } + const AudioMixer *get_audio_mixer() const { return &audio_mixer; } - void set_locut_enabled(bool enabled) + void schedule_cut() { - locut_enabled = enabled; + should_cut = true; } - float get_limiter_threshold_dbfs() - { - return limiter_threshold_dbfs; - } + unsigned get_num_cards() const { return num_cards; } - float get_compressor_threshold_dbfs() - { - return compressor_threshold_dbfs; + std::string get_card_description(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].capture->get_description(); } - void set_limiter_threshold_dbfs(float threshold_dbfs) - { - limiter_threshold_dbfs = threshold_dbfs; + std::map get_available_video_modes(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].capture->get_available_video_modes(); } - void set_compressor_threshold_dbfs(float threshold_dbfs) - { - compressor_threshold_dbfs = threshold_dbfs; + uint32_t get_current_video_mode(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].capture->get_current_video_mode(); } - void set_limiter_enabled(bool enabled) - { - limiter_enabled = enabled; + void set_video_mode(unsigned card_index, uint32_t mode) { + assert(card_index < num_cards); + cards[card_index].capture->set_video_mode(mode); } - void set_compressor_enabled(bool enabled) - { - compressor_enabled = enabled; - } + void start_mode_scanning(unsigned card_index); - void set_gain_staging_db(float gain_db) - { - std::unique_lock lock(compressor_mutex); - level_compressor_enabled = false; - gain_staging_db = gain_db; + std::map get_available_video_inputs(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].capture->get_available_video_inputs(); } - void set_gain_staging_auto(bool enabled) - { - std::unique_lock lock(compressor_mutex); - level_compressor_enabled = enabled; + uint32_t get_current_video_input(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].capture->get_current_video_input(); } - void set_final_makeup_gain_db(float gain_db) - { - std::unique_lock lock(compressor_mutex); - final_makeup_gain_auto = false; - final_makeup_gain = pow(10.0f, gain_db / 20.0f); + void set_video_input(unsigned card_index, uint32_t input) { + assert(card_index < num_cards); + cards[card_index].capture->set_video_input(input); } - void set_final_makeup_gain_auto(bool enabled) - { - std::unique_lock lock(compressor_mutex); - final_makeup_gain_auto = enabled; + std::map get_available_audio_inputs(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].capture->get_available_audio_inputs(); } - void schedule_cut() - { - should_cut = true; + uint32_t get_current_audio_input(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].capture->get_current_audio_input(); } - void reset_meters(); - - unsigned get_num_cards() const { return num_cards; } - - std::string get_card_description(unsigned card_index) const { + void set_audio_input(unsigned card_index, uint32_t input) { assert(card_index < num_cards); - return cards[card_index].usb->get_description(); + cards[card_index].capture->set_audio_input(input); + } + + void change_x264_bitrate(unsigned rate_kbit) { + video_encoder->change_x264_bitrate(rate_kbit); } private: + void configure_card(unsigned card_index, bmusb::CaptureInterface *capture, bool is_fake_capture); void bm_frame(unsigned card_index, uint16_t timecode, - FrameAllocator::Frame video_frame, size_t video_offset, VideoFormat video_format, - FrameAllocator::Frame audio_frame, size_t audio_offset, uint16_t audio_format); + bmusb::FrameAllocator::Frame video_frame, size_t video_offset, bmusb::VideoFormat video_format, + bmusb::FrameAllocator::Frame audio_frame, size_t audio_offset, bmusb::AudioFormat audio_format); + void bm_hotplug_add(libusb_device *dev); + void bm_hotplug_remove(unsigned card_index); void place_rectangle(movit::Effect *resample_effect, movit::Effect *padding_effect, float x0, float y0, float x1, float y1); void thread_func(); + void handle_hotplugged_cards(); + void schedule_audio_resampling_tasks(unsigned dropped_frames, int num_samples_per_frame, int length_per_frame); + void render_one_frame(int64_t duration); void audio_thread_func(); - void process_audio_one_frame(int64_t frame_pts_int, int num_samples); void subsample_chroma(GLuint src_tex, GLuint dst_dst); void release_display_frame(DisplayFrame *frame); double pts() { return double(pts_int) / TIMEBASE; } @@ -260,11 +315,12 @@ private: std::unique_ptr resource_pool; std::unique_ptr theme; std::atomic audio_source_channel{0}; + std::atomic master_clock_channel{0}; std::unique_ptr display_chain; GLuint cbcr_program_num; // Owned by . GLuint cbcr_vbo; // Holds position and texcoord data. GLuint cbcr_position_attribute_index, cbcr_texcoord_attribute_index; - std::unique_ptr h264_encoder; + std::unique_ptr video_encoder; // Effects part of . Owned by . movit::FlatInput *display_input; @@ -272,53 +328,71 @@ private: int64_t pts_int = 0; // In TIMEBASE units. std::mutex bmusb_mutex; + bool has_bmusb_thread = false; struct CaptureCard { - BMUSBCapture *usb; + bmusb::CaptureInterface *capture = nullptr; + bool is_fake_capture; std::unique_ptr frame_allocator; // Stuff for the OpenGL context (for texture uploading). - QSurface *surface; - QOpenGLContext *context; - - bool new_data_ready = false; // Whether new_frame contains anything. + QSurface *surface = nullptr; + + struct NewFrame { + RefCountedFrame frame; + int64_t length; // In TIMEBASE units. + bool interlaced; + unsigned field; // Which field (0 or 1) of the frame to use. Always 0 for progressive. + std::function upload_func; // Needs to be called to actually upload the texture to OpenGL. + unsigned dropped_frames = 0; // Number of dropped frames before this one. + }; + std::queue new_frames; bool should_quit = false; - RefCountedFrame new_frame; - int64_t new_frame_length; // In TIMEBASE units. - bool new_frame_interlaced; - unsigned new_frame_field; // Which field (0 or 1) of the frame to use. Always 0 for progressive. - GLsync new_data_ready_fence; // Whether new_frame is ready for rendering. - std::condition_variable new_data_ready_changed; // Set whenever new_data_ready is changed. - unsigned dropped_frames = 0; // Before new_frame. + std::condition_variable new_frames_changed; // Set whenever new_frames (or should_quit) is changed. + + QueueLengthPolicy queue_length_policy; // Refers to the "new_frames" queue. // Accumulated errors in number of 1/TIMEBASE samples. If OUTPUT_FREQUENCY divided by // frame rate is integer, will always stay zero. unsigned fractional_samples = 0; - std::mutex audio_mutex; - std::unique_ptr resampling_queue; // Under audio_mutex. int last_timecode = -1; // Unwrapped. - int64_t next_local_pts = 0; // Beginning of next frame, in TIMEBASE units. }; - CaptureCard cards[MAX_CARDS]; // protected by + CaptureCard cards[MAX_VIDEO_CARDS]; // protected by + AudioMixer audio_mixer; // Same as global_audio_mixer (see audio_mixer.h). + void get_one_frame_from_each_card(unsigned master_card_index, CaptureCard::NewFrame new_frames[MAX_VIDEO_CARDS], bool has_new_frame[MAX_VIDEO_CARDS], int num_samples[MAX_VIDEO_CARDS]); InputState input_state; + // Cards we have been noticed about being hotplugged, but haven't tried adding yet. + // Protected by its own mutex. + std::mutex hotplug_mutex; + std::vector hotplugged_cards; + class OutputChannel { public: ~OutputChannel(); void output_frame(DisplayFrame frame); bool get_display_frame(DisplayFrame *frame); void set_frame_ready_callback(new_frame_ready_callback_t callback); + void set_transition_names_updated_callback(transition_names_updated_callback_t callback); + void set_name_updated_callback(name_updated_callback_t callback); + void set_color_updated_callback(color_updated_callback_t callback); private: friend class Mixer; + unsigned channel; Mixer *parent = nullptr; // Not owned. std::mutex frame_mutex; DisplayFrame current_frame, ready_frame; // protected by bool has_current_frame = false, has_ready_frame = false; // protected by new_frame_ready_callback_t new_frame_ready_callback; - bool has_new_frame_ready_callback = false; + transition_names_updated_callback_t transition_names_updated_callback; + name_updated_callback_t name_updated_callback; + color_updated_callback_t color_updated_callback; + + std::vector last_transition_names; + std::string last_name, last_color; }; OutputChannel output_channel[NUM_OUTPUTS]; @@ -327,47 +401,25 @@ private: std::atomic should_quit{false}; std::atomic should_cut{false}; - audio_level_callback_t audio_level_callback = nullptr; - std::mutex compressor_mutex; - Ebu_r128_proc r128; // Under compressor_mutex. - CorrelationMeasurer correlation; // Under compressor_mutex. - - Resampler peak_resampler; - std::atomic peak{0.0f}; - - StereoFilter locut; // Default cutoff 120 Hz, 24 dB/oct. - std::atomic locut_cutoff_hz; - std::atomic locut_enabled{true}; - - // First compressor; takes us up to about -12 dBFS. - StereoCompressor level_compressor; // Under compressor_mutex. Used to set/override gain_staging_db if . - float gain_staging_db = 0.0f; // Under compressor_mutex. - bool level_compressor_enabled = true; // Under compressor_mutex. - - static constexpr float ref_level_dbfs = -14.0f; // Chosen so that we end up around 0 LU in practice. - static constexpr float ref_level_lufs = -23.0f; // 0 LU, more or less by definition. - - StereoCompressor limiter; - std::atomic limiter_threshold_dbfs{ref_level_dbfs + 4.0f}; // 4 dB. - std::atomic limiter_enabled{true}; - StereoCompressor compressor; - std::atomic compressor_threshold_dbfs{ref_level_dbfs - 12.0f}; // -12 dB. - std::atomic compressor_enabled{true}; - - double final_makeup_gain = 1.0; // Under compressor_mutex. Read/write by the user. Note: Not in dB, we want the numeric precision so that we can change it slowly. - bool final_makeup_gain_auto = true; // Under compressor_mutex. - std::unique_ptr alsa; struct AudioTask { int64_t pts_int; int num_samples; + bool adjust_rate; }; std::mutex audio_mutex; std::condition_variable audio_task_queue_changed; std::queue audio_task_queue; // Under audio_mutex. + + // For mode scanning. + bool is_mode_scanning[MAX_VIDEO_CARDS]{ false }; + std::vector mode_scanlist[MAX_VIDEO_CARDS]; + unsigned mode_scanlist_index[MAX_VIDEO_CARDS]{ 0 }; + std::chrono::steady_clock::time_point last_mode_scan_change[MAX_VIDEO_CARDS]; }; extern Mixer *global_mixer; +extern bool uses_mlock; #endif // !defined(_MIXER_H)