X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=mixer.h;h=6f04c67844113d53010fa3ca93a451c0f6fcb821;hb=f006b5b162841dbc764fb620025b87a3272ac79a;hp=ef112b8c73c07648bba43fd262d3f54c87fc5094;hpb=adaeecd2ab8f1da45169861b7af22a0c7af066be;p=nageru diff --git a/mixer.h b/mixer.h index ef112b8..6f04c67 100644 --- a/mixer.h +++ b/mixer.h @@ -52,6 +52,51 @@ class YCbCrInput; class QOpenGLContext; class QSurfaceFormat; +// For any card that's not the master (where we pick out the frames as they +// come, as fast as we can process), there's going to be a queue. The question +// is when we should drop frames from that queue (apart from the obvious +// dropping if the 16-frame queue should become full), especially given that +// the frame rate could be lower or higher than the master (either subtly or +// dramatically). We have two (conflicting) demands: +// +// 1. We want to avoid starving the queue. +// 2. We don't want to add more delay than is needed. +// +// Our general strategy is to drop as many frames as we can (helping for #2) +// that we think is safe for #1 given jitter. To this end, we set a lower floor N, +// where we assume that if we have N frames in the queue, we're always safe from +// starvation. (Typically, N will be 0 or 1. It starts off at 0.) If we have +// more than N frames in the queue after reading out the one we need, we head-drop +// them to reduce the queue. +// +// N is reduced as follows: If the queue has had at least one spare frame for +// at least 50 (master) frames (ie., it's been too conservative for a second), +// we reduce N by 1 and reset the timers. TODO: Only do this if N ever actually +// touched the limit. +// +// Whenever the queue is starved (we needed a frame but there was none), +// and we've been at N since the last starvation, N was obviously too low, +// so we increment it. We will never set N above 5, though. +class QueueLengthPolicy { +public: + QueueLengthPolicy() {} + void reset(unsigned card_index) { + this->card_index = card_index; + safe_queue_length = 0; + frames_with_at_least_one = 0; + been_at_safe_point_since_last_starvation = false; + } + + void update_policy(int queue_length); // Give in -1 for starvation. + unsigned get_safe_queue_length() const { return safe_queue_length; } + +private: + unsigned card_index; // For debugging only. + unsigned safe_queue_length = 0; // Called N in the comments. + unsigned frames_with_at_least_one = 0; + bool been_at_safe_point_since_last_starvation = false; +}; + class Mixer { public: // The surface format is used for offscreen destinations for OpenGL contexts we need. @@ -126,6 +171,11 @@ public: return theme->get_channel_name(channel); } + std::string get_channel_color(unsigned channel) const + { + return theme->get_channel_color(channel); + } + int get_channel_signal(unsigned channel) const { return theme->get_channel_signal(channel); @@ -146,6 +196,16 @@ public: audio_source_channel = channel; } + unsigned get_master_clock() const + { + return master_clock_channel; + } + + void set_master_clock(unsigned channel) + { + master_clock_channel = channel; + } + void set_signal_mapping(int signal, int card) { return theme->set_signal_mapping(signal, card); @@ -238,15 +298,66 @@ public: std::string get_card_description(unsigned card_index) const { assert(card_index < num_cards); - return cards[card_index].usb->get_description(); + return cards[card_index].capture->get_description(); + } + + std::map get_available_video_modes(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].capture->get_available_video_modes(); + } + + uint32_t get_current_video_mode(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].capture->get_current_video_mode(); + } + + void set_video_mode(unsigned card_index, uint32_t mode) { + assert(card_index < num_cards); + cards[card_index].capture->set_video_mode(mode); + } + + void start_mode_scanning(unsigned card_index); + + std::map get_available_video_inputs(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].capture->get_available_video_inputs(); + } + + uint32_t get_current_video_input(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].capture->get_current_video_input(); + } + + void set_video_input(unsigned card_index, uint32_t input) { + assert(card_index < num_cards); + cards[card_index].capture->set_video_input(input); + } + + std::map get_available_audio_inputs(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].capture->get_available_audio_inputs(); + } + + uint32_t get_current_audio_input(unsigned card_index) const { + assert(card_index < num_cards); + return cards[card_index].capture->get_current_audio_input(); + } + + void set_audio_input(unsigned card_index, uint32_t input) { + assert(card_index < num_cards); + cards[card_index].capture->set_audio_input(input); } private: + void configure_card(unsigned card_index, const QSurfaceFormat &format, CaptureInterface *capture); void bm_frame(unsigned card_index, uint16_t timecode, - FrameAllocator::Frame video_frame, size_t video_offset, uint16_t video_format, - FrameAllocator::Frame audio_frame, size_t audio_offset, uint16_t audio_format); + FrameAllocator::Frame video_frame, size_t video_offset, VideoFormat video_format, + FrameAllocator::Frame audio_frame, size_t audio_offset, AudioFormat audio_format); void place_rectangle(movit::Effect *resample_effect, movit::Effect *padding_effect, float x0, float y0, float x1, float y1); void thread_func(); + void schedule_audio_resampling_tasks(unsigned dropped_frames, int num_samples_per_frame, int length_per_frame); + void render_one_frame(); + void send_audio_level_callback(); void audio_thread_func(); void process_audio_one_frame(int64_t frame_pts_int, int num_samples); void subsample_chroma(GLuint src_tex, GLuint dst_dst); @@ -260,8 +371,11 @@ private: std::unique_ptr resource_pool; std::unique_ptr theme; std::atomic audio_source_channel{0}; + std::atomic master_clock_channel{0}; std::unique_ptr display_chain; GLuint cbcr_program_num; // Owned by . + GLuint cbcr_vbo; // Holds position and texcoord data. + GLuint cbcr_position_attribute_index, cbcr_texcoord_attribute_index; std::unique_ptr h264_encoder; // Effects part of . Owned by . @@ -271,22 +385,26 @@ private: std::mutex bmusb_mutex; struct CaptureCard { - BMUSBCapture *usb; + CaptureInterface *capture; std::unique_ptr frame_allocator; // Stuff for the OpenGL context (for texture uploading). QSurface *surface; QOpenGLContext *context; - bool new_data_ready = false; // Whether new_frame contains anything. + struct NewFrame { + RefCountedFrame frame; + int64_t length; // In TIMEBASE units. + bool interlaced; + unsigned field; // Which field (0 or 1) of the frame to use. Always 0 for progressive. + RefCountedGLsync ready_fence; // Whether frame is ready for rendering. + unsigned dropped_frames = 0; // Number of dropped frames before this one. + }; + std::queue new_frames; bool should_quit = false; - RefCountedFrame new_frame; - int64_t new_frame_length; // In TIMEBASE units. - bool new_frame_interlaced; - unsigned new_frame_field; // Which field (0 or 1) of the frame to use. Always 0 for progressive. - GLsync new_data_ready_fence; // Whether new_frame is ready for rendering. - std::condition_variable new_data_ready_changed; // Set whenever new_data_ready is changed. - unsigned dropped_frames = 0; // Before new_frame. + std::condition_variable new_frames_changed; // Set whenever new_frames (or should_quit) is changed. + + QueueLengthPolicy queue_length_policy; // Refers to the "new_frames" queue. // Accumulated errors in number of 1/TIMEBASE samples. If OUTPUT_FREQUENCY divided by // frame rate is integer, will always stay zero. @@ -298,6 +416,7 @@ private: int64_t next_local_pts = 0; // Beginning of next frame, in TIMEBASE units. }; CaptureCard cards[MAX_CARDS]; // protected by + void get_one_frame_from_each_card(unsigned master_card_index, CaptureCard::NewFrame new_frames[MAX_CARDS], bool has_new_frame[MAX_CARDS], int num_samples[MAX_CARDS]); InputState input_state; @@ -364,6 +483,12 @@ private: std::mutex audio_mutex; std::condition_variable audio_task_queue_changed; std::queue audio_task_queue; // Under audio_mutex. + + // For mode scanning. + bool is_mode_scanning[MAX_CARDS]{ false }; + std::vector mode_scanlist[MAX_CARDS]; + unsigned mode_scanlist_index[MAX_CARDS]{ 0 }; + timespec last_mode_scan_change[MAX_CARDS]; }; extern Mixer *global_mixer;