From 86f32b5a80b232e6d824776a87ecc1b5f8fbfa96 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Sun, 22 Nov 2015 21:20:56 +0100 Subject: [PATCH] Rework signal connection in preparations for deinterlacing. Several improvements: - Hold three fields in history instead of one. - Make input_frames exact; no more chains holding on to input frames that they do not actually use for anything (so they can be released sooner). - Much less roundabout code to give the textures to the input; the LiveInputWrapper will now simply fetch them from the mixer itself. --- defs.h | 4 ++++ mixer.cpp | 45 ++++++++++++++++++--------------------------- mixer.h | 20 ++++++++++++++++++-- theme.cpp | 46 +++++++++++++++++++++++++++++++++++----------- theme.h | 33 +++++++++++++++++---------------- 5 files changed, 92 insertions(+), 56 deletions(-) diff --git a/defs.h b/defs.h index 9bed232..daef360 100644 --- a/defs.h +++ b/defs.h @@ -5,6 +5,10 @@ #define MAX_FPS 60 #define WIDTH 1280 #define HEIGHT 720 +#define MAX_CARDS 16 + +// For deinterlacing. See also comments on Mixer::frame_history. +#define FRAME_HISTORY_LENGTH 3 #define AUDIO_OUTPUT_CODEC AV_CODEC_ID_PCM_S32LE #define AUDIO_OUTPUT_SAMPLE_FMT AV_SAMPLE_FMT_S32 diff --git a/mixer.cpp b/mixer.cpp index 13fa299..b7e720d 100644 --- a/mixer.cpp +++ b/mixer.cpp @@ -525,7 +525,16 @@ void Mixer::thread_func() continue; assert(card->new_frame != nullptr); - bmusb_current_rendering_frame[card_index] = card->new_frame; + if (card->new_frame_interlaced) { + for (unsigned frame_num = FRAME_HISTORY_LENGTH; frame_num --> 1; ) { // :-) + buffered_frames[card_index][frame_num] = buffered_frames[card_index][frame_num - 1]; + } + buffered_frames[card_index][0] = { card->new_frame, card->new_frame_field }; + } else { + for (unsigned frame_num = 0; frame_num < FRAME_HISTORY_LENGTH; ++frame_num) { + buffered_frames[card_index][frame_num] = { card->new_frame, card->new_frame_field }; + } + } check_error(); // The new texture might still be uploaded, @@ -536,19 +545,12 @@ void Mixer::thread_func() glDeleteSync(card->new_data_ready_fence); check_error(); } - const PBOFrameAllocator::Userdata *userdata = (const PBOFrameAllocator::Userdata *)card->new_frame->userdata; - theme->set_input_textures( - card_index, - userdata->tex_y[card->new_frame_field], - userdata->tex_cbcr[card->new_frame_field], - userdata->last_width[card->new_frame_field], - userdata->last_height[card->new_frame_field]); } // Get the main chain from the theme, and set its state immediately. - pair> theme_main_chain = theme->get_chain(0, pts(), WIDTH, HEIGHT); - EffectChain *chain = theme_main_chain.first; - theme_main_chain.second(); + Theme::Chain theme_main_chain = theme->get_chain(0, pts(), WIDTH, HEIGHT); + EffectChain *chain = theme_main_chain.chain; + theme_main_chain.setup_chain(); GLuint y_tex, cbcr_tex; bool got_frame = h264_encoder->begin_frame(&y_tex, &cbcr_tex); @@ -575,15 +577,8 @@ void Mixer::thread_func() RefCountedGLsync fence(GL_SYNC_GPU_COMMANDS_COMPLETE, /*flags=*/0); check_error(); - // Make sure the H.264 gets a reference to all the - // input frames needed, so that they are not released back - // until the rendering is done. - vector input_frames; - for (unsigned card_index = 0; card_index < num_cards; ++card_index) { - input_frames.push_back(bmusb_current_rendering_frame[card_index]); - } const int64_t av_delay = TIMEBASE / 10; // Corresponds to the fixed delay in resampling_queue.h. TODO: Make less hard-coded. - h264_encoder->end_frame(fence, pts_int + av_delay, input_frames); + h264_encoder->end_frame(fence, pts_int + av_delay, theme_main_chain.input_frames); ++frame; pts_int += card_copy[0].new_frame_length; @@ -602,15 +597,11 @@ void Mixer::thread_func() // Set up preview and any additional channels. for (int i = 1; i < theme->get_num_channels() + 2; ++i) { DisplayFrame display_frame; - pair> chain = theme->get_chain(i, pts(), WIDTH, HEIGHT); // FIXME: dimensions - display_frame.chain = chain.first; - display_frame.setup_chain = chain.second; + Theme::Chain chain = theme->get_chain(i, pts(), WIDTH, HEIGHT); // FIXME: dimensions + display_frame.chain = chain.chain; + display_frame.setup_chain = chain.setup_chain; display_frame.ready_fence = fence; - - // FIXME: possible to do better? - for (unsigned card_index = 0; card_index < num_cards; ++card_index) { - display_frame.input_frames.push_back(bmusb_current_rendering_frame[card_index]); - } + display_frame.input_frames = chain.input_frames; display_frame.temp_textures = {}; output_channel[i].output_frame(display_frame); } diff --git a/mixer.h b/mixer.h index 76aec68..b9acfc4 100644 --- a/mixer.h +++ b/mixer.h @@ -170,6 +170,16 @@ public: void reset_meters(); + struct BufferedFrame { + RefCountedFrame frame; + unsigned field_number; + }; + + BufferedFrame get_buffered_frame(int card, int history_pos) + { + return buffered_frames[card][history_pos]; + } + private: void bm_frame(unsigned card_index, uint16_t timecode, FrameAllocator::Frame video_frame, size_t video_offset, uint16_t video_format, @@ -210,7 +220,8 @@ private: bool should_quit = false; RefCountedFrame new_frame; int64_t new_frame_length; // In TIMEBASE units. - unsigned new_frame_field; // Which field (0 or 1) of the frame to use. + bool new_frame_interlaced; + unsigned new_frame_field; // Which field (0 or 1) of the frame to use. Always 0 for progressive. GLsync new_data_ready_fence; // Whether new_frame is ready for rendering. std::condition_variable new_data_ready_changed; // Set whenever new_data_ready is changed. unsigned dropped_frames = 0; // Before new_frame. @@ -226,7 +237,12 @@ private: }; CaptureCard cards[MAX_CARDS]; // protected by - RefCountedFrame bmusb_current_rendering_frame[MAX_CARDS]; + // For each card, the last three frames (or fields), with 0 being the + // most recent one. Note that we only need the actual history if we have + // interlaced output (for deinterlacing), so if we detect progressive input, + // we immediately clear out all history and all entries will point to the same + // frame. + BufferedFrame buffered_frames[MAX_CARDS][FRAME_HISTORY_LENGTH]; class OutputChannel { public: diff --git a/theme.cpp b/theme.cpp index 61e149c..53a96d2 100644 --- a/theme.cpp +++ b/theme.cpp @@ -24,6 +24,7 @@ #include "defs.h" #include "image_input.h" +#include "mixer.h" namespace movit { class ResourcePool; @@ -32,6 +33,8 @@ class ResourcePool; using namespace std; using namespace movit; +extern Mixer *global_mixer; + namespace { vector live_inputs; @@ -441,7 +444,25 @@ LiveInputWrapper::LiveInputWrapper(Theme *theme, EffectChain *chain, bool overri void LiveInputWrapper::connect_signal(int signal_num) { - theme->connect_signal(input, signal_num); + if (global_mixer == nullptr) { + return; + } + + signal_num = theme->map_signal(signal_num); + + Mixer::BufferedFrame frame = global_mixer->get_buffered_frame(signal_num, 0); + const PBOFrameAllocator::Userdata *userdata = (const PBOFrameAllocator::Userdata *)frame.frame->userdata; + + input->set_texture_num(0, userdata->tex_y[frame.field_number]); + input->set_texture_num(1, userdata->tex_cbcr[frame.field_number]); + input->set_width(userdata->last_width[frame.field_number]); + input->set_height(userdata->last_height[frame.field_number]); + + // Hold on to the refcount so that we don't release the input frame + // until we're done rendering from it. + if (theme->used_input_frames_collector != nullptr) { + theme->used_input_frames_collector->push_back(frame.frame); + } } Theme::Theme(const char *filename, ResourcePool *resource_pool, unsigned num_cards) @@ -495,9 +516,10 @@ void Theme::register_class(const char *class_name, const luaL_Reg *funcs) assert(lua_gettop(L) == 0); } -pair> -Theme::get_chain(unsigned num, float t, unsigned width, unsigned height) +Theme::Chain Theme::get_chain(unsigned num, float t, unsigned width, unsigned height) { + Chain chain; + unique_lock lock(m); assert(lua_gettop(L) == 0); lua_getglobal(L, "get_chain"); /* function to be called */ @@ -506,12 +528,14 @@ Theme::get_chain(unsigned num, float t, unsigned width, unsigned height) lua_pushnumber(L, width); lua_pushnumber(L, height); + this->used_input_frames_collector = &chain.input_frames; if (lua_pcall(L, 4, 2, 0) != 0) { fprintf(stderr, "error running function `get_chain': %s\n", lua_tostring(L, -1)); exit(1); } + this->used_input_frames_collector = nullptr; - EffectChain *chain = (EffectChain *)luaL_checkudata(L, -2, "EffectChain"); + chain.chain = (EffectChain *)luaL_checkudata(L, -2, "EffectChain"); if (!lua_isfunction(L, -1)) { fprintf(stderr, "Argument #-1 should be a function\n"); exit(1); @@ -520,7 +544,8 @@ Theme::get_chain(unsigned num, float t, unsigned width, unsigned height) shared_ptr funcref(new LuaRefWithDeleter(&m, L, luaL_ref(L, LUA_REGISTRYINDEX))); lua_pop(L, 2); assert(lua_gettop(L) == 0); - return make_pair(chain, [this, funcref]{ + + chain.setup_chain = [this, funcref]{ unique_lock lock(m); // Set up state, including connecting signals. @@ -530,7 +555,9 @@ Theme::get_chain(unsigned num, float t, unsigned width, unsigned height) exit(1); } assert(lua_gettop(L) == 0); - }); + }; + + return chain; } std::string Theme::get_channel_name(unsigned channel) @@ -602,7 +629,7 @@ std::vector Theme::get_transition_names(float t) return ret; } -void Theme::connect_signal(YCbCrInput *input, int signal_num) +int Theme::map_signal(int signal_num) { if (signal_num >= int(num_cards)) { if (signals_warned_about.insert(signal_num).second) { @@ -611,10 +638,7 @@ void Theme::connect_signal(YCbCrInput *input, int signal_num) } signal_num %= num_cards; } - input->set_texture_num(0, input_textures[signal_num].tex_y); - input->set_texture_num(1, input_textures[signal_num].tex_cbcr); - input->set_width(input_textures[signal_num].width); - input->set_height(input_textures[signal_num].height); + return signal_num; } void Theme::transition_clicked(int transition_num, float t) diff --git a/theme.h b/theme.h index a5ecebc..2b3b4ab 100644 --- a/theme.h +++ b/theme.h @@ -16,6 +16,7 @@ #include #include "defs.h" +#include "ref_counted_frame.h" namespace movit { class ResourcePool; @@ -23,8 +24,6 @@ struct ImageFormat; struct YCbCrFormat; } // namespace movit -#define MAX_CARDS 16 - class NonBouncingYCbCrInput : public movit::YCbCrInput { public: NonBouncingYCbCrInput(const movit::ImageFormat &image_format, @@ -40,17 +39,18 @@ class Theme { public: Theme(const char *filename, movit::ResourcePool *resource_pool, unsigned num_cards); - std::pair> - get_chain(unsigned num, float t, unsigned width, unsigned height); + struct Chain { + movit::EffectChain *chain; + std::function setup_chain; - void set_input_textures(int signal_num, GLuint tex_y, GLuint tex_cbcr, GLuint width, GLuint height) { - auto &tex = input_textures[signal_num]; - tex.tex_y = tex_y; - tex.tex_cbcr = tex_cbcr; - tex.width = width; - tex.height = height; - } - int get_num_channels() { return num_channels; } + // May have duplicates. + std::vector input_frames; + }; + + Chain get_chain(unsigned num, float t, unsigned width, unsigned height); + + int get_num_channels() const { return num_channels; } + int map_signal(int signal_num); std::string get_channel_name(unsigned channel); bool get_supports_set_wb(unsigned channel); void set_wb(unsigned channel, double r, double g, double b); @@ -67,13 +67,14 @@ private: std::mutex m; lua_State *L; movit::ResourcePool *resource_pool; - struct { - GLuint tex_y = 0, tex_cbcr = 0; - GLuint width = WIDTH, height = HEIGHT; - } input_textures[MAX_CARDS]; int num_channels; unsigned num_cards; std::set signals_warned_about; + + // All input frames needed for the current chain. Filled during call to get_chain(), + // as inputs get connected. + std::vector *used_input_frames_collector; + friend class LiveInputWrapper; }; class LiveInputWrapper { -- 2.39.2