From 44d0e2ac6f2c3b969244054efb2f80498756aeb1 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Wed, 7 Oct 2015 23:43:33 +0200 Subject: [PATCH] Initial implementation of moving the theming logic to Lua. Still lots to do. --- Makefile | 4 +- mixer.cpp | 57 +++---------- mixer.h | 8 +- theme.cpp | 250 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ theme.h | 65 ++++++++++++++ theme.lua | 77 +++++++++++++++++ 6 files changed, 408 insertions(+), 53 deletions(-) create mode 100644 theme.cpp create mode 100644 theme.h create mode 100644 theme.lua diff --git a/Makefile b/Makefile index 624ddcd..79d7bad 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ CXX=g++ -PKG_MODULES = Qt5Core Qt5Gui Qt5Widgets Qt5OpenGLExtensions Qt5OpenGL libusb-1.0 movit +PKG_MODULES = Qt5Core Qt5Gui Qt5Widgets Qt5OpenGLExtensions Qt5OpenGL libusb-1.0 movit lua5.2 CXXFLAGS := -O2 -march=native -g -std=gnu++11 -Wall -Wno-deprecated-declarations -fPIC $(shell pkg-config --cflags $(PKG_MODULES)) -pthread -DMOVIT_SHADER_DIR=\"$(shell pkg-config --variable=shaderdir movit)\" LDFLAGS=$(shell pkg-config --libs $(PKG_MODULES)) -lEGL -lGL -pthread -lva -lva-drm -lva-x11 -lX11 -lavformat -lavcodec -lavutil @@ -8,7 +8,7 @@ OBJS=glwidget.o main.o mainwindow.o window.o OBJS += glwidget.moc.o mainwindow.moc.o window.moc.o # Mixer objects -OBJS += h264encode.o mixer.o bmusb.o pbo_frame_allocator.o context.o ref_counted_frame.o +OBJS += h264encode.o mixer.o bmusb.o pbo_frame_allocator.o context.o ref_counted_frame.o theme.o %.o: %.cpp $(CXX) -MMD -MP $(CPPFLAGS) $(CXXFLAGS) -o $@ -c $< diff --git a/mixer.cpp b/mixer.cpp index 6de33f0..4713702 100644 --- a/mixer.cpp +++ b/mixer.cpp @@ -58,6 +58,7 @@ Mixer::Mixer(const QSurfaceFormat &format) check_error(); resource_pool.reset(new ResourcePool); + theme.reset(new Theme("theme.lua", resource_pool.get())); output_channel[OUTPUT_LIVE].parent = this; output_channel[OUTPUT_PREVIEW].parent = this; output_channel[OUTPUT_INPUT0].parent = this; @@ -77,39 +78,6 @@ Mixer::Mixer(const QSurfaceFormat &format) input_ycbcr_format.luma_coefficients = YCBCR_REC_601; input_ycbcr_format.full_range = false; - YCbCrFormat output_ycbcr_format; - output_ycbcr_format.chroma_subsampling_x = 1; - output_ycbcr_format.chroma_subsampling_y = 1; - output_ycbcr_format.luma_coefficients = YCBCR_REC_601; - output_ycbcr_format.full_range = false; - - // Main chain. - chain.reset(new EffectChain(WIDTH, HEIGHT, resource_pool.get())); - check_error(); - input[0] = new YCbCrInput(inout_format, input_ycbcr_format, WIDTH, HEIGHT, YCBCR_INPUT_SPLIT_Y_AND_CBCR); - chain->add_input(input[0]); - input[1] = new YCbCrInput(inout_format, input_ycbcr_format, WIDTH, HEIGHT, YCBCR_INPUT_SPLIT_Y_AND_CBCR); - chain->add_input(input[1]); - resample_effect = chain->add_effect(new ResampleEffect(), input[0]); - padding_effect = chain->add_effect(new IntegralPaddingEffect()); - float border_color[] = { 0.0f, 0.0f, 0.0f, 1.0f }; - CHECK(padding_effect->set_vec4("border_color", border_color)); - - resample2_effect = chain->add_effect(new ResampleEffect(), input[1]); - Effect *saturation_effect = chain->add_effect(new SaturationEffect()); - CHECK(saturation_effect->set_float("saturation", 0.3f)); - Effect *wb_effect = chain->add_effect(new WhiteBalanceEffect()); - CHECK(wb_effect->set_float("output_color_temperature", 3500.0)); - padding2_effect = chain->add_effect(new IntegralPaddingEffect()); - - chain->add_effect(new OverlayEffect(), padding_effect, padding2_effect); - - chain->add_output(inout_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED); - chain->add_ycbcr_output(inout_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED, output_ycbcr_format, YCBCR_OUTPUT_SPLIT_Y_AND_CBCR); - chain->set_dither_bits(8); - chain->set_output_origin(OUTPUT_ORIGIN_TOP_LEFT); - chain->finalize(); - // Display chain; shows the live output produced by the main chain (its RGBA version). display_chain.reset(new EffectChain(WIDTH, HEIGHT, resource_pool.get())); check_error(); @@ -162,8 +130,6 @@ Mixer::Mixer(const QSurfaceFormat &format) for (int card_index = 0; card_index < NUM_CARDS; ++card_index) { cards[card_index].usb->start_bm_capture(); - input[card_index]->set_pixel_data(0, nullptr, 0); - input[card_index]->set_pixel_data(1, nullptr, 0); } //chain->enable_phase_timing(true); @@ -388,8 +354,10 @@ void Mixer::thread_func() right1 = right1 * scale0 + tx0; } +#if 0 place_rectangle(resample_effect, padding_effect, left0, top0, right0, bottom0); place_rectangle(resample2_effect, padding2_effect, left1, top1, right1, bottom1); +#endif CaptureCard card_copy[NUM_CARDS]; @@ -416,6 +384,7 @@ void Mixer::thread_func() if (!card->new_data_ready) continue; + assert(card->new_frame != nullptr); bmusb_current_rendering_frame[card_index] = card->new_frame; check_error(); @@ -427,16 +396,14 @@ void Mixer::thread_func() glDeleteSync(card->new_data_ready_fence); check_error(); const PBOFrameAllocator::Userdata *userdata = (const PBOFrameAllocator::Userdata *)card->new_frame->userdata; - input[card_index]->set_texture_num(0, userdata->tex_y); - input[card_index]->set_texture_num(1, userdata->tex_cbcr); - - if (NUM_CARDS == 1) { - // Set to the other one, too. - input[1]->set_texture_num(0, userdata->tex_y); - input[1]->set_texture_num(1, userdata->tex_cbcr); - } + theme->set_input_textures(card_index, userdata->tex_y, userdata->tex_cbcr); } + // Get the main chain from the theme, and set its state immediately. + pair> theme_main_chain = theme->get_chain(0, frame / 60.0f, WIDTH, HEIGHT); + EffectChain *chain = theme_main_chain.first; + theme_main_chain.second(); + GLuint y_tex, cbcr_tex; bool got_frame = h264_encoder->begin_frame(&y_tex, &cbcr_tex); assert(got_frame); @@ -484,7 +451,7 @@ void Mixer::thread_func() // The preview frame shows the first input. Note that the textures // are owned by the input frame, not the display frame. - { + if (bmusb_current_rendering_frame[0] != nullptr) { const PBOFrameAllocator::Userdata *input0_userdata = (const PBOFrameAllocator::Userdata *)bmusb_current_rendering_frame[0]->userdata; GLuint input0_y_tex = input0_userdata->tex_y; GLuint input0_cbcr_tex = input0_userdata->tex_cbcr; @@ -503,7 +470,7 @@ void Mixer::thread_func() // Same for the other preview. // TODO: Use a for loop. Gah. - { + if (bmusb_current_rendering_frame[1] != nullptr) { const PBOFrameAllocator::Userdata *input1_userdata = (const PBOFrameAllocator::Userdata *)bmusb_current_rendering_frame[1]->userdata; GLuint input1_y_tex = input1_userdata->tex_y; GLuint input1_cbcr_tex = input1_userdata->tex_cbcr; diff --git a/mixer.h b/mixer.h index 03aab6d..96613f9 100644 --- a/mixer.h +++ b/mixer.h @@ -14,6 +14,7 @@ #include "pbo_frame_allocator.h" #include "ref_counted_frame.h" #include "ref_counted_gl_sync.h" +#include "theme.h" #define NUM_CARDS 2 @@ -89,18 +90,13 @@ private: QSurface *mixer_surface, *h264_encoder_surface; std::unique_ptr resource_pool; - std::unique_ptr chain; + std::unique_ptr theme; std::unique_ptr display_chain; std::unique_ptr preview0_chain; std::unique_ptr preview1_chain; GLuint cbcr_program_num; // Owned by . std::unique_ptr h264_encoder; - // Effects part of . Owned by . - movit::YCbCrInput *input[NUM_CARDS]; - movit::Effect *resample_effect, *resample2_effect; - movit::Effect *padding_effect, *padding2_effect; - // Effects part of . Owned by . movit::FlatInput *display_input; diff --git a/theme.cpp b/theme.cpp new file mode 100644 index 0000000..26f7937 --- /dev/null +++ b/theme.cpp @@ -0,0 +1,250 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "theme.h" + +#define WIDTH 1280 // FIXME +#define HEIGHT 720 // FIXME + +using namespace std; +using namespace movit; + +namespace { + +vector live_inputs; + +template +int wrap_lua_object(lua_State* L, const char *class_name, Args&&... args) +{ + // Construct the C++ object and put it on the stack. + void *mem = lua_newuserdata(L, sizeof(T)); + new(mem) T(std::forward(args)...); + + // Look up the metatable named , and set it on the new object. + luaL_getmetatable(L, class_name); + lua_setmetatable(L, -2); + + return 1; +} + +Theme *get_theme_updata(lua_State* L) +{ + luaL_checktype(L, lua_upvalueindex(1), LUA_TLIGHTUSERDATA); + return (Theme *)lua_touserdata(L, lua_upvalueindex(1)); +} + +bool checkbool(lua_State* L, int idx) +{ + luaL_checktype(L, idx, LUA_TBOOLEAN); + return lua_toboolean(L, idx); +} + +int EffectChain_new(lua_State* L) +{ + assert(lua_gettop(L) == 2); + int aspect_w = luaL_checknumber(L, 1); + int aspect_h = luaL_checknumber(L, 2); + + return wrap_lua_object(L, "EffectChain", aspect_w, aspect_h); +} + +int EffectChain_add_live_input(lua_State* L) +{ + assert(lua_gettop(L) == 1); + Theme *theme = get_theme_updata(L); + EffectChain *chain = (EffectChain *)luaL_checkudata(L, 1, "EffectChain"); + return wrap_lua_object(L, "LiveInputWrapper", theme, chain); +} + +int EffectChain_add_effect(lua_State* L) +{ + assert(lua_gettop(L) == 3); + EffectChain *chain = (EffectChain *)luaL_checkudata(L, 1, "EffectChain"); + + // FIXME: This needs a lot of work. + Effect *effect1 = (Effect *)luaL_checkudata(L, 2, "WhiteBalanceEffect"); + LiveInputWrapper *effect2 = (LiveInputWrapper *)luaL_checkudata(L, 3, "LiveInputWrapper"); + chain->add_effect(effect1, effect2->get_input()); + + lua_settop(L, 2); + return 1; +} + +int EffectChain_finalize(lua_State* L) +{ + assert(lua_gettop(L) == 2); + EffectChain *chain = (EffectChain *)luaL_checkudata(L, 1, "EffectChain"); + bool is_main_chain = checkbool(L, 2); + + // Add outputs as needed. + ImageFormat inout_format; + inout_format.color_space = COLORSPACE_sRGB; + inout_format.gamma_curve = GAMMA_sRGB; + if (is_main_chain) { + YCbCrFormat output_ycbcr_format; + output_ycbcr_format.chroma_subsampling_x = 1; + output_ycbcr_format.chroma_subsampling_y = 1; + output_ycbcr_format.luma_coefficients = YCBCR_REC_601; + output_ycbcr_format.full_range = false; + + chain->add_ycbcr_output(inout_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED, output_ycbcr_format, YCBCR_OUTPUT_SPLIT_Y_AND_CBCR); + chain->set_dither_bits(8); + chain->set_output_origin(OUTPUT_ORIGIN_TOP_LEFT); + } + chain->add_output(inout_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED); + + chain->finalize(); + return 0; +} + +int LiveInputWrapper_connect_signal(lua_State* L) +{ + assert(lua_gettop(L) == 2); + LiveInputWrapper *input = (LiveInputWrapper *)luaL_checkudata(L, 1, "LiveInputWrapper"); + int signal_num = luaL_checknumber(L, 2); + input->connect_signal(signal_num); + return 0; +} + +int WhiteBalanceEffect_new(lua_State* L) +{ + assert(lua_gettop(L) == 0); + return wrap_lua_object(L, "WhiteBalanceEffect"); +} + +int WhiteBalanceEffect_set_float(lua_State *L) +{ + assert(lua_gettop(L) == 3); + WhiteBalanceEffect *effect = (WhiteBalanceEffect *)luaL_checkudata(L, 1, "WhiteBalanceEffect"); + size_t len; + const char* cstr = lua_tolstring(L, 2, &len); + std::string key(cstr, len); + float value = luaL_checknumber(L, 3); + (void)effect->set_float(key, value); + return 0; +} + +const luaL_Reg EffectChain_funcs[] = { + { "new", EffectChain_new }, + { "add_live_input", EffectChain_add_live_input }, + { "add_effect", EffectChain_add_effect }, + { "finalize", EffectChain_finalize }, + { NULL, NULL } +}; + +const luaL_Reg LiveInputWrapper_funcs[] = { + { "connect_signal", LiveInputWrapper_connect_signal }, + { NULL, NULL } +}; + +const luaL_Reg WhiteBalanceEffect_funcs[] = { + { "new", WhiteBalanceEffect_new }, + { "set_float", WhiteBalanceEffect_set_float }, + { NULL, NULL } +}; + +} // namespace + +LiveInputWrapper::LiveInputWrapper(Theme *theme, EffectChain *chain) + : theme(theme) +{ + ImageFormat inout_format; + inout_format.color_space = COLORSPACE_sRGB; + inout_format.gamma_curve = GAMMA_sRGB; + + YCbCrFormat input_ycbcr_format; + input_ycbcr_format.chroma_subsampling_x = 2; + input_ycbcr_format.chroma_subsampling_y = 1; + input_ycbcr_format.cb_x_position = 0.0; + input_ycbcr_format.cr_x_position = 0.0; + input_ycbcr_format.cb_y_position = 0.5; + input_ycbcr_format.cr_y_position = 0.5; + input_ycbcr_format.luma_coefficients = YCBCR_REC_601; + input_ycbcr_format.full_range = false; + + input = new YCbCrInput(inout_format, input_ycbcr_format, WIDTH, HEIGHT, YCBCR_INPUT_SPLIT_Y_AND_CBCR); + chain->add_input(input); +} + +void LiveInputWrapper::connect_signal(int signal_num) +{ + theme->connect_signal(input, signal_num); +} + +Theme::Theme(const char *filename, ResourcePool *resource_pool) + : resource_pool(resource_pool) +{ + L = luaL_newstate(); + luaL_openlibs(L); + + printf("constructing, this=%p\n", this); + + register_class("EffectChain", EffectChain_funcs); + register_class("LiveInputWrapper", LiveInputWrapper_funcs); + register_class("WhiteBalanceEffect", WhiteBalanceEffect_funcs); + + // Run script. + lua_settop(L, 0); + if (luaL_dofile(L, filename)) { + fprintf(stderr, "error: %s\n", lua_tostring(L, -1)); + lua_pop(L, 1); + exit(1); + } + assert(lua_gettop(L) == 0); +} + +void Theme::register_class(const char *class_name, const luaL_Reg *funcs) +{ + luaL_newmetatable(L, class_name); + lua_pushlightuserdata(L, this); + luaL_setfuncs(L, funcs, 1); + lua_pushvalue(L, -1); + lua_setfield(L, -2, "__index"); + lua_setglobal(L, class_name); +} + +pair> +Theme::get_chain(unsigned num, float t, unsigned width, unsigned height) +{ + unique_lock lock(m); + lua_getglobal(L, "get_chain"); /* function to be called */ + lua_pushnumber(L, num); + lua_pushnumber(L, t); + lua_pushnumber(L, width); + lua_pushnumber(L, height); + + if (lua_pcall(L, 4, 2, 0) != 0) { + fprintf(stderr, "error running function `get_chain': %s", lua_tostring(L, -1)); + exit(1); + } + + EffectChain *chain = (EffectChain *)luaL_checkudata(L, -2, "EffectChain"); + if (!lua_isfunction(L, -1)) { + fprintf(stderr, "Argument #-1 should be a function\n"); + exit(1); + } + lua_pushvalue(L, -1); + int funcref = luaL_ref(L, LUA_REGISTRYINDEX); // TODO: leak! + lua_pop(L, 2); + return make_pair(chain, [this, funcref]{ + unique_lock lock(m); + + // Set up state, including connecting signals. + lua_rawgeti(L, LUA_REGISTRYINDEX, funcref); + lua_pcall(L, 0, 0, 0); + }); +} + +void Theme::connect_signal(YCbCrInput *input, int signal_num) +{ + input->set_texture_num(0, input_textures[signal_num].tex_y); + input->set_texture_num(1, input_textures[signal_num].tex_cbcr); +} diff --git a/theme.h b/theme.h new file mode 100644 index 0000000..2e06347 --- /dev/null +++ b/theme.h @@ -0,0 +1,65 @@ +#ifndef _THEME_H +#define _THEME_H 1 + +#include +#include +#include + +#include +#include +#include + +#include +#include + +class Theme { +public: + Theme(const char *filename, movit::ResourcePool *resource_pool); + void register_class(const char *class_name, const luaL_Reg *funcs); + + std::pair> + get_chain(unsigned num, float t, unsigned width, unsigned height); + + void set_input_textures(int signal_num, GLuint tex_y, GLuint tex_cbcr) { + input_textures[signal_num].tex_y = tex_y; + input_textures[signal_num].tex_cbcr = tex_cbcr; + } + + void connect_signal(movit::YCbCrInput *input, int signal_num); + +private: + std::mutex m; + lua_State *L; + movit::ResourcePool *resource_pool; + struct { + GLuint tex_y = 0, tex_cbcr = 0; + } input_textures[16]; // FIXME +}; + +class LiveInputWrapper { +public: + LiveInputWrapper(Theme *theme, movit::EffectChain *chain); + + void connect_signal(int signal_num); +#if 0 + { + connected_signal_num = signal_num; + } + + int get_connected_signal_num() const { + return connected_signal_num; + } +#endif + + movit::YCbCrInput *get_input() const + { + return input; + } + +private: + Theme *theme; // Not owned by us. + movit::YCbCrInput *input; // Owned by the chain. + int connected_signal_num = 0; +}; + +#endif // !defined(_THEME_H) diff --git a/theme.lua b/theme.lua new file mode 100644 index 0000000..d7e46d6 --- /dev/null +++ b/theme.lua @@ -0,0 +1,77 @@ +-- The theme is what decides what's actually shown on screen, what kind of +-- transitions are available (if any), and what kind of inputs there are, +-- if any. In general, it drives the entire display logic by creating Movit +-- chains, setting their parameters and then deciding which to show when. +-- +-- Themes are written in Lua, which reflects a simplified form of the Movit API +-- where all the low-level details (such as texture formats) are handled by the +-- C++ side and you generally just build chains. +io.write("hello from lua\n"); + +-- A chain to show input 0 on screen. +local input0_chain = EffectChain.new(16, 9); +--input0_chain:add_input(Inputs.create(0)); -- TODO: We probably want something more fluid. +local preview_input0 = input0_chain:add_live_input(); +preview_input0:connect_signal(0); -- First input card. Can be changed whenever you want. +input0_chain:finalize(false); + +-- The main live chain. Currently just about input 0 with some color correction. +local main_chain = EffectChain.new(16, 9); +-- local input0 = main_chain:add_input(Inputs.create(0)); +local input0 = main_chain:add_live_input(); +input0:connect_signal(0); +local wb_effect = main_chain:add_effect(WhiteBalanceEffect.new(), input0); +wb_effect:set_float("output_color_temperature", 1234.0); +main_chain:finalize(true); +-- local input1 = main_chain.add_input(Inputs.create(1)); +-- local resample_effect = main_chain.add_effect(ResampleEffect.new(), input0); +-- local padding_effect = main_chain.add_effect(IntegralPaddingEffect.new()); + +-- Returns the number of outputs in addition to the live (0) and preview (1). +-- Called only once, at the start of the program. +function num_channels() + return 0; +end + +-- Called every frame. +function get_transitions() + return {"Cut", "Fade", "Zoom!"}; +end + +function transition_clicked(num, t) + -- Presumably do some sort of transition here. + io.write("STUB: transition_clicked\n"); +end + +function channel_clicked(num, t) + -- Presumably change the preview here. + io.write("STUB: channel_clicked\n"); +end + +-- Called every frame. Get the chain for displaying at input , +-- where 0 is live, 1 is preview, 2 is the first channel to display +-- in the bottom bar, and so on up to num_channels()+1. t is the +-- current time in seconds. width and height are the dimensions of +-- the output, although you can ignore them if you don't need them +-- (they're useful if you want to e.g. know what to resample by). +-- If you want to change any parameters in the chain, this is also +-- the right place. +-- +-- NOTE: The chain returned must be finalized with the Y'CbCr flag +-- if and only if num==0. +function get_chain(num, t, width, height) + if num == 0 then + prepare = function() + -- io.write("prepare is being called back\n"); + input0:connect_signal(1); + wb_effect:set_float("output_color_temperature", 3500.0 + t * 100.0); + end + return main_chain, prepare; + end + if num == 1 then + prepare = function() + end + return input0_chain, prepare; + end +end + -- 2.39.2