From: Steinar H. Gunderson Date: Sun, 9 Feb 2020 13:43:17 +0000 (+0100) Subject: Support auto white balance (ie., not controlled by the theme). X-Git-Tag: 1.9.2~49 X-Git-Url: https://git.sesse.net/?p=nageru;a=commitdiff_plain;h=9ece26cae09110a3c6a0c74aefb2269a6dd9a7d9 Support auto white balance (ie., not controlled by the theme). This is added as a regular (optional) white balance effect, but it automatically intercepts set_wb() calls and stores it locally. (You can still set neutral_color manually, but it's probably sort of pointless.) This simplifies the themes somewhat and allows in some cases for fewer different scene instantiations (since image inputs don't get white balance), but more importantly, it will allow exporting the white balance setting in the MJPEG export. --- diff --git a/nageru/scene.cpp b/nageru/scene.cpp index 178755a..06392c0 100644 --- a/nageru/scene.cpp +++ b/nageru/scene.cpp @@ -176,6 +176,14 @@ bool Scene::is_noncanonical_chain(size_t chain_idx) const return true; } } + + // Auto white balance is always disabled for image inputs. + if (block->root_input_block != nullptr) { + const Block *input = block->root_input_block; + if (input->alternatives[input->chosen_alternative(chain_idx)]->effect_type == IMAGE_INPUT) { + return true; + } + } } } return false; @@ -227,16 +235,16 @@ int Scene::add_input(lua_State* L) return wrap_lua_existing_object_nonowned(L, "Block", block); } -void Scene::find_inputs_for_block(lua_State *L, Scene *scene, Block *block) +void Scene::find_inputs_for_block(lua_State *L, Scene *scene, Block *block, int first_input_idx) { - if (lua_gettop(L) == 2) { + if (lua_gettop(L) == first_input_idx - 1) { // Implicitly the last added effect. assert(!scene->blocks.empty()); block->inputs.push_back(scene->blocks.size() - 1); return; } - for (int idx = 3; idx <= lua_gettop(L); ++idx) { + for (int idx = first_input_idx; idx <= lua_gettop(L); ++idx) { Block *input_block = nullptr; if (luaL_testudata(L, idx, "Block")) { input_block = *(Block **)luaL_checkudata(L, idx, "Block"); @@ -318,6 +326,55 @@ int Scene::add_optional_effect(lua_State* L) return wrap_lua_existing_object_nonowned(L, "Block", block); } +const Block *Scene::find_root_input_block(lua_State *L, const Block *block) +{ + if (block->is_input) { + assert(block->inputs.size() == 0); + return block; + } + + const Block *ret = nullptr; + for (size_t input_idx : block->inputs) { + const Block *parent = find_root_input_block(L, blocks[input_idx]); + if (parent != nullptr) { + if (ret != nullptr) { + luaL_error(L, "add_auto_white_balance() was connected to more than one input"); + } + ret = parent; + } + } + return ret; +} + +int Scene::add_auto_white_balance(lua_State* L) +{ + assert(lua_gettop(L) >= 1); + Scene *scene = (Scene *)luaL_checkudata(L, 1, "Scene"); + + Block *block = new Block; + block->declaration_point = get_declaration_point(L); + block->idx = scene->blocks.size(); + + block->alternatives.push_back(new EffectBlueprint(WHITE_BALANCE_EFFECT)); + block->alternatives.push_back(new EffectBlueprint(IDENTITY_EFFECT)); + + block->canonical_alternative = 1; + + find_inputs_for_block(L, scene, block, /*first_input_idx=*/2); + + if (block->inputs.size() != 1) { + luaL_error(L, "add_auto_white_balance() needs exactly one input"); + } + block->root_input_block = scene->find_root_input_block(L, block); + if (block->root_input_block == nullptr) { + luaL_error(L, "add_auto_white_balance() was not connected to an input"); + } + + scene->blocks.push_back(block); + + return wrap_lua_existing_object_nonowned(L, "Block", block); +} + Effect *Scene::instantiate_effects(const Block *block, size_t chain_idx, Scene::Instantiation *instantiation) { // Find the chosen alternative for this block in this instance. @@ -463,6 +520,38 @@ Scene::get_chain(Theme *theme, lua_State *L, unsigned num, const InputState &inp } } + // Find all auto white balance blocks, turn on and off the effect as needed, + // and fetch the actual white balance set (it is stored in Theme). + map> white_balance; + for (size_t block_idx = 0; block_idx < blocks.size(); ++block_idx) { + Block *block = blocks[block_idx]; + const Block *input = block->root_input_block; + if (input == nullptr) { + continue; // Not an auto white balance block. + } + + EffectType chosen_type = current_type(input); + if (chosen_type == IMAGE_INPUT) { + // Image inputs never get white balance applied. + block->currently_chosen_alternative = find_index_of(block, IDENTITY_EFFECT); + continue; + } + + assert(chosen_type == LIVE_INPUT_YCBCR || + chosen_type == LIVE_INPUT_YCBCR_WITH_DEINTERLACE || + chosen_type == LIVE_INPUT_YCBCR_PLANAR || + chosen_type == LIVE_INPUT_BGRA); + int signal = find_signal_to_connect(L, input); + Theme::WhiteBalance wb = theme->get_white_balance_for_signal(signal); + if (fabs(wb.r - 1.0) < 1e-3 && fabs(wb.g - 1.0) < 1e-3 && fabs(wb.b - 1.0) < 1e-3) { + // Neutral white balance. + block->currently_chosen_alternative = find_index_of(block, IDENTITY_EFFECT); + } else { + block->currently_chosen_alternative = find_index_of(block, WHITE_BALANCE_EFFECT); + white_balance.emplace(block, array{ wb.r, wb.g, wb.b }); + } + } + // Pick out the right chain based on the current selections, // and snapshot all the set variables so that we can set them // in the prepare function even if they're being changed by @@ -536,6 +625,9 @@ Scene::get_chain(Theme *theme, lua_State *L, unsigned num, const InputState &inp for (const auto &key_and_tuple : block->float_parameters) { float_to_set.emplace(make_pair(effect, key_and_tuple.first), key_and_tuple.second); } + if (white_balance.count(block)) { + vec3_to_set.emplace(make_pair(effect, "neutral_color"), white_balance[block]); + } for (const auto &key_and_tuple : block->vec3_parameters) { vec3_to_set.emplace(make_pair(effect, key_and_tuple.first), key_and_tuple.second); } diff --git a/nageru/scene.h b/nageru/scene.h index 8db7554..8cff06a 100644 --- a/nageru/scene.h +++ b/nageru/scene.h @@ -131,6 +131,10 @@ struct Block { std::map> vec4_parameters; std::string declaration_point; // For error messages. + + // Only for AUTO_WHITE_BALANCE_EFFECT. Points to the parent block with is_input = true, + // so that we know which signal to get the white balance from. + const Block *root_input_block = nullptr; }; int Block_display(lua_State* L); @@ -162,7 +166,7 @@ private: movit::Effect *instantiate_effects(const Block *block, size_t chain_idx, Instantiation *instantiation); size_t compute_chain_number_for_block(size_t block_idx, const std::bitset<256> &disabled) const; - static void find_inputs_for_block(lua_State *L, Scene *scene, Block *block); + static void find_inputs_for_block(lua_State *L, Scene *scene, Block *block, int first_input_idx = 3); // Find out which blocks (indexed by position in the “blocks” array), // if any, are disabled in a given instantiation. A disabled block is @@ -182,6 +186,11 @@ private: // it returns true. bool is_noncanonical_chain(size_t chain_idx) const; + // For a given block, find any parents it may have that are inputs. + // If there is more than one, throws an error. If there are zero, + // returns nullptr (should probably also be an error). + const Block *find_root_input_block(lua_State *L, const Block *block); + public: Scene(Theme *theme, float aspect_nom, float aspect_denom); size_t compute_chain_number(bool is_main_chain) const; @@ -192,6 +201,7 @@ public: static int add_input(lua_State *L); static int add_effect(lua_State *L); static int add_optional_effect(lua_State *L); + static int add_auto_white_balance(lua_State *L); static int finalize(lua_State *L); }; diff --git a/nageru/theme.cpp b/nageru/theme.cpp index 78b0fde..e3cdac2 100644 --- a/nageru/theme.cpp +++ b/nageru/theme.cpp @@ -116,6 +116,7 @@ Effect *instantiate_effect(EffectChain *chain, EffectType effect_type) case IDENTITY_EFFECT: return new IdentityEffect; case WHITE_BALANCE_EFFECT: + case AUTO_WHITE_BALANCE_EFFECT: return new WhiteBalanceEffect; case RESAMPLE_EFFECT: return new ResampleEffect; @@ -860,6 +861,7 @@ const luaL_Reg Scene_funcs[] = { { "new", Scene_new }, { "__gc", Scene_gc }, { "add_input", Scene::add_input }, + { "add_auto_white_balance", Scene::add_auto_white_balance }, { "add_effect", Scene::add_effect }, { "add_optional_effect", Scene::add_optional_effect }, { "finalize", Scene::finalize }, @@ -1682,10 +1684,21 @@ bool Theme::get_supports_set_wb(unsigned channel) return ret; } -void Theme::set_wb(unsigned channel, double r, double g, double b) +void Theme::set_wb(unsigned channel, float r, float g, float b) { lock_guard lock(m); + + if (channel_signals.count(channel)) { + white_balance_for_signal[channel_signals[channel]] = WhiteBalance{ r, g, b }; + } + lua_getglobal(L, "set_wb"); + if (lua_isnil(L, -1)) { + // The function doesn't exist, to just ignore. We've stored the white balance, + // and most likely, it will be picked up by auto white balance instead. + lua_pop(L, 1); + return; + } lua_pushnumber(L, channel); lua_pushnumber(L, r); lua_pushnumber(L, g); @@ -1698,6 +1711,15 @@ void Theme::set_wb(unsigned channel, double r, double g, double b) assert(lua_gettop(L) == 0); } +Theme::WhiteBalance Theme::get_white_balance_for_signal(int signal) +{ + if (white_balance_for_signal.count(signal)) { + return white_balance_for_signal[signal]; + } else { + return WhiteBalance{ 1.0, 1.0, 1.0 }; + } +} + vector Theme::get_transition_names(float t) { lock_guard lock(m); diff --git a/nageru/theme.h b/nageru/theme.h index 37113b2..7fd1325 100644 --- a/nageru/theme.h +++ b/nageru/theme.h @@ -39,6 +39,7 @@ enum EffectType { IDENTITY_EFFECT, WHITE_BALANCE_EFFECT, + AUTO_WHITE_BALANCE_EFFECT, // Same as WHITE_BALANCE_EFFECT, but sets its value automatically. RESAMPLE_EFFECT, PADDING_EFFECT, INTEGRAL_PADDING_EFFECT, @@ -93,6 +94,9 @@ public: // for non-interlaced inputs. std::vector input_frames; }; + struct WhiteBalance { + float r, g, b; + }; Chain get_chain(unsigned num, float t, unsigned width, unsigned height, const InputState &input_state); @@ -102,9 +106,12 @@ public: std::string get_channel_name(unsigned channel); int get_channel_signal(unsigned channel); bool get_supports_set_wb(unsigned channel); - void set_wb(unsigned channel, double r, double g, double b); + void set_wb(unsigned channel, float r, float g, float b); + WhiteBalance get_white_balance_for_signal(int signal); std::string get_channel_color(unsigned channel); + std::unordered_map white_balance_for_signal; + std::vector get_transition_names(float t); void transition_clicked(int transition_num, float t); diff --git a/nageru/theme.lua b/nageru/theme.lua index 349d33b..195e89b 100644 --- a/nageru/theme.lua +++ b/nageru/theme.lua @@ -16,11 +16,6 @@ local state = { transition_src_signal = 0, transition_dst_signal = 0, - neutral_colors = { - {0.5, 0.5, 0.5}, -- Input 0. - {0.5, 0.5, 0.5} -- Input 1. - }, - live_signal_num = 0, preview_signal_num = 1 } @@ -38,9 +33,9 @@ local FADE_TRANSITION = 2 function make_sbs_input(scene) return { - input = scene:add_input(), + input = scene:add_input(0), -- Live inputs only. resample_effect = scene:add_effect({ResampleEffect.new(), ResizeEffect.new()}), - wb_effect = scene:add_effect(WhiteBalanceEffect.new()), + wb_effect = scene:add_auto_white_balance(), padding_effect = scene:add_effect(IntegralPaddingEffect.new()) } end @@ -72,7 +67,7 @@ function make_fade_input(scene) return { input = scene:add_input(), resample_effect = scene:add_optional_effect(ResampleEffect.new()), -- Activated if scaling. - wb_effect = scene:add_optional_effect(WhiteBalanceEffect.new()) -- Activated for video inputs. + wb_effect = scene:add_auto_white_balance() -- Activated for video inputs. } end @@ -100,7 +95,7 @@ local simple_scene = { scene = scene, input = scene:add_input(), resample_effect = scene:add_effect({ResampleEffect.new(), ResizeEffect.new(), IdentityEffect.new()}), - wb_effect = scene:add_effect(WhiteBalanceEffect.new()) + wb_effect = scene:add_auto_white_balance() } scene:finalize() @@ -168,15 +163,6 @@ function channel_involved_in(channel, signal_num) return false end --- API ENTRY POINT --- Gets called with a new gray point when the white balance is changing. --- The color is in linear light (not sRGB gamma). -function set_wb(channel, red, green, blue) - if is_plain_signal(channel - 2) then - state.neutral_colors[channel - 2 + 1] = { red, green, blue } - end -end - function finish_transitions(t) if state.transition_type ~= NO_TRANSITION and t >= state.transition_end then state.live_signal_num = state.transition_dst_signal @@ -296,7 +282,6 @@ function setup_fade_input(state, input, signals, signal_num, width, height) else input.input:display(signal_num) input.wb_effect:enable() - set_neutral_color(input.wb_effect, state.neutral_colors[signal_num - INPUT0_SIGNAL_NUM + 1]) if (signals:get_width(signal_num) ~= width or signals:get_height(signal_num) ~= height) then input.resample_effect:enable() @@ -330,7 +315,6 @@ function setup_simple_input(state, signals, signal_num, width, height, hq) else simple_scene.resample_effect:disable() -- No scaling. end - set_neutral_color_from_signal(state, simple_scene.wb_effect, signal_num) end -- API ENTRY POINT @@ -535,9 +519,6 @@ function pos_from_top_left(x, y, width, height, screen_width, screen_height) end function prepare_sbs_scene(state, t, transition_type, src_signal, dst_signal, screen_width, screen_height, input_resolution, hq) - set_neutral_color(sbs_scene.input0.wb_effect, state.neutral_colors[1]) - set_neutral_color(sbs_scene.input1.wb_effect, state.neutral_colors[2]) - -- First input is positioned (16,48) from top-left. -- Second input is positioned (16,48) from the bottom-right. local pos0 = pos_from_top_left(16, 48, 848, 477, screen_width, screen_height) @@ -594,16 +575,6 @@ function place_rectangle_with_affine(input, pos, aff, screen_width, screen_heigh place_rectangle(input, x0, y0, x1, y1, screen_width, screen_height, input_width, input_height, hq) end -function set_neutral_color(effect, color) - effect:set_vec3("neutral_color", color[1], color[2], color[3]) -end - -function set_neutral_color_from_signal(state, effect, signal) - if is_plain_signal(signal) then - set_neutral_color(effect, state.neutral_colors[signal - INPUT0_SIGNAL_NUM + 1]) - end -end - function calc_zoom_progress(state, t) if t < state.transition_start then return 0.0