8 #include "cef_capture.h"
10 #include "ffmpeg_capture.h"
12 #include "image_input.h"
13 #include "input_state.h"
14 #include "lua_utils.h"
18 using namespace movit;
21 bool display(Block *block, lua_State *L, int idx);
23 EffectType current_type(const Block *block)
25 return block->alternatives[block->currently_chosen_alternative]->effect_type;
28 int find_index_of(const Block *block, EffectType val)
30 for (size_t idx = 0; idx < block->alternatives.size(); ++idx) {
31 if (block->alternatives[idx]->effect_type == val) {
38 Scene::Scene(Theme *theme, float aspect_nom, float aspect_denom)
39 : theme(theme), aspect_nom(aspect_nom), aspect_denom(aspect_denom), resource_pool(theme->get_resource_pool()) {}
41 size_t Scene::compute_chain_number(bool is_main_chain) const
43 assert(chains.size() > 0);
44 assert(chains.size() % 2 == 0);
45 size_t chain_number = compute_chain_number_for_block(blocks.size() - 1);
46 assert(chain_number < chains.size() / 2);
48 chain_number += chains.size() / 2;
53 size_t Scene::compute_chain_number_for_block(size_t block_idx) const
55 Block *block = blocks[block_idx];
58 assert(block->cardinality_base == 1);
59 chain_number = block->currently_chosen_alternative;
61 chain_number = compute_chain_number_for_block(block_idx - 1) + block->cardinality_base * block->currently_chosen_alternative;
63 assert(block->currently_chosen_alternative < int(block->alternatives.size()));
67 int Scene::add_input(lua_State* L)
69 assert(lua_gettop(L) == 1 || lua_gettop(L) == 2);
70 Scene *scene = (Scene *)luaL_checkudata(L, 1, "Scene");
72 Block *block = new Block;
73 block->idx = scene->blocks.size();
74 if (lua_gettop(L) == 1) {
75 // No parameter given, so a flexible input.
76 block->alternatives.emplace_back(new EffectBlueprint(LIVE_INPUT_YCBCR));
77 block->alternatives.emplace_back(new EffectBlueprint(LIVE_INPUT_YCBCR_WITH_DEINTERLACE));
78 block->alternatives.emplace_back(new EffectBlueprint(LIVE_INPUT_YCBCR_PLANAR));
79 block->alternatives.emplace_back(new EffectBlueprint(LIVE_INPUT_BGRA));
80 block->alternatives.emplace_back(new EffectBlueprint(IMAGE_INPUT));
82 // Input of a given type. We'll specialize it here, plus connect the input as given.
83 if (lua_isnumber(L, 2)) {
84 block->alternatives.emplace_back(new EffectBlueprint(LIVE_INPUT_YCBCR));
85 block->alternatives.emplace_back(new EffectBlueprint(LIVE_INPUT_YCBCR_WITH_DEINTERLACE));
87 } else if (luaL_testudata(L, 2, "HTMLInput")) {
88 block->alternatives.emplace_back(new EffectBlueprint(LIVE_INPUT_BGRA));
90 } else if (luaL_testudata(L, 2, "VideoInput")) {
91 FFmpegCapture *capture = *(FFmpegCapture **)luaL_checkudata(L, 2, "VideoInput");
92 if (capture->get_current_pixel_format() == bmusb::PixelFormat_8BitYCbCrPlanar) {
93 block->alternatives.emplace_back(new EffectBlueprint(LIVE_INPUT_YCBCR_PLANAR));
95 assert(capture->get_current_pixel_format() == bmusb::PixelFormat_8BitBGRA);
96 block->alternatives.emplace_back(new EffectBlueprint(LIVE_INPUT_BGRA));
98 } else if (luaL_testudata(L, 2, "ImageInput")) {
99 block->alternatives.emplace_back(new EffectBlueprint(IMAGE_INPUT));
101 luaL_error(L, "add_input() called with something that's not a signal (a signal number, a HTML input, or a VideoInput)");
103 bool ok = display(block, L, 2);
106 block->is_input = true;
107 scene->blocks.push_back(block);
109 return wrap_lua_existing_object_nonowned<Block>(L, "Block", block);
112 int Scene::add_effect(lua_State* L)
114 assert(lua_gettop(L) >= 2);
115 Scene *scene = (Scene *)luaL_checkudata(L, 1, "Scene");
117 Block *block = new Block;
118 block->idx = scene->blocks.size();
120 if (lua_istable(L, 2)) {
121 size_t len = lua_objlen(L, 2);
122 for (size_t i = 0; i < len; ++i) {
123 lua_rawgeti(L, 2, i + 1);
124 EffectBlueprint *blueprint = *(EffectBlueprint **)luaL_checkudata(L, -1, "EffectBlueprint");
125 block->alternatives.push_back(blueprint);
129 EffectBlueprint *blueprint = *(EffectBlueprint **)luaL_checkudata(L, 2, "EffectBlueprint");
130 block->alternatives.push_back(blueprint);
134 if (lua_gettop(L) == 2) {
135 assert(!scene->blocks.empty());
136 block->inputs.push_back(scene->blocks.size() - 1);
138 for (int idx = 3; idx <= lua_gettop(L); ++idx) {
139 Block *input_block = nullptr;
140 if (luaL_testudata(L, idx, "Block")) {
141 input_block = *(Block **)luaL_checkudata(L, idx, "Block");
143 EffectBlueprint *blueprint = *(EffectBlueprint **)luaL_checkudata(L, idx, "EffectBlueprint");
145 // Search through all the blocks to figure out which one contains this effect.
146 for (Block *block : scene->blocks) {
147 if (find(block->alternatives.begin(), block->alternatives.end(), blueprint) != block->alternatives.end()) {
152 if (input_block == nullptr) {
153 luaL_error(L, "Input effect in parameter #%d has not been added to this scene", idx - 1);
156 block->inputs.push_back(input_block->idx);
160 scene->blocks.push_back(block);
162 return wrap_lua_existing_object_nonowned<Block>(L, "Block", block);
165 int Scene::add_optional_effect(lua_State* L)
167 assert(lua_gettop(L) >= 2);
168 Scene *scene = (Scene *)luaL_checkudata(L, 1, "Scene");
170 // NOTE: We only support effects with a single parent, since that's what IdentityEffect does.
171 Block *block = new Block;
172 block->idx = scene->blocks.size();
174 EffectBlueprint *blueprint = *(EffectBlueprint **)luaL_checkudata(L, 2, "EffectBlueprint");
175 block->alternatives.push_back(blueprint);
177 // An IdentityEffect will be the alternative for when the effect is disabled.
178 block->alternatives.push_back(new EffectBlueprint(IDENTITY_EFFECT));
180 block->inputs.push_back(scene->blocks.size() - 1);
181 scene->blocks.push_back(block);
183 return wrap_lua_existing_object_nonowned<Block>(L, "Block", block);
186 Effect *Scene::instantiate_effects(const Block *block, size_t chain_idx, Scene::Instantiation *instantiation)
188 vector<Effect *> inputs;
189 for (size_t input_idx : block->inputs) {
190 inputs.push_back(instantiate_effects(blocks[input_idx], chain_idx, instantiation));
193 // Find the chosen alternative for this block in this instance.
194 size_t chosen_alternative = (chain_idx / block->cardinality_base) % block->alternatives.size();
195 EffectType chosen_type = block->alternatives[chosen_alternative]->effect_type;
198 switch (chosen_type) {
199 case LIVE_INPUT_YCBCR:
200 case LIVE_INPUT_YCBCR_WITH_DEINTERLACE:
201 case LIVE_INPUT_YCBCR_PLANAR:
202 case LIVE_INPUT_BGRA: {
203 bool deinterlace = (chosen_type == LIVE_INPUT_YCBCR_WITH_DEINTERLACE);
204 bool override_bounce = !deinterlace; // For most chains, this will be fine. Reconsider if we see real problems somewhere; it's better than having the user try to understand it.
205 bmusb::PixelFormat pixel_format;
206 if (chosen_type == LIVE_INPUT_BGRA) {
207 pixel_format = bmusb::PixelFormat_8BitBGRA;
208 } else if (chosen_type == LIVE_INPUT_YCBCR_PLANAR) {
209 pixel_format = bmusb::PixelFormat_8BitYCbCrPlanar;
210 } else if (global_flags.ten_bit_input) {
211 pixel_format = bmusb::PixelFormat_10BitYCbCr;
213 pixel_format = bmusb::PixelFormat_8BitYCbCr;
215 LiveInputWrapper *input = new LiveInputWrapper(theme, instantiation->chain.get(), pixel_format, override_bounce, deinterlace, /*user_connectable=*/true);
216 effect = input->get_effect(); // Adds itself to the chain, so no need to call add_effect().
217 instantiation->inputs.emplace(block->idx, input);
221 ImageInput *input = new ImageInput;
222 instantiation->chain->add_input(input);
223 instantiation->image_inputs.emplace(block->idx, input);
228 effect = instantiate_effect(instantiation->chain.get(), block->alternatives[chosen_alternative]->effect_type);
229 instantiation->chain->add_effect(effect, inputs);
232 instantiation->effects.emplace(block->idx, effect);
236 int Scene::finalize(lua_State* L)
238 bool only_one_mode = false;
239 bool chosen_mode = false;
240 if (lua_gettop(L) == 2) {
241 only_one_mode = true;
242 chosen_mode = checkbool(L, 2);
244 assert(lua_gettop(L) == 1);
246 Scene *scene = (Scene *)luaL_checkudata(L, 1, "Scene");
247 Theme *theme = get_theme_updata(L);
250 for (Block *block : scene->blocks) {
251 block->cardinality_base = base;
252 base *= block->alternatives.size();
255 const size_t cardinality = base;
256 const size_t total_cardinality = cardinality * (only_one_mode ? 1 : 2);
257 if (total_cardinality > 200) {
258 print_warning(L, "The given Scene will instantiate %zu different versions. This will take a lot of time and RAM to compile; see if you could limit some options by e.g. locking the input type in some cases (by giving a fixed input to add_input()).\n",
262 Block *output_block = scene->blocks.back();
263 for (bool is_main_chain : { false, true }) {
264 for (size_t chain_idx = 0; chain_idx < cardinality; ++chain_idx) {
265 if (only_one_mode && is_main_chain != chosen_mode) {
266 scene->chains.emplace_back();
270 Scene::Instantiation instantiation;
271 instantiation.chain.reset(new EffectChain(scene->aspect_nom, scene->aspect_denom, theme->get_resource_pool()));
272 scene->instantiate_effects(output_block, chain_idx, &instantiation);
274 add_outputs_and_finalize(instantiation.chain.get(), is_main_chain);
275 scene->chains.emplace_back(move(instantiation));
281 std::pair<movit::EffectChain *, std::function<void()>>
282 Scene::get_chain(Theme *theme, lua_State *L, unsigned num, const InputState &input_state)
284 // For video inputs, pick the right interlaced/progressive version
285 // based on the current state of the signals.
286 InputStateInfo info(input_state);
287 for (Block *block : blocks) {
288 if (block->is_input && block->signal_type_to_connect == Block::CONNECT_SIGNAL) {
289 EffectType chosen_type = current_type(block);
290 assert(chosen_type == LIVE_INPUT_YCBCR || chosen_type == LIVE_INPUT_YCBCR_WITH_DEINTERLACE);
291 if (info.last_interlaced[block->signal_to_connect]) {
292 block->currently_chosen_alternative = find_index_of(block, LIVE_INPUT_YCBCR_WITH_DEINTERLACE);
294 block->currently_chosen_alternative = find_index_of(block, LIVE_INPUT_YCBCR);
299 // Pick out the right chain based on the current selections,
300 // and snapshot all the set variables so that we can set them
301 // in the prepare function even if they're being changed by
302 // the Lua code later.
303 bool is_main_chain = (num == 0);
304 size_t chain_idx = compute_chain_number(is_main_chain);
305 const Scene::Instantiation &instantiation = chains[chain_idx];
306 EffectChain *effect_chain = instantiation.chain.get();
308 map<LiveInputWrapper *, int> signals_to_connect;
309 map<ImageInput *, string> images_to_select;
310 map<pair<Effect *, string>, int> int_to_set;
311 map<pair<Effect *, string>, float> float_to_set;
312 map<pair<Effect *, string>, array<float, 3>> vec3_to_set;
313 map<pair<Effect *, string>, array<float, 4>> vec4_to_set;
314 for (const auto &index_and_input : instantiation.inputs) {
315 Block *block = blocks[index_and_input.first];
316 EffectType chosen_type = current_type(block);
317 LiveInputWrapper *input = index_and_input.second;
318 if (chosen_type == LIVE_INPUT_YCBCR ||
319 chosen_type == LIVE_INPUT_YCBCR_WITH_DEINTERLACE ||
320 chosen_type == LIVE_INPUT_YCBCR_PLANAR ||
321 chosen_type == LIVE_INPUT_BGRA) {
322 if (block->signal_type_to_connect == Block::CONNECT_SIGNAL) {
323 signals_to_connect.emplace(input, block->signal_to_connect);
325 } else if (block->signal_type_to_connect == Block::CONNECT_CEF) {
326 signals_to_connect.emplace(input, block->cef_to_connect->get_card_index());
328 } else if (block->signal_type_to_connect == Block::CONNECT_VIDEO) {
329 signals_to_connect.emplace(input, block->video_to_connect->get_card_index());
330 } else if (block->signal_type_to_connect == Block::CONNECT_NONE) {
331 luaL_error(L, "An input in a scene was not connected to anything (forgot to call display())");
337 for (const auto &index_and_input : instantiation.image_inputs) {
338 Block *block = blocks[index_and_input.first];
339 ImageInput *input = index_and_input.second;
340 if (current_type(block) == IMAGE_INPUT) {
341 images_to_select.emplace(input, block->pathname);
344 for (const auto &index_and_effect : instantiation.effects) {
345 Block *block = blocks[index_and_effect.first];
346 Effect *effect = index_and_effect.second;
348 // Get the effects currently set on the block.
349 if (current_type(block) != IDENTITY_EFFECT) { // Ignore settings on optional effects.
350 for (const auto &key_and_tuple : block->int_parameters) {
351 int_to_set.emplace(make_pair(effect, key_and_tuple.first), key_and_tuple.second);
353 for (const auto &key_and_tuple : block->float_parameters) {
354 float_to_set.emplace(make_pair(effect, key_and_tuple.first), key_and_tuple.second);
356 for (const auto &key_and_tuple : block->vec3_parameters) {
357 vec3_to_set.emplace(make_pair(effect, key_and_tuple.first), key_and_tuple.second);
359 for (const auto &key_and_tuple : block->vec4_parameters) {
360 vec4_to_set.emplace(make_pair(effect, key_and_tuple.first), key_and_tuple.second);
364 // Parameters set on the blueprint itself override those that are set for the block,
365 // so they are set afterwards.
366 if (!block->alternatives.empty()) {
367 EffectBlueprint *blueprint = block->alternatives[block->currently_chosen_alternative];
368 for (const auto &key_and_tuple : blueprint->int_parameters) {
369 int_to_set[make_pair(effect, key_and_tuple.first)] = key_and_tuple.second;
371 for (const auto &key_and_tuple : blueprint->float_parameters) {
372 float_to_set[make_pair(effect, key_and_tuple.first)] = key_and_tuple.second;
374 for (const auto &key_and_tuple : blueprint->vec3_parameters) {
375 vec3_to_set[make_pair(effect, key_and_tuple.first)] = key_and_tuple.second;
377 for (const auto &key_and_tuple : blueprint->vec4_parameters) {
378 vec4_to_set[make_pair(effect, key_and_tuple.first)] = key_and_tuple.second;
385 auto setup_chain = [L, theme, signals_to_connect, images_to_select, int_to_set, float_to_set, vec3_to_set, vec4_to_set, input_state]{
386 lock_guard<mutex> lock(theme->m);
388 // Set up state, including connecting signals.
389 for (const auto &input_and_signal : signals_to_connect) {
390 LiveInputWrapper *input = input_and_signal.first;
391 input->connect_signal_raw(input_and_signal.second, input_state);
393 for (const auto &input_and_filename : images_to_select) {
394 input_and_filename.first->switch_image(input_and_filename.second);
396 for (const auto &effect_and_key_and_value : int_to_set) {
397 Effect *effect = effect_and_key_and_value.first.first;
398 const string &key = effect_and_key_and_value.first.second;
399 const int value = effect_and_key_and_value.second;
400 if (!effect->set_int(key, value)) {
401 luaL_error(L, "Effect refused set_int(\"%s\", %d) (invalid key?)", key.c_str(), value);
404 for (const auto &effect_and_key_and_value : float_to_set) {
405 Effect *effect = effect_and_key_and_value.first.first;
406 const string &key = effect_and_key_and_value.first.second;
407 const float value = effect_and_key_and_value.second;
408 if (!effect->set_float(key, value)) {
409 luaL_error(L, "Effect refused set_float(\"%s\", %f) (invalid key?)", key.c_str(), value);
412 for (const auto &effect_and_key_and_value : vec3_to_set) {
413 Effect *effect = effect_and_key_and_value.first.first;
414 const string &key = effect_and_key_and_value.first.second;
415 const float *value = effect_and_key_and_value.second.data();
416 if (!effect->set_vec3(key, value)) {
417 luaL_error(L, "Effect refused set_vec3(\"%s\", %f, %f, %f) (invalid key?)", key.c_str(),
418 value[0], value[1], value[2]);
421 for (const auto &effect_and_key_and_value : vec4_to_set) {
422 Effect *effect = effect_and_key_and_value.first.first;
423 const string &key = effect_and_key_and_value.first.second;
424 const float *value = effect_and_key_and_value.second.data();
425 if (!effect->set_vec4(key, value)) {
426 luaL_error(L, "Effect refused set_vec4(\"%s\", %f, %f, %f, %f) (invalid key?)", key.c_str(),
427 value[0], value[1], value[2], value[3]);
431 return make_pair(effect_chain, move(setup_chain));
434 bool display(Block *block, lua_State *L, int idx)
436 if (lua_isnumber(L, idx)) {
437 Theme *theme = get_theme_updata(L);
438 int signal_idx = luaL_checknumber(L, idx);
439 block->signal_type_to_connect = Block::CONNECT_SIGNAL;
440 block->signal_to_connect = theme->map_signal(signal_idx);
441 block->currently_chosen_alternative = find_index_of(block, LIVE_INPUT_YCBCR); // Will be changed to deinterlaced at get_chain() time if needed.
444 } else if (luaL_testudata(L, idx, "HTMLInput")) {
445 CEFCapture *capture = *(CEFCapture **)luaL_checkudata(L, idx, "HTMLInput");
446 block->signal_type_to_connect = Block::CONNECT_CEF;
447 block->cef_to_connect = capture;
448 block->currently_chosen_alternative = find_index_of(block, LIVE_INPUT_BGRA);
449 assert(capture->get_current_pixel_format() == bmusb::PixelFormat_8BitBGRA);
452 } else if (luaL_testudata(L, idx, "VideoInput")) {
453 FFmpegCapture *capture = *(FFmpegCapture **)luaL_checkudata(L, idx, "VideoInput");
454 block->signal_type_to_connect = Block::CONNECT_VIDEO;
455 block->video_to_connect = capture;
456 if (capture->get_current_pixel_format() == bmusb::PixelFormat_8BitYCbCrPlanar) {
457 block->currently_chosen_alternative = find_index_of(block, LIVE_INPUT_YCBCR_PLANAR);
459 assert(capture->get_current_pixel_format() == bmusb::PixelFormat_8BitBGRA);
460 block->currently_chosen_alternative = find_index_of(block, LIVE_INPUT_BGRA);
463 } else if (luaL_testudata(L, idx, "ImageInput")) {
464 ImageInput *image = *(ImageInput **)luaL_checkudata(L, idx, "ImageInput");
465 block->signal_type_to_connect = Block::CONNECT_NONE;
466 block->currently_chosen_alternative = find_index_of(block, IMAGE_INPUT);
467 block->pathname = image->get_pathname();
474 int Block_display(lua_State* L)
476 assert(lua_gettop(L) == 2);
477 Block *block = *(Block **)luaL_checkudata(L, 1, "Block");
478 if (!block->is_input) {
479 luaL_error(L, "display() called on something that isn't an input");
482 bool ok = display(block, L, 2);
484 luaL_error(L, "display() called with something that's not a signal (a signal number, a HTML input, or a VideoInput)");
487 if (block->currently_chosen_alternative == -1) {
488 luaL_error(L, "display() called on an input whose type was fixed at construction time, with a signal of different type");
494 int Block_choose(lua_State* L)
496 assert(lua_gettop(L) == 2);
497 Block *block = *(Block **)luaL_checkudata(L, 1, "Block");
498 int alternative_idx = -1;
499 if (lua_isnumber(L, 2)) {
500 alternative_idx = luaL_checknumber(L, 2);
501 } else if (lua_istable(L, 2)) {
502 // See if it's an Effect metatable (e.g. foo:choose(ResampleEffect))
503 lua_getfield(L, 2, "__effect_type_id");
504 if (lua_isnumber(L, -1)) {
505 EffectType effect_type = EffectType(luaL_checknumber(L, -1));
506 alternative_idx = find_index_of(block, effect_type);
511 if (alternative_idx == -1) {
512 luaL_error(L, "choose() called with something that was not an index or an effect type (e.g. ResampleEffect) that was part of the alternatives");
515 assert(alternative_idx >= 0);
516 assert(size_t(alternative_idx) < block->alternatives.size());
517 block->currently_chosen_alternative = alternative_idx;
519 return wrap_lua_existing_object_nonowned<EffectBlueprint>(L, "EffectBlueprint", block->alternatives[alternative_idx]);
522 int Block_enable(lua_State *L)
524 assert(lua_gettop(L) == 1);
525 Block *block = *(Block **)luaL_checkudata(L, 1, "Block");
527 if (block->alternatives.size() != 2 ||
528 block->alternatives[1]->effect_type != IDENTITY_EFFECT) {
529 luaL_error(L, "enable() called on something that wasn't added with add_optional_effect()");
531 block->currently_chosen_alternative = 0; // The actual effect.
535 int Block_disable(lua_State *L)
537 assert(lua_gettop(L) == 1);
538 Block *block = *(Block **)luaL_checkudata(L, 1, "Block");
540 block->currently_chosen_alternative = find_index_of(block, IDENTITY_EFFECT);
541 if (block->currently_chosen_alternative == -1) {
542 luaL_error(L, "disable() called on something that didn't have an IdentityEffect fallback (try add_optional_effect())");
544 assert(block->currently_chosen_alternative != -1);
548 int Block_set_int(lua_State *L)
550 assert(lua_gettop(L) == 3);
551 Block *block = *(Block **)luaL_checkudata(L, 1, "Block");
552 string key = checkstdstring(L, 2);
553 float value = luaL_checknumber(L, 3);
555 // TODO: check validity already here, if possible?
556 block->int_parameters[key] = value;
561 int Block_set_float(lua_State *L)
563 assert(lua_gettop(L) == 3);
564 Block *block = *(Block **)luaL_checkudata(L, 1, "Block");
565 string key = checkstdstring(L, 2);
566 float value = luaL_checknumber(L, 3);
568 // TODO: check validity already here, if possible?
569 block->float_parameters[key] = value;
574 int Block_set_vec3(lua_State *L)
576 assert(lua_gettop(L) == 5);
577 Block *block = *(Block **)luaL_checkudata(L, 1, "Block");
578 string key = checkstdstring(L, 2);
580 v[0] = luaL_checknumber(L, 3);
581 v[1] = luaL_checknumber(L, 4);
582 v[2] = luaL_checknumber(L, 5);
584 // TODO: check validity already here, if possible?
585 block->vec3_parameters[key] = v;
590 int Block_set_vec4(lua_State *L)
592 assert(lua_gettop(L) == 6);
593 Block *block = *(Block **)luaL_checkudata(L, 1, "Block");
594 string key = checkstdstring(L, 2);
596 v[0] = luaL_checknumber(L, 3);
597 v[1] = luaL_checknumber(L, 4);
598 v[2] = luaL_checknumber(L, 5);
599 v[3] = luaL_checknumber(L, 6);
601 // TODO: check validity already here, if possible?
602 block->vec4_parameters[key] = v;