8 #include "cef_capture.h"
10 #include "ffmpeg_capture.h"
12 #include "image_input.h"
13 #include "input_state.h"
14 #include "lua_utils.h"
18 using namespace movit;
21 bool display(Block *block, lua_State *L, int idx);
23 EffectType current_type(const Block *block)
25 return block->alternatives[block->currently_chosen_alternative]->effect_type;
28 int find_index_of(const Block *block, EffectType val)
30 for (size_t idx = 0; idx < block->alternatives.size(); ++idx) {
31 if (block->alternatives[idx]->effect_type == val) {
38 Scene::Scene(Theme *theme, float aspect_nom, float aspect_denom)
39 : theme(theme), aspect_nom(aspect_nom), aspect_denom(aspect_denom), resource_pool(theme->get_resource_pool()) {}
41 size_t Scene::compute_chain_number(bool is_main_chain) const
43 assert(chains.size() > 0);
44 assert(chains.size() % 2 == 0);
45 size_t chain_number = compute_chain_number_for_block(blocks.size() - 1);
46 assert(chain_number < chains.size() / 2);
48 chain_number += chains.size() / 2;
53 size_t Scene::compute_chain_number_for_block(size_t block_idx) const
55 Block *block = blocks[block_idx];
58 assert(block->cardinality_base == 1);
59 chain_number = block->currently_chosen_alternative;
61 chain_number = compute_chain_number_for_block(block_idx - 1) + block->cardinality_base * block->currently_chosen_alternative;
63 assert(block->currently_chosen_alternative < int(block->alternatives.size()));
67 int Scene::add_input(lua_State* L)
69 assert(lua_gettop(L) == 1 || lua_gettop(L) == 2);
70 Scene *scene = (Scene *)luaL_checkudata(L, 1, "Scene");
72 Block *block = new Block;
73 block->idx = scene->blocks.size();
74 if (lua_gettop(L) == 1) {
75 // No parameter given, so a flexible input.
76 block->alternatives.emplace_back(new EffectBlueprint(LIVE_INPUT_YCBCR));
77 block->alternatives.emplace_back(new EffectBlueprint(LIVE_INPUT_YCBCR_WITH_DEINTERLACE));
78 block->alternatives.emplace_back(new EffectBlueprint(LIVE_INPUT_YCBCR_PLANAR));
79 block->alternatives.emplace_back(new EffectBlueprint(LIVE_INPUT_BGRA));
80 block->alternatives.emplace_back(new EffectBlueprint(IMAGE_INPUT));
82 // Input of a given type. We'll specialize it here, plus connect the input as given.
83 if (lua_isnumber(L, 2)) {
84 block->alternatives.emplace_back(new EffectBlueprint(LIVE_INPUT_YCBCR));
85 block->alternatives.emplace_back(new EffectBlueprint(LIVE_INPUT_YCBCR_WITH_DEINTERLACE));
87 } else if (luaL_testudata(L, 2, "HTMLInput")) {
88 block->alternatives.emplace_back(new EffectBlueprint(LIVE_INPUT_BGRA));
90 } else if (luaL_testudata(L, 2, "VideoInput")) {
91 FFmpegCapture *capture = *(FFmpegCapture **)luaL_checkudata(L, 2, "VideoInput");
92 if (capture->get_current_pixel_format() == bmusb::PixelFormat_8BitYCbCrPlanar) {
93 block->alternatives.emplace_back(new EffectBlueprint(LIVE_INPUT_YCBCR_PLANAR));
95 assert(capture->get_current_pixel_format() == bmusb::PixelFormat_8BitBGRA);
96 block->alternatives.emplace_back(new EffectBlueprint(LIVE_INPUT_BGRA));
98 } else if (luaL_testudata(L, 2, "ImageInput")) {
99 block->alternatives.emplace_back(new EffectBlueprint(IMAGE_INPUT));
101 luaL_error(L, "add_input() called with something that's not a signal (a signal number, a HTML input, or a VideoInput)");
103 bool ok = display(block, L, 2);
106 block->is_input = true;
107 scene->blocks.push_back(block);
109 return wrap_lua_existing_object_nonowned<Block>(L, "Block", block);
112 void Scene::find_inputs_for_block(lua_State *L, Scene *scene, Block *block)
114 if (lua_gettop(L) == 2) {
115 // Implicitly the last added effect.
116 assert(!scene->blocks.empty());
117 block->inputs.push_back(scene->blocks.size() - 1);
121 for (int idx = 3; idx <= lua_gettop(L); ++idx) {
122 Block *input_block = nullptr;
123 if (luaL_testudata(L, idx, "Block")) {
124 input_block = *(Block **)luaL_checkudata(L, idx, "Block");
126 EffectBlueprint *blueprint = *(EffectBlueprint **)luaL_checkudata(L, idx, "EffectBlueprint");
128 // Search through all the blocks to figure out which one contains this effect.
129 for (Block *block : scene->blocks) {
130 if (find(block->alternatives.begin(), block->alternatives.end(), blueprint) != block->alternatives.end()) {
135 if (input_block == nullptr) {
136 luaL_error(L, "Input effect in parameter #%d has not been added to this scene", idx - 1);
139 block->inputs.push_back(input_block->idx);
143 int Scene::add_effect(lua_State* L)
145 assert(lua_gettop(L) >= 2);
146 Scene *scene = (Scene *)luaL_checkudata(L, 1, "Scene");
148 Block *block = new Block;
149 block->idx = scene->blocks.size();
151 if (lua_istable(L, 2)) {
152 size_t len = lua_objlen(L, 2);
153 for (size_t i = 0; i < len; ++i) {
154 lua_rawgeti(L, 2, i + 1);
155 EffectBlueprint *blueprint = *(EffectBlueprint **)luaL_checkudata(L, -1, "EffectBlueprint");
156 block->alternatives.push_back(blueprint);
160 EffectBlueprint *blueprint = *(EffectBlueprint **)luaL_checkudata(L, 2, "EffectBlueprint");
161 block->alternatives.push_back(blueprint);
164 find_inputs_for_block(L, scene, block);
165 scene->blocks.push_back(block);
167 return wrap_lua_existing_object_nonowned<Block>(L, "Block", block);
170 int Scene::add_optional_effect(lua_State* L)
172 assert(lua_gettop(L) >= 2);
173 Scene *scene = (Scene *)luaL_checkudata(L, 1, "Scene");
175 // NOTE: We only support effects with a single parent, since that's what IdentityEffect does.
176 Block *block = new Block;
177 block->idx = scene->blocks.size();
179 EffectBlueprint *blueprint = *(EffectBlueprint **)luaL_checkudata(L, 2, "EffectBlueprint");
180 block->alternatives.push_back(blueprint);
182 // An IdentityEffect will be the alternative for when the effect is disabled.
183 block->alternatives.push_back(new EffectBlueprint(IDENTITY_EFFECT));
185 block->inputs.push_back(scene->blocks.size() - 1);
186 scene->blocks.push_back(block);
188 return wrap_lua_existing_object_nonowned<Block>(L, "Block", block);
191 Effect *Scene::instantiate_effects(const Block *block, size_t chain_idx, Scene::Instantiation *instantiation)
193 vector<Effect *> inputs;
194 for (size_t input_idx : block->inputs) {
195 inputs.push_back(instantiate_effects(blocks[input_idx], chain_idx, instantiation));
198 // Find the chosen alternative for this block in this instance.
199 EffectType chosen_type = block->alternatives[block->chosen_alternative(chain_idx)]->effect_type;
202 switch (chosen_type) {
203 case LIVE_INPUT_YCBCR:
204 case LIVE_INPUT_YCBCR_WITH_DEINTERLACE:
205 case LIVE_INPUT_YCBCR_PLANAR:
206 case LIVE_INPUT_BGRA: {
207 bool deinterlace = (chosen_type == LIVE_INPUT_YCBCR_WITH_DEINTERLACE);
208 bool override_bounce = !deinterlace; // For most chains, this will be fine. Reconsider if we see real problems somewhere; it's better than having the user try to understand it.
209 bmusb::PixelFormat pixel_format;
210 if (chosen_type == LIVE_INPUT_BGRA) {
211 pixel_format = bmusb::PixelFormat_8BitBGRA;
212 } else if (chosen_type == LIVE_INPUT_YCBCR_PLANAR) {
213 pixel_format = bmusb::PixelFormat_8BitYCbCrPlanar;
214 } else if (global_flags.ten_bit_input) {
215 pixel_format = bmusb::PixelFormat_10BitYCbCr;
217 pixel_format = bmusb::PixelFormat_8BitYCbCr;
219 LiveInputWrapper *input = new LiveInputWrapper(theme, instantiation->chain.get(), pixel_format, override_bounce, deinterlace, /*user_connectable=*/true);
220 effect = input->get_effect(); // Adds itself to the chain, so no need to call add_effect().
221 instantiation->inputs.emplace(block->idx, input);
225 ImageInput *input = new ImageInput;
226 instantiation->chain->add_input(input);
227 instantiation->image_inputs.emplace(block->idx, input);
232 effect = instantiate_effect(instantiation->chain.get(), chosen_type);
233 instantiation->chain->add_effect(effect, inputs);
236 instantiation->effects.emplace(block->idx, effect);
240 int Scene::finalize(lua_State* L)
242 bool only_one_mode = false;
243 bool chosen_mode = false;
244 if (lua_gettop(L) == 2) {
245 only_one_mode = true;
246 chosen_mode = checkbool(L, 2);
248 assert(lua_gettop(L) == 1);
250 Scene *scene = (Scene *)luaL_checkudata(L, 1, "Scene");
251 Theme *theme = get_theme_updata(L);
254 for (Block *block : scene->blocks) {
255 block->cardinality_base = base;
256 base *= block->alternatives.size();
259 const size_t cardinality = base;
260 const size_t total_cardinality = cardinality * (only_one_mode ? 1 : 2);
261 if (total_cardinality > 200) {
262 print_warning(L, "The given Scene will instantiate %zu different versions. This will take a lot of time and RAM to compile; see if you could limit some options by e.g. locking the input type in some cases (by giving a fixed input to add_input()).\n",
266 Block *output_block = scene->blocks.back();
267 for (bool is_main_chain : { false, true }) {
268 for (size_t chain_idx = 0; chain_idx < cardinality; ++chain_idx) {
269 if (only_one_mode && is_main_chain != chosen_mode) {
270 scene->chains.emplace_back();
274 Scene::Instantiation instantiation;
275 instantiation.chain.reset(new EffectChain(scene->aspect_nom, scene->aspect_denom, theme->get_resource_pool()));
276 scene->instantiate_effects(output_block, chain_idx, &instantiation);
278 add_outputs_and_finalize(instantiation.chain.get(), is_main_chain);
279 scene->chains.emplace_back(move(instantiation));
285 std::pair<movit::EffectChain *, std::function<void()>>
286 Scene::get_chain(Theme *theme, lua_State *L, unsigned num, const InputState &input_state)
288 // For video inputs, pick the right interlaced/progressive version
289 // based on the current state of the signals.
290 InputStateInfo info(input_state);
291 for (Block *block : blocks) {
292 if (block->is_input && block->signal_type_to_connect == Block::CONNECT_SIGNAL) {
293 EffectType chosen_type = current_type(block);
294 assert(chosen_type == LIVE_INPUT_YCBCR || chosen_type == LIVE_INPUT_YCBCR_WITH_DEINTERLACE);
295 if (info.last_interlaced[block->signal_to_connect]) {
296 block->currently_chosen_alternative = find_index_of(block, LIVE_INPUT_YCBCR_WITH_DEINTERLACE);
298 block->currently_chosen_alternative = find_index_of(block, LIVE_INPUT_YCBCR);
303 // Pick out the right chain based on the current selections,
304 // and snapshot all the set variables so that we can set them
305 // in the prepare function even if they're being changed by
306 // the Lua code later.
307 bool is_main_chain = (num == 0);
308 size_t chain_idx = compute_chain_number(is_main_chain);
309 const Scene::Instantiation &instantiation = chains[chain_idx];
310 EffectChain *effect_chain = instantiation.chain.get();
312 map<LiveInputWrapper *, int> signals_to_connect;
313 map<ImageInput *, string> images_to_select;
314 map<pair<Effect *, string>, int> int_to_set;
315 map<pair<Effect *, string>, float> float_to_set;
316 map<pair<Effect *, string>, array<float, 3>> vec3_to_set;
317 map<pair<Effect *, string>, array<float, 4>> vec4_to_set;
318 for (const auto &index_and_input : instantiation.inputs) {
319 Block *block = blocks[index_and_input.first];
320 EffectType chosen_type = current_type(block);
321 LiveInputWrapper *input = index_and_input.second;
322 if (chosen_type == LIVE_INPUT_YCBCR ||
323 chosen_type == LIVE_INPUT_YCBCR_WITH_DEINTERLACE ||
324 chosen_type == LIVE_INPUT_YCBCR_PLANAR ||
325 chosen_type == LIVE_INPUT_BGRA) {
326 if (block->signal_type_to_connect == Block::CONNECT_SIGNAL) {
327 signals_to_connect.emplace(input, block->signal_to_connect);
329 } else if (block->signal_type_to_connect == Block::CONNECT_CEF) {
330 signals_to_connect.emplace(input, block->cef_to_connect->get_card_index());
332 } else if (block->signal_type_to_connect == Block::CONNECT_VIDEO) {
333 signals_to_connect.emplace(input, block->video_to_connect->get_card_index());
334 } else if (block->signal_type_to_connect == Block::CONNECT_NONE) {
335 luaL_error(L, "An input in a scene was not connected to anything (forgot to call display())");
341 for (const auto &index_and_input : instantiation.image_inputs) {
342 Block *block = blocks[index_and_input.first];
343 ImageInput *input = index_and_input.second;
344 if (current_type(block) == IMAGE_INPUT) {
345 images_to_select.emplace(input, block->pathname);
348 for (const auto &index_and_effect : instantiation.effects) {
349 Block *block = blocks[index_and_effect.first];
350 Effect *effect = index_and_effect.second;
352 // Get the effects currently set on the block.
353 if (current_type(block) != IDENTITY_EFFECT) { // Ignore settings on optional effects.
354 for (const auto &key_and_tuple : block->int_parameters) {
355 int_to_set.emplace(make_pair(effect, key_and_tuple.first), key_and_tuple.second);
357 for (const auto &key_and_tuple : block->float_parameters) {
358 float_to_set.emplace(make_pair(effect, key_and_tuple.first), key_and_tuple.second);
360 for (const auto &key_and_tuple : block->vec3_parameters) {
361 vec3_to_set.emplace(make_pair(effect, key_and_tuple.first), key_and_tuple.second);
363 for (const auto &key_and_tuple : block->vec4_parameters) {
364 vec4_to_set.emplace(make_pair(effect, key_and_tuple.first), key_and_tuple.second);
368 // Parameters set on the blueprint itself override those that are set for the block,
369 // so they are set afterwards.
370 if (!block->alternatives.empty()) {
371 EffectBlueprint *blueprint = block->alternatives[block->currently_chosen_alternative];
372 for (const auto &key_and_tuple : blueprint->int_parameters) {
373 int_to_set[make_pair(effect, key_and_tuple.first)] = key_and_tuple.second;
375 for (const auto &key_and_tuple : blueprint->float_parameters) {
376 float_to_set[make_pair(effect, key_and_tuple.first)] = key_and_tuple.second;
378 for (const auto &key_and_tuple : blueprint->vec3_parameters) {
379 vec3_to_set[make_pair(effect, key_and_tuple.first)] = key_and_tuple.second;
381 for (const auto &key_and_tuple : blueprint->vec4_parameters) {
382 vec4_to_set[make_pair(effect, key_and_tuple.first)] = key_and_tuple.second;
389 auto setup_chain = [L, theme, signals_to_connect, images_to_select, int_to_set, float_to_set, vec3_to_set, vec4_to_set, input_state]{
390 lock_guard<mutex> lock(theme->m);
392 // Set up state, including connecting signals.
393 for (const auto &input_and_signal : signals_to_connect) {
394 LiveInputWrapper *input = input_and_signal.first;
395 input->connect_signal_raw(input_and_signal.second, input_state);
397 for (const auto &input_and_filename : images_to_select) {
398 input_and_filename.first->switch_image(input_and_filename.second);
400 for (const auto &effect_and_key_and_value : int_to_set) {
401 Effect *effect = effect_and_key_and_value.first.first;
402 const string &key = effect_and_key_and_value.first.second;
403 const int value = effect_and_key_and_value.second;
404 if (!effect->set_int(key, value)) {
405 luaL_error(L, "Effect refused set_int(\"%s\", %d) (invalid key?)", key.c_str(), value);
408 for (const auto &effect_and_key_and_value : float_to_set) {
409 Effect *effect = effect_and_key_and_value.first.first;
410 const string &key = effect_and_key_and_value.first.second;
411 const float value = effect_and_key_and_value.second;
412 if (!effect->set_float(key, value)) {
413 luaL_error(L, "Effect refused set_float(\"%s\", %f) (invalid key?)", key.c_str(), value);
416 for (const auto &effect_and_key_and_value : vec3_to_set) {
417 Effect *effect = effect_and_key_and_value.first.first;
418 const string &key = effect_and_key_and_value.first.second;
419 const float *value = effect_and_key_and_value.second.data();
420 if (!effect->set_vec3(key, value)) {
421 luaL_error(L, "Effect refused set_vec3(\"%s\", %f, %f, %f) (invalid key?)", key.c_str(),
422 value[0], value[1], value[2]);
425 for (const auto &effect_and_key_and_value : vec4_to_set) {
426 Effect *effect = effect_and_key_and_value.first.first;
427 const string &key = effect_and_key_and_value.first.second;
428 const float *value = effect_and_key_and_value.second.data();
429 if (!effect->set_vec4(key, value)) {
430 luaL_error(L, "Effect refused set_vec4(\"%s\", %f, %f, %f, %f) (invalid key?)", key.c_str(),
431 value[0], value[1], value[2], value[3]);
435 return make_pair(effect_chain, move(setup_chain));
438 bool display(Block *block, lua_State *L, int idx)
440 if (lua_isnumber(L, idx)) {
441 Theme *theme = get_theme_updata(L);
442 int signal_idx = luaL_checknumber(L, idx);
443 block->signal_type_to_connect = Block::CONNECT_SIGNAL;
444 block->signal_to_connect = theme->map_signal(signal_idx);
445 block->currently_chosen_alternative = find_index_of(block, LIVE_INPUT_YCBCR); // Will be changed to deinterlaced at get_chain() time if needed.
448 } else if (luaL_testudata(L, idx, "HTMLInput")) {
449 CEFCapture *capture = *(CEFCapture **)luaL_checkudata(L, idx, "HTMLInput");
450 block->signal_type_to_connect = Block::CONNECT_CEF;
451 block->cef_to_connect = capture;
452 block->currently_chosen_alternative = find_index_of(block, LIVE_INPUT_BGRA);
453 assert(capture->get_current_pixel_format() == bmusb::PixelFormat_8BitBGRA);
456 } else if (luaL_testudata(L, idx, "VideoInput")) {
457 FFmpegCapture *capture = *(FFmpegCapture **)luaL_checkudata(L, idx, "VideoInput");
458 block->signal_type_to_connect = Block::CONNECT_VIDEO;
459 block->video_to_connect = capture;
460 if (capture->get_current_pixel_format() == bmusb::PixelFormat_8BitYCbCrPlanar) {
461 block->currently_chosen_alternative = find_index_of(block, LIVE_INPUT_YCBCR_PLANAR);
463 assert(capture->get_current_pixel_format() == bmusb::PixelFormat_8BitBGRA);
464 block->currently_chosen_alternative = find_index_of(block, LIVE_INPUT_BGRA);
467 } else if (luaL_testudata(L, idx, "ImageInput")) {
468 ImageInput *image = *(ImageInput **)luaL_checkudata(L, idx, "ImageInput");
469 block->signal_type_to_connect = Block::CONNECT_NONE;
470 block->currently_chosen_alternative = find_index_of(block, IMAGE_INPUT);
471 block->pathname = image->get_pathname();
478 int Block_display(lua_State* L)
480 assert(lua_gettop(L) == 2);
481 Block *block = *(Block **)luaL_checkudata(L, 1, "Block");
482 if (!block->is_input) {
483 luaL_error(L, "display() called on something that isn't an input");
486 bool ok = display(block, L, 2);
488 luaL_error(L, "display() called with something that's not a signal (a signal number, a HTML input, or a VideoInput)");
491 if (block->currently_chosen_alternative == -1) {
492 luaL_error(L, "display() called on an input whose type was fixed at construction time, with a signal of different type");
498 int Block_choose(lua_State* L)
500 assert(lua_gettop(L) == 2);
501 Block *block = *(Block **)luaL_checkudata(L, 1, "Block");
502 int alternative_idx = -1;
503 if (lua_isnumber(L, 2)) {
504 alternative_idx = luaL_checknumber(L, 2);
505 } else if (lua_istable(L, 2)) {
506 // See if it's an Effect metatable (e.g. foo:choose(ResampleEffect))
507 lua_getfield(L, 2, "__effect_type_id");
508 if (lua_isnumber(L, -1)) {
509 EffectType effect_type = EffectType(luaL_checknumber(L, -1));
510 alternative_idx = find_index_of(block, effect_type);
515 if (alternative_idx == -1) {
516 luaL_error(L, "choose() called with something that was not an index or an effect type (e.g. ResampleEffect) that was part of the alternatives");
519 assert(alternative_idx >= 0);
520 assert(size_t(alternative_idx) < block->alternatives.size());
521 block->currently_chosen_alternative = alternative_idx;
523 return wrap_lua_existing_object_nonowned<EffectBlueprint>(L, "EffectBlueprint", block->alternatives[alternative_idx]);
526 int Block_enable(lua_State *L)
528 assert(lua_gettop(L) == 1);
529 Block *block = *(Block **)luaL_checkudata(L, 1, "Block");
531 if (block->alternatives.size() != 2 ||
532 block->alternatives[1]->effect_type != IDENTITY_EFFECT) {
533 luaL_error(L, "enable() called on something that wasn't added with add_optional_effect()");
535 block->currently_chosen_alternative = 0; // The actual effect.
539 int Block_enable_if(lua_State *L)
541 assert(lua_gettop(L) == 2);
542 Block *block = *(Block **)luaL_checkudata(L, 1, "Block");
544 if (block->alternatives.size() != 2 ||
545 block->alternatives[1]->effect_type != IDENTITY_EFFECT) {
546 luaL_error(L, "enable_if() called on something that wasn't added with add_optional_effect()");
548 bool enabled = checkbool(L, 2);
549 block->currently_chosen_alternative = enabled ? 0 : 1;
553 int Block_disable(lua_State *L)
555 assert(lua_gettop(L) == 1);
556 Block *block = *(Block **)luaL_checkudata(L, 1, "Block");
558 block->currently_chosen_alternative = find_index_of(block, IDENTITY_EFFECT);
559 if (block->currently_chosen_alternative == -1) {
560 luaL_error(L, "disable() called on something that didn't have an IdentityEffect fallback (try add_optional_effect())");
562 assert(block->currently_chosen_alternative != -1);
566 int Block_set_int(lua_State *L)
568 assert(lua_gettop(L) == 3);
569 Block *block = *(Block **)luaL_checkudata(L, 1, "Block");
570 string key = checkstdstring(L, 2);
571 float value = luaL_checknumber(L, 3);
573 // TODO: check validity already here, if possible?
574 block->int_parameters[key] = value;
579 int Block_set_float(lua_State *L)
581 assert(lua_gettop(L) == 3);
582 Block *block = *(Block **)luaL_checkudata(L, 1, "Block");
583 string key = checkstdstring(L, 2);
584 float value = luaL_checknumber(L, 3);
586 // TODO: check validity already here, if possible?
587 block->float_parameters[key] = value;
592 int Block_set_vec3(lua_State *L)
594 assert(lua_gettop(L) == 5);
595 Block *block = *(Block **)luaL_checkudata(L, 1, "Block");
596 string key = checkstdstring(L, 2);
598 v[0] = luaL_checknumber(L, 3);
599 v[1] = luaL_checknumber(L, 4);
600 v[2] = luaL_checknumber(L, 5);
602 // TODO: check validity already here, if possible?
603 block->vec3_parameters[key] = v;
608 int Block_set_vec4(lua_State *L)
610 assert(lua_gettop(L) == 6);
611 Block *block = *(Block **)luaL_checkudata(L, 1, "Block");
612 string key = checkstdstring(L, 2);
614 v[0] = luaL_checknumber(L, 3);
615 v[1] = luaL_checknumber(L, 4);
616 v[2] = luaL_checknumber(L, 5);
617 v[3] = luaL_checknumber(L, 6);
619 // TODO: check validity already here, if possible?
620 block->vec4_parameters[key] = v;