X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=core%2Fproducer%2Fframerate%2Fframerate_producer.cpp;h=ffaf00551a6cfae5c944304c3c50d5c80dae29c3;hb=2ecf756bb0033fbdbacd130c667a019e9eb47e12;hp=bc2c9084b8c8ba353dde32a228b3618c89a319e0;hpb=2b27c527f556da8528b4602edaa295e4bbc0f908;p=casparcg diff --git a/core/producer/framerate/framerate_producer.cpp b/core/producer/framerate/framerate_producer.cpp index bc2c9084b..ffaf00551 100644 --- a/core/producer/framerate/framerate_producer.cpp +++ b/core/producer/framerate/framerate_producer.cpp @@ -30,12 +30,15 @@ #include "../../frame/frame_transform.h" #include "../../frame/pixel_format.h" #include "../../monitor/monitor.h" +#include "../../help/help_sink.h" #include +#include #include #include #include +#include namespace caspar { namespace core { @@ -46,14 +49,14 @@ draw_frame drop_and_skip(const draw_frame& source, const draw_frame&, const boos // Blends next frame with current frame when the distance is not 0. // Completely sharp when distance is 0 but blurry when in between. -draw_frame blend(const draw_frame& source, const draw_frame& destination, const boost::rational& distance) +draw_frame blend2(const draw_frame& source, const draw_frame& destination, const boost::rational& distance) { if (destination == draw_frame::empty()) return source; auto under = source; auto over = destination; - double float_distance = static_cast(distance.numerator()) / static_cast(distance.denominator()); + double float_distance = boost::rational_cast(distance); under.transform().image_transform.is_mix = true; under.transform().image_transform.opacity = 1 - float_distance; @@ -67,8 +70,8 @@ draw_frame blend(const draw_frame& source, const draw_frame& destination, const // * A distance of 0.0 gives 50% previous, 50% current and 0% next. // * A distance of 0.5 gives 25% previous, 50% current and 25% next. // * A distance of 0.75 gives 12.5% previous, 50% current and 37.5% next. -// This is blurrier than blend, but gives a more even bluriness, instead of sharp, blurry, sharp, blurry. -struct blend_all +// This is blurrier than blend2, but gives a more even bluriness, instead of sharp, blurry, sharp, blurry. +struct blend3 { draw_frame previous_frame = draw_frame::empty(); draw_frame last_source = draw_frame::empty(); @@ -90,7 +93,7 @@ struct blend_all bool has_previous = previous_frame != draw_frame::empty(); if (!has_previous) - return blend(source, destination, distance); + return blend2(source, destination, distance); auto middle = last_source; auto next_frame = destination; @@ -98,7 +101,7 @@ struct blend_all middle.transform().image_transform.is_mix = true; next_frame.transform().image_transform.is_mix = true; - double float_distance = static_cast(distance.numerator()) / static_cast(distance.denominator()); + double float_distance = boost::rational_cast(distance); previous_frame.transform().image_transform.opacity = std::max(0.0, 0.5 - float_distance * 0.5); middle.transform().image_transform.opacity = 0.5; next_frame.transform().image_transform.opacity = 1.0 - previous_frame.transform().image_transform.opacity - middle.transform().image_transform.opacity; @@ -109,99 +112,119 @@ struct blend_all } }; -struct audio_extractor : public frame_visitor +class audio_extractor : public frame_visitor { - std::function on_frame_; - + std::stack transform_stack_; + std::function on_frame_; +public: audio_extractor(std::function on_frame) : on_frame_(std::move(on_frame)) { + transform_stack_.push(audio_transform()); + } + + void push(const frame_transform& transform) override + { + transform_stack_.push(transform_stack_.top() * transform.audio_transform); + } + + void pop() override + { + transform_stack_.pop(); } - void push(const frame_transform& transform) override { } - void pop() override { } void visit(const const_frame& frame) override { - if (!frame.audio_data().empty()) + if (!frame.audio_data().empty() && !transform_stack_.top().is_still) on_frame_(frame); } }; +// Like tweened_transform but for framerates +class speed_tweener +{ + boost::rational source_ = 1LL; + boost::rational dest_ = 1LL; + int duration_ = 0; + int time_ = 0; + tweener tweener_; +public: + speed_tweener() = default; + speed_tweener( + const boost::rational& source, + const boost::rational& dest, + int duration, + const tweener& tween) + : source_(source) + , dest_(dest) + , duration_(duration) + , time_(0) + , tweener_(tween) + { + } + + const boost::rational& dest() const + { + return dest_; + } + + boost::rational fetch() const + { + if (time_ == duration_) + return dest_; + + double source = boost::rational_cast(source_); + double delta = boost::rational_cast(dest_) - source; + double result = tweener_(time_, source, delta, duration_); + + return boost::rational(static_cast(result * 1000000.0), 1000000); + } + + boost::rational fetch_and_tick() + { + time_ = std::min(time_ + 1, duration_); + return fetch(); + } +}; + class framerate_producer : public frame_producer_base { spl::shared_ptr source_; - boost::rational source_framerate_; - audio_channel_layout source_channel_layout_ = audio_channel_layout::invalid(); + std::function()> get_source_framerate_; + boost::rational source_framerate_ = -1; + audio_channel_layout source_channel_layout_ = audio_channel_layout::invalid(); + boost::rational original_destination_framerate_; + field_mode original_destination_fieldmode_; boost::rational destination_framerate_; field_mode destination_fieldmode_; std::vector destination_audio_cadence_; boost::rational speed_; + speed_tweener user_speed_; std::function& distance)> interpolator_ = drop_and_skip; - - boost::rational current_frame_number_ = 0; - draw_frame previous_frame_ = draw_frame::empty(); - draw_frame next_frame_ = draw_frame::empty(); + const boost::rational& distance)> interpolator_ = drop_and_skip; + + boost::rational current_frame_number_ = 0; + draw_frame previous_frame_ = draw_frame::empty(); + draw_frame next_frame_ = draw_frame::empty(); mutable_audio_buffer audio_samples_; - unsigned int output_repeat_ = 0; - unsigned int output_frame_ = 0; + unsigned int output_repeat_ = 0; + unsigned int output_frame_ = 0; public: framerate_producer( spl::shared_ptr source, - boost::rational source_framerate, + std::function ()> get_source_framerate, boost::rational destination_framerate, field_mode destination_fieldmode, std::vector destination_audio_cadence) : source_(std::move(source)) - , source_framerate_(std::move(source_framerate)) - , destination_framerate_(std::move(destination_framerate)) - , destination_fieldmode_(destination_fieldmode) + , get_source_framerate_(std::move(get_source_framerate)) + , original_destination_framerate_(std::move(destination_framerate)) + , original_destination_fieldmode_(destination_fieldmode) , destination_audio_cadence_(std::move(destination_audio_cadence)) { - // Coarse adjustment to correct fps family (23.98 - 30 vs 47.95 - 60) - if (destination_fieldmode_ != field_mode::progressive) // Interlaced output - { - auto diff_double = boost::abs(source_framerate_ - destination_framerate_ * 2); - auto diff_keep = boost::abs(source_framerate_ - destination_framerate_); - - if (diff_double < diff_keep) // Double rate interlaced - { - destination_framerate_ *= 2; - } - else // Progressive non interlaced - { - destination_fieldmode_ = field_mode::progressive; - } - } - else // Progressive - { - auto diff_halve = boost::abs(source_framerate_ * 2 - destination_framerate_); - auto diff_keep = boost::abs(source_framerate_ - destination_framerate_); - - if (diff_halve < diff_keep) // Repeat every frame two times - { - destination_framerate_ /= 2; - output_repeat_ = 2; - } - } - - speed_ = boost::rational(source_framerate_ / destination_framerate_); - - // drop_and_skip will only be used by default for exact framerate multiples (half, same and double) - // for all other framerates a frame interpolator will be chosen. - if (speed_ != 1 && speed_ * 2 != 1 && speed_ != 2) - { - if (source_framerate_ > 47) // The bluriness of blend_all is acceptable on high framerates. - interpolator_ = blend_all(); - else // blend_all is mostly too blurry on low framerates. blend provides a compromise. - interpolator_ = &blend; - - CASPAR_LOG(warning) << source_->print() << L" Frame blending frame rate conversion required to conform to channel frame rate."; - } - // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601) // This cadence fills the audio mixer most optimally. boost::range::rotate(destination_audio_cadence_, std::end(destination_audio_cadence_) - 1); @@ -224,17 +247,29 @@ public: std::future call(const std::vector& params) override { - if (!boost::iequals(params.at(0), L"framerate") || params.size() != 3) + if (!boost::iequals(params.at(0), L"framerate")) return source_->call(params); - if (boost::iequals(params.at(1), L"interpolation")) + if (boost::iequals(params.at(1), L"speed")) { - if (boost::iequals(params.at(2), L"blend")) - interpolator_ = &blend; - else if (boost::iequals(params.at(2), L"blend_all")) - interpolator_ = blend_all(); - else + auto destination_user_speed = boost::rational( + static_cast(boost::lexical_cast(params.at(2)) * 1000000.0), + 1000000); + auto frames = params.size() > 3 ? boost::lexical_cast(params.at(3)) : 0; + auto easing = params.size() > 4 ? params.at(4) : L"linear"; + + user_speed_ = speed_tweener(user_speed_.fetch(), destination_user_speed, frames, tweener(easing)); + } + else if (boost::iequals(params.at(1), L"interpolation")) + { + if (boost::iequals(params.at(2), L"blend2")) + interpolator_ = &blend2; + else if (boost::iequals(params.at(2), L"blend3")) + interpolator_ = blend3(); + else if (boost::iequals(params.at(2), L"drop_and_skip")) interpolator_ = &drop_and_skip; + else + CASPAR_THROW_EXCEPTION(user_error() << msg_info("Valid interpolations are DROP_AND_SKIP, BLEND2 and BLEND3")); } else if (boost::iequals(params.at(1), L"output_repeat")) // Only for debugging purposes { @@ -261,7 +296,33 @@ public: boost::property_tree::wptree info() const override { - return source_->info(); + auto info = source_->info(); + + auto incorrect_frame_number = info.get_child_optional(L"frame-number"); + if (incorrect_frame_number) + incorrect_frame_number->put_value(frame_number()); + + auto incorrect_nb_frames = info.get_child_optional(L"nb-frames"); + if (incorrect_nb_frames) + incorrect_nb_frames->put_value(nb_frames()); + + return info; + } + + uint32_t nb_frames() const override + { + auto source_nb_frames = source_->nb_frames(); + auto multiple = boost::rational_cast(1 / get_speed() * (output_repeat_ != 0 ? 2 : 1)); + + return static_cast(source_nb_frames * multiple); + } + + uint32_t frame_number() const override + { + auto source_frame_number = source_->frame_number() - 1; // next frame already received + auto multiple = boost::rational_cast(1 / get_speed() * (output_repeat_ != 0 ? 2 : 1)); + + return static_cast(source_frame_number * multiple); } constraints& pixel_constraints() override @@ -271,7 +332,9 @@ public: private: draw_frame do_render_progressive_frame(bool sound) { - if (output_repeat_ && ++output_frame_ % output_repeat_) + user_speed_.fetch_and_tick(); + + if (output_repeat_ && output_frame_++ % output_repeat_) { auto frame = draw_frame::still(last_frame()); @@ -311,39 +374,54 @@ private: for (std::int64_t i = 0; i < num_frames; ++i) { - previous_frame_ = std::move(next_frame_); + if (next_frame_ == draw_frame::empty()) + previous_frame_ = pop_frame_from_source(); + else + { + previous_frame_ = std::move(next_frame_); - next_frame_ = pop_frame_from_source(); + next_frame_ = pop_frame_from_source(); + } } } boost::rational get_speed() const { - return speed_; + return speed_ * user_speed_.fetch(); } draw_frame pop_frame_from_source() { auto frame = source_->receive(); + update_source_framerate(); - audio_extractor extractor([this](const const_frame& frame) + if (user_speed_.fetch() == 1) { - if (source_channel_layout_ != frame.audio_channel_layout()) + audio_extractor extractor([this](const const_frame& frame) { - source_channel_layout_ = frame.audio_channel_layout(); - - // Insert silence samples so that the audio mixer is guaranteed to be filled. - auto min_num_samples_per_frame = *boost::min_element(destination_audio_cadence_); - auto max_num_samples_per_frame = *boost::max_element(destination_audio_cadence_); - auto cadence_safety_samples = max_num_samples_per_frame - min_num_samples_per_frame; - audio_samples_.resize(source_channel_layout_.num_channels * cadence_safety_samples, 0); - } - - auto& buffer = frame.audio_data(); - audio_samples_.insert(audio_samples_.end(), buffer.begin(), buffer.end()); - }); + if (source_channel_layout_ != frame.audio_channel_layout()) + { + source_channel_layout_ = frame.audio_channel_layout(); + + // Insert silence samples so that the audio mixer is guaranteed to be filled. + auto min_num_samples_per_frame = *boost::min_element(destination_audio_cadence_); + auto max_num_samples_per_frame = *boost::max_element(destination_audio_cadence_); + auto cadence_safety_samples = max_num_samples_per_frame - min_num_samples_per_frame; + audio_samples_.resize(source_channel_layout_.num_channels * cadence_safety_samples, 0); + } + + auto& buffer = frame.audio_data(); + audio_samples_.insert(audio_samples_.end(), buffer.begin(), buffer.end()); + }); + + frame.accept(extractor); + } + else + { + source_channel_layout_ = audio_channel_layout::invalid(); + audio_samples_.clear(); + } - frame.accept(extractor); frame.transform().audio_transform.volume = 0.0; return frame; @@ -351,7 +429,7 @@ private: draw_frame attach_sound(draw_frame frame) { - if (source_channel_layout_ == audio_channel_layout::invalid()) + if (user_speed_.fetch() != 1 || source_channel_layout_ == audio_channel_layout::invalid()) return frame; mutable_audio_buffer buffer; @@ -372,7 +450,8 @@ private: { auto needed = destination_audio_cadence_.front(); auto got = audio_samples_.size() / source_channel_layout_.num_channels; - CASPAR_LOG(debug) << print() << L" Too few audio samples. Needed " << needed << L" but got " << got; + if (got != 0) // If at end of stream we don't care + CASPAR_LOG(debug) << print() << L" Too few audio samples. Needed " << needed << L" but got " << got; buffer.swap(audio_samples_); buffer.resize(needed * source_channel_layout_.num_channels, 0); } @@ -391,24 +470,95 @@ private: bool enough_sound() const { return source_channel_layout_ == core::audio_channel_layout::invalid() + || user_speed_.fetch() != 1 || audio_samples_.size() / source_channel_layout_.num_channels >= destination_audio_cadence_.at(0); } + + void update_source_framerate() + { + auto source_framerate = get_source_framerate_(); + + if (source_framerate_ == source_framerate) + return; + + source_framerate_ = source_framerate; + destination_framerate_ = original_destination_framerate_; + destination_fieldmode_ = original_destination_fieldmode_; + + // Coarse adjustment to correct fps family (23.98 - 30 vs 47.95 - 60) + if (destination_fieldmode_ != field_mode::progressive) // Interlaced output + { + auto diff_double = boost::abs(source_framerate_ - destination_framerate_ * 2); + auto diff_keep = boost::abs(source_framerate_ - destination_framerate_); + + if (diff_double < diff_keep) // Double rate interlaced + { + destination_framerate_ *= 2; + } + else // Progressive non interlaced + { + destination_fieldmode_ = field_mode::progressive; + } + } + else // Progressive + { + auto diff_halve = boost::abs(source_framerate_ * 2 - destination_framerate_); + auto diff_keep = boost::abs(source_framerate_ - destination_framerate_); + + if (diff_halve < diff_keep) // Repeat every frame two times + { + destination_framerate_ /= 2; + output_repeat_ = 2; + } + } + + speed_ = boost::rational(source_framerate_ / destination_framerate_); + + // drop_and_skip will only be used by default for exact framerate multiples (half, same and double) + // for all other framerates a frame interpolator will be chosen. + if (speed_ != 1 && speed_ * 2 != 1 && speed_ != 2) + { + auto high_source_framerate = source_framerate_ > 47; + auto high_destination_framerate = destination_framerate_ > 47 + || destination_fieldmode_ != field_mode::progressive; + + if (high_source_framerate && high_destination_framerate) // The bluriness of blend3 is acceptable on high framerates. + interpolator_ = blend3(); + else // blend3 is mostly too blurry on low framerates. blend2 provides a compromise. + interpolator_ = &blend2; + + CASPAR_LOG(warning) << source_->print() << L" Frame blending frame rate conversion required to conform to channel frame rate."; + } + else + interpolator_ = &drop_and_skip; + } }; +void describe_framerate_producer(help_sink& sink) +{ + sink.para()->text(L"Framerate conversion control / Slow motion examples:"); + sink.example(L">> CALL 1-10 FRAMERATE INTERPOLATION BLEND2", L"enables 2 frame blend interpolation."); + sink.example(L">> CALL 1-10 FRAMERATE INTERPOLATION BLEND3", L"enables 3 frame blend interpolation."); + sink.example(L">> CALL 1-10 FRAMERATE INTERPOLATION DROP_AND_SKIP", L"disables frame interpolation."); + sink.example(L">> CALL 1-10 FRAMERATE SPEED 0.25", L"immediately changes the speed to 25%. Sound will be disabled."); + sink.example(L">> CALL 1-10 FRAMERATE SPEED 0.25 50", L"changes the speed to 25% linearly over 50 frames. Sound will be disabled."); + sink.example(L">> CALL 1-10 FRAMERATE SPEED 0.25 50 easeinoutsine", L"changes the speed to 25% over 50 frames using specified easing curve. Sound will be disabled."); + sink.example(L">> CALL 1-10 FRAMERATE SPEED 1 50", L"changes the speed to 100% linearly over 50 frames. Sound will be enabled when the destination speed of 100% has been reached."); +} + spl::shared_ptr create_framerate_producer( spl::shared_ptr source, - boost::rational source_framerate, + std::function ()> get_source_framerate, boost::rational destination_framerate, field_mode destination_fieldmode, std::vector destination_audio_cadence) { return spl::make_shared( std::move(source), - std::move(source_framerate), + std::move(get_source_framerate), std::move(destination_framerate), destination_fieldmode, std::move(destination_audio_cadence)); } }} -