2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Helge Norberg, helge.norberg@svt.se
22 #include "../../StdAfx.h"
24 #include "framerate_producer.h"
26 #include "../frame_producer.h"
27 #include "../../frame/audio_channel_layout.h"
28 #include "../../frame/draw_frame.h"
29 #include "../../frame/frame.h"
30 #include "../../frame/frame_transform.h"
31 #include "../../frame/pixel_format.h"
32 #include "../../monitor/monitor.h"
33 #include "../../help/help_sink.h"
35 #include <common/future.h>
36 #include <common/tweener.h>
43 namespace caspar { namespace core {
45 draw_frame drop_or_repeat(const draw_frame& source, const draw_frame&, const boost::rational<int64_t>&)
50 // Blends next frame with current frame when the distance is not 0.
51 // Completely sharp when distance is 0 but blurry when in between.
52 draw_frame blend2(const draw_frame& source, const draw_frame& destination, const boost::rational<int64_t>& distance)
54 if (destination == draw_frame::empty())
58 auto over = destination;
59 double float_distance = boost::rational_cast<double>(distance);
61 under.transform().image_transform.is_mix = true;
62 under.transform().image_transform.opacity = 1 - float_distance;
63 over.transform().image_transform.is_mix = true;
64 over.transform().image_transform.opacity = float_distance;
66 return draw_frame::over(under, over);
69 // Blends a moving window with a width of 1 frame duration.
70 // * A distance of 0.0 gives 50% previous, 50% current and 0% next.
71 // * A distance of 0.5 gives 25% previous, 50% current and 25% next.
72 // * A distance of 0.75 gives 12.5% previous, 50% current and 37.5% next.
73 // This is blurrier than blend2, but gives a more even bluriness, instead of sharp, blurry, sharp, blurry.
76 draw_frame previous_frame = draw_frame::empty();
77 draw_frame last_source = draw_frame::empty();
78 draw_frame last_destination = draw_frame::empty();
80 draw_frame operator()(const draw_frame& source, const draw_frame& destination, const boost::rational<int64_t>& distance)
82 if (last_source != draw_frame::empty() && last_source != source)
84 if (last_destination == source)
85 previous_frame = last_source;
86 else // A two frame jump
87 previous_frame = last_destination;
91 last_destination = destination;
93 bool has_previous = previous_frame != draw_frame::empty();
96 return blend2(source, destination, distance);
98 auto middle = last_source;
99 auto next_frame = destination;
100 previous_frame.transform().image_transform.is_mix = true;
101 middle.transform().image_transform.is_mix = true;
102 next_frame.transform().image_transform.is_mix = true;
104 double float_distance = boost::rational_cast<double>(distance);
105 previous_frame.transform().image_transform.opacity = std::max(0.0, 0.5 - float_distance * 0.5);
106 middle.transform().image_transform.opacity = 0.5;
107 next_frame.transform().image_transform.opacity = 1.0 - previous_frame.transform().image_transform.opacity - middle.transform().image_transform.opacity;
109 std::vector<draw_frame> combination { previous_frame, middle, next_frame };
111 return draw_frame(std::move(combination));
115 class audio_extractor : public frame_visitor
117 std::stack<core::audio_transform> transform_stack_;
118 std::function<void(const const_frame& frame)> on_frame_;
120 audio_extractor(std::function<void(const const_frame& frame)> on_frame)
121 : on_frame_(std::move(on_frame))
123 transform_stack_.push(audio_transform());
126 void push(const frame_transform& transform) override
128 transform_stack_.push(transform_stack_.top() * transform.audio_transform);
133 transform_stack_.pop();
136 void visit(const const_frame& frame) override
138 if (!frame.audio_data().empty() && !transform_stack_.top().is_still)
143 // Like tweened_transform but for framerates
146 boost::rational<int64_t> source_ = 1LL;
147 boost::rational<int64_t> dest_ = 1LL;
152 speed_tweener() = default;
154 const boost::rational<int64_t>& source,
155 const boost::rational<int64_t>& dest,
157 const tweener& tween)
160 , duration_(duration)
166 const boost::rational<int64_t>& dest() const
171 boost::rational<int64_t> fetch() const
173 if (time_ == duration_)
176 double source = boost::rational_cast<double>(source_);
177 double delta = boost::rational_cast<double>(dest_) - source;
178 double result = tweener_(time_, source, delta, duration_);
180 return boost::rational<int64_t>(static_cast<int64_t>(result * 1000000.0), 1000000);
183 boost::rational<int64_t> fetch_and_tick()
185 time_ = std::min(time_ + 1, duration_);
190 class framerate_producer : public frame_producer_base
192 spl::shared_ptr<frame_producer> source_;
193 std::function<boost::rational<int>()> get_source_framerate_;
194 boost::rational<int> source_framerate_ = -1;
195 audio_channel_layout source_channel_layout_ = audio_channel_layout::invalid();
196 const boost::rational<int> original_destination_framerate_;
197 const field_mode original_destination_fieldmode_;
198 field_mode destination_fieldmode_ = field_mode::empty;
199 std::vector<int> destination_audio_cadence_;
200 boost::rational<std::int64_t> speed_;
201 speed_tweener user_speed_;
202 std::function<draw_frame (
203 const draw_frame& source,
204 const draw_frame& destination,
205 const boost::rational<int64_t>& distance)> interpolator_ = drop_or_repeat;
207 boost::rational<std::int64_t> current_frame_number_ = 0;
208 draw_frame previous_frame_ = draw_frame::empty();
209 draw_frame next_frame_ = draw_frame::empty();
210 mutable_audio_buffer audio_samples_;
212 unsigned int output_repeat_ = 0;
213 unsigned int output_frame_ = 0;
214 draw_frame last_frame_ = draw_frame::empty();
217 spl::shared_ptr<frame_producer> source,
218 std::function<boost::rational<int> ()> get_source_framerate,
219 boost::rational<int> destination_framerate,
220 field_mode destination_fieldmode,
221 std::vector<int> destination_audio_cadence)
222 : source_(std::move(source))
223 , get_source_framerate_(std::move(get_source_framerate))
224 , original_destination_framerate_(std::move(destination_framerate))
225 , original_destination_fieldmode_(destination_fieldmode)
226 , destination_audio_cadence_(std::move(destination_audio_cadence))
228 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
229 // This cadence fills the audio mixer most optimally.
230 boost::range::rotate(destination_audio_cadence_, std::end(destination_audio_cadence_) - 1);
233 draw_frame receive_impl() override
235 // destination field mode initially unknown but known after the first update_source_framerate().
236 auto field1 = do_render_progressive_frame(true);
238 if (destination_fieldmode_ == field_mode::progressive)
244 auto field2 = do_render_progressive_frame(false);
246 return draw_frame::interlace(field1, field2, destination_fieldmode_);
250 std::future<std::wstring> call(const std::vector<std::wstring>& params) override
252 if (!boost::iequals(params.at(0), L"framerate"))
253 return source_->call(params);
255 if (boost::iequals(params.at(1), L"speed"))
257 auto destination_user_speed = boost::rational<std::int64_t>(
258 static_cast<std::int64_t>(boost::lexical_cast<double>(params.at(2)) * 1000000.0),
260 auto frames = params.size() > 3 ? boost::lexical_cast<int>(params.at(3)) : 0;
261 auto easing = params.size() > 4 ? params.at(4) : L"linear";
263 user_speed_ = speed_tweener(user_speed_.fetch(), destination_user_speed, frames, tweener(easing));
265 else if (boost::iequals(params.at(1), L"interpolation"))
267 if (boost::iequals(params.at(2), L"blend2"))
268 interpolator_ = &blend2;
269 else if (boost::iequals(params.at(2), L"blend3"))
270 interpolator_ = blend3();
271 else if (boost::iequals(params.at(2), L"drop_or_repeat"))
272 interpolator_ = &drop_or_repeat;
274 CASPAR_THROW_EXCEPTION(user_error() << msg_info("Valid interpolations are DROP_OR_REPEAT, BLEND2 and BLEND3"));
276 else if (boost::iequals(params.at(1), L"output_repeat")) // Only for debugging purposes
278 output_repeat_ = boost::lexical_cast<unsigned int>(params.at(2));
281 return make_ready_future<std::wstring>(L"");
284 monitor::subject& monitor_output() override
286 return source_->monitor_output();
289 std::wstring print() const override
291 return source_->print();
294 std::wstring name() const override
296 return source_->name();
299 boost::property_tree::wptree info() const override
301 auto info = source_->info();
303 auto incorrect_frame_number = info.get_child_optional(L"frame-number");
304 if (incorrect_frame_number)
305 incorrect_frame_number->put_value(frame_number());
307 auto incorrect_nb_frames = info.get_child_optional(L"nb-frames");
308 if (incorrect_nb_frames)
309 incorrect_nb_frames->put_value(nb_frames());
314 uint32_t nb_frames() const override
316 if (!is_initialized())
317 return std::numeric_limits<uint32_t>::max();
319 auto source_nb_frames = source_->nb_frames();
320 auto multiple = boost::rational_cast<double>(1 / get_speed() * (output_repeat_ != 0 ? 2 : 1));
322 return static_cast<uint32_t>(source_nb_frames * multiple);
325 uint32_t frame_number() const override
327 if (!is_initialized())
330 auto source_frame_number = source_->frame_number() - 1; // next frame already received
331 auto multiple = boost::rational_cast<double>(1 / get_speed() * (output_repeat_ != 0 ? 2 : 1));
333 return static_cast<uint32_t>(source_frame_number * multiple);
336 constraints& pixel_constraints() override
338 return source_->pixel_constraints();
341 bool is_initialized() const
343 return source_framerate_ != -1;
346 draw_frame do_render_progressive_frame(bool sound)
348 user_speed_.fetch_and_tick();
350 if (output_repeat_ && output_frame_++ % output_repeat_)
352 auto frame = last_frame_;
354 frame.transform().audio_transform.volume = 0.0;
356 return attach_sound(frame);
359 if (previous_frame_ == draw_frame::empty())
360 previous_frame_ = pop_frame_from_source();
362 auto current_frame_number = current_frame_number_;
363 auto distance = current_frame_number_ - boost::rational_cast<int64_t>(current_frame_number_);
364 bool needs_next = distance > 0 || !enough_sound();
366 if (needs_next && next_frame_ == draw_frame::empty())
367 next_frame_ = pop_frame_from_source();
369 auto result = interpolator_(previous_frame_, next_frame_, distance);
371 auto next_frame_number = current_frame_number_ += get_speed();
372 auto integer_current_frame = boost::rational_cast<std::int64_t>(current_frame_number);
373 auto integer_next_frame = boost::rational_cast<std::int64_t>(next_frame_number);
375 fast_forward_integer_frames(integer_next_frame - integer_current_frame);
377 last_frame_ = result;
380 return attach_sound(result);
385 void fast_forward_integer_frames(std::int64_t num_frames)
390 for (std::int64_t i = 0; i < num_frames; ++i)
392 if (next_frame_ == draw_frame::empty())
393 previous_frame_ = pop_frame_from_source();
396 previous_frame_ = std::move(next_frame_);
398 next_frame_ = pop_frame_from_source();
403 boost::rational<std::int64_t> get_speed() const
405 return speed_ * user_speed_.fetch();
408 draw_frame pop_frame_from_source()
410 auto frame = source_->receive();
411 update_source_framerate();
413 if (user_speed_.fetch() == 1)
415 audio_extractor extractor([this](const const_frame& frame)
417 if (source_channel_layout_ != frame.audio_channel_layout())
419 source_channel_layout_ = frame.audio_channel_layout();
421 // Insert silence samples so that the audio mixer is guaranteed to be filled.
422 auto min_num_samples_per_frame = *boost::min_element(destination_audio_cadence_);
423 auto max_num_samples_per_frame = *boost::max_element(destination_audio_cadence_);
424 auto cadence_safety_samples = max_num_samples_per_frame - min_num_samples_per_frame;
425 audio_samples_.resize(source_channel_layout_.num_channels * cadence_safety_samples, 0);
428 auto& buffer = frame.audio_data();
429 audio_samples_.insert(audio_samples_.end(), buffer.begin(), buffer.end());
432 frame.accept(extractor);
436 source_channel_layout_ = audio_channel_layout::invalid();
437 audio_samples_.clear();
440 frame.transform().audio_transform.volume = 0.0;
445 draw_frame attach_sound(draw_frame frame)
447 if (user_speed_.fetch() != 1 || source_channel_layout_ == audio_channel_layout::invalid())
450 mutable_audio_buffer buffer;
452 if (destination_audio_cadence_.front() * source_channel_layout_.num_channels == audio_samples_.size())
454 buffer.swap(audio_samples_);
456 else if (audio_samples_.size() >= destination_audio_cadence_.front() * source_channel_layout_.num_channels)
458 auto begin = audio_samples_.begin();
459 auto end = begin + destination_audio_cadence_.front() * source_channel_layout_.num_channels;
461 buffer.insert(buffer.begin(), begin, end);
462 audio_samples_.erase(begin, end);
466 auto needed = destination_audio_cadence_.front();
467 auto got = audio_samples_.size() / source_channel_layout_.num_channels;
468 if (got != 0) // If at end of stream we don't care
469 CASPAR_LOG(debug) << print() << L" Too few audio samples. Needed " << needed << L" but got " << got;
470 buffer.swap(audio_samples_);
471 buffer.resize(needed * source_channel_layout_.num_channels, 0);
474 boost::range::rotate(destination_audio_cadence_, std::begin(destination_audio_cadence_) + 1);
476 auto audio_frame = mutable_frame(
481 source_channel_layout_);
482 return draw_frame::over(frame, draw_frame(std::move(audio_frame)));
485 bool enough_sound() const
487 return source_channel_layout_ == core::audio_channel_layout::invalid()
488 || user_speed_.fetch() != 1
489 || audio_samples_.size() / source_channel_layout_.num_channels >= destination_audio_cadence_.at(0);
492 void update_source_framerate()
494 auto source_framerate = get_source_framerate_();
496 if (source_framerate_ == source_framerate)
501 source_framerate_ = source_framerate;
502 auto destination_framerate = original_destination_framerate_;
503 destination_fieldmode_ = original_destination_fieldmode_;
505 // Coarse adjustment to correct fps family (23.98 - 30 vs 47.95 - 60)
506 if (destination_fieldmode_ != field_mode::progressive) // Interlaced output
508 auto diff_double = boost::abs(source_framerate_ - destination_framerate * 2);
509 auto diff_keep = boost::abs(source_framerate_ - destination_framerate);
511 if (diff_double < diff_keep) // Double rate interlaced
513 destination_framerate *= 2;
515 else // Progressive non interlaced
517 destination_fieldmode_ = field_mode::progressive;
522 auto diff_halve = boost::abs(source_framerate_ * 2 - destination_framerate);
523 auto diff_keep = boost::abs(source_framerate_ - destination_framerate);
525 if (diff_halve < diff_keep) // Repeat every frame two times
527 destination_framerate /= 2;
532 speed_ = boost::rational<int64_t>(source_framerate_ / destination_framerate);
534 // drop_or_repeat will only be used by default for exact framerate multiples (half, same and double)
535 // for all other framerates a frame interpolator will be chosen.
536 if (speed_ != 1 && speed_ * 2 != 1 && speed_ != 2)
538 auto high_source_framerate = source_framerate_ > 47;
539 auto high_destination_framerate = destination_framerate > 47
540 || destination_fieldmode_ != field_mode::progressive;
542 if (high_source_framerate && high_destination_framerate) // The bluriness of blend3 is acceptable on high framerates.
543 interpolator_ = blend3();
544 else // blend3 is mostly too blurry on low framerates. blend2 provides a compromise.
545 interpolator_ = &blend2;
547 CASPAR_LOG(warning) << source_->print() << L" Frame blending frame rate conversion required to conform to channel frame rate.";
550 interpolator_ = &drop_or_repeat;
554 void describe_framerate_producer(help_sink& sink)
556 sink.para()->text(L"Framerate conversion control / Slow motion examples:");
557 sink.example(L">> CALL 1-10 FRAMERATE INTERPOLATION BLEND2", L"enables 2 frame blend interpolation.");
558 sink.example(L">> CALL 1-10 FRAMERATE INTERPOLATION BLEND3", L"enables 3 frame blend interpolation.");
559 sink.example(L">> CALL 1-10 FRAMERATE INTERPOLATION DROP_OR_REPEAT", L"disables frame interpolation.");
560 sink.example(L">> CALL 1-10 FRAMERATE SPEED 0.25", L"immediately changes the speed to 25%. Sound will be disabled.");
561 sink.example(L">> CALL 1-10 FRAMERATE SPEED 0.25 50", L"changes the speed to 25% linearly over 50 frames. Sound will be disabled.");
562 sink.example(L">> CALL 1-10 FRAMERATE SPEED 0.25 50 easeinoutsine", L"changes the speed to 25% over 50 frames using specified easing curve. Sound will be disabled.");
563 sink.example(L">> CALL 1-10 FRAMERATE SPEED 1 50", L"changes the speed to 100% linearly over 50 frames. Sound will be enabled when the destination speed of 100% has been reached.");
566 spl::shared_ptr<frame_producer> create_framerate_producer(
567 spl::shared_ptr<frame_producer> source,
568 std::function<boost::rational<int> ()> get_source_framerate,
569 boost::rational<int> destination_framerate,
570 field_mode destination_fieldmode,
571 std::vector<int> destination_audio_cadence)
573 return spl::make_shared<framerate_producer>(
575 std::move(get_source_framerate),
576 std::move(destination_framerate),
577 destination_fieldmode,
578 std::move(destination_audio_cadence));