git.sesse.net Git - casparcg/blob - core/producer/framerate/framerate_producer.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Helge Norberg, helge.norberg@svt.se
  20 */
  21
  22 #include "../../StdAfx.h"
  23
  24 #include "framerate_producer.h"
  25
  26 #include "../frame_producer.h"
  27 #include "../../frame/audio_channel_layout.h"
  28 #include "../../frame/draw_frame.h"
  29 #include "../../frame/frame.h"
  30 #include "../../frame/frame_transform.h"
  31 #include "../../frame/pixel_format.h"
  32 #include "../../monitor/monitor.h"
  33 #include "../../help/help_sink.h"
  34
  35 #include <common/future.h>
  36 #include <common/tweener.h>
  37
  38 #include <functional>
  39 #include <queue>
  40 #include <future>
  41 #include <stack>
  42
  43 namespace caspar { namespace core {
  44
  45 draw_frame drop_or_repeat(const draw_frame& source, const draw_frame&, const boost::rational<int64_t>&)
  46 {
  47         return source;
  48 }
  49
  50 // Blends next frame with current frame when the distance is not 0.
  51 // Completely sharp when distance is 0 but blurry when in between.
  52 draw_frame blend2(const draw_frame& source, const draw_frame& destination, const boost::rational<int64_t>& distance)
  53 {
  54         if (destination == draw_frame::empty())
  55                 return source;
  56
  57         auto under                                      = source;
  58         auto over                                       = destination;
  59         double float_distance           = boost::rational_cast<double>(distance);
  60
  61         under.transform().image_transform.is_mix        = true;
  62         under.transform().image_transform.opacity       = 1 - float_distance;
  63         over.transform().image_transform.is_mix         = true;
  64         over.transform().image_transform.opacity        = float_distance;
  65
  66         return draw_frame::over(under, over);
  67 }
  68
  69 // Blends a moving window with a width of 1 frame duration.
  70 // * A distance of 0.0 gives 50% previous, 50% current and 0% next.
  71 // * A distance of 0.5 gives 25% previous, 50% current and 25% next.
  72 // * A distance of 0.75 gives 12.5% previous, 50% current and 37.5% next.
  73 // This is blurrier than blend2, but gives a more even bluriness, instead of sharp, blurry, sharp, blurry.
  74 struct blend3
  75 {
  76         draw_frame previous_frame       = draw_frame::empty();
  77         draw_frame last_source          = draw_frame::empty();
  78         draw_frame last_destination     = draw_frame::empty();
  79
  80         draw_frame operator()(const draw_frame& source, const draw_frame& destination, const boost::rational<int64_t>& distance)
  81         {
  82                 if (last_source != draw_frame::empty() && last_source != source)
  83                 {
  84                         if (last_destination == source)
  85                                 previous_frame = last_source;
  86                         else // A two frame jump
  87                                 previous_frame = last_destination;
  88                 }
  89
  90                 last_source                     = source;
  91                 last_destination        = destination;
  92
  93                 bool has_previous = previous_frame != draw_frame::empty();
  94
  95                 if (!has_previous)
  96                         return blend2(source, destination, distance);
  97
  98                 auto middle                                                                                     = last_source;
  99                 auto next_frame                                                                         = destination;
 100                 previous_frame.transform().image_transform.is_mix       = true;
 101                 middle.transform().image_transform.is_mix                       = true;
 102                 next_frame.transform().image_transform.is_mix           = true;
 103
 104                 double float_distance                                                           = boost::rational_cast<double>(distance);
 105                 previous_frame.transform().image_transform.opacity      = std::max(0.0, 0.5 - float_distance * 0.5);
 106                 middle.transform().image_transform.opacity                      = 0.5;
 107                 next_frame.transform().image_transform.opacity          = 1.0 - previous_frame.transform().image_transform.opacity - middle.transform().image_transform.opacity;
 108
 109                 std::vector<draw_frame> combination { previous_frame, middle, next_frame };
 110
 111                 return draw_frame(std::move(combination));
 112         }
 113 };
 114
 115 class audio_extractor : public frame_visitor
 116 {
 117         std::stack<core::audio_transform>                               transform_stack_;
 118         std::function<void(const const_frame& frame)>   on_frame_;
 119 public:
 120         audio_extractor(std::function<void(const const_frame& frame)> on_frame)
 121                 : on_frame_(std::move(on_frame))
 122         {
 123                 transform_stack_.push(audio_transform());
 124         }
 125
 126         void push(const frame_transform& transform) override
 127         {
 128                 transform_stack_.push(transform_stack_.top() * transform.audio_transform);
 129         }
 130
 131         void pop() override
 132         {
 133                 transform_stack_.pop();
 134         }
 135
 136         void visit(const const_frame& frame) override
 137         {
 138                 if (!frame.audio_data().empty() && !transform_stack_.top().is_still && !transform_stack_.top().volume == 0.0)
 139                         on_frame_(frame);
 140         }
 141 };
 142
 143 // Like tweened_transform but for framerates
 144 class speed_tweener
 145 {
 146         boost::rational<int64_t>        source_         = 1LL;
 147         boost::rational<int64_t>        dest_           = 1LL;
 148         int                                                     duration_       = 0;
 149         int                                                     time_           = 0;
 150         tweener                                         tweener_;
 151 public:
 152         speed_tweener() = default;
 153         speed_tweener(
 154                         const boost::rational<int64_t>& source,
 155                         const boost::rational<int64_t>& dest,
 156                         int duration,
 157                         const tweener& tween)
 158                 : source_(source)
 159                 , dest_(dest)
 160                 , duration_(duration)
 161                 , time_(0)
 162                 , tweener_(tween)
 163         {
 164         }
 165
 166         const boost::rational<int64_t>& dest() const
 167         {
 168                 return dest_;
 169         }
 170
 171         boost::rational<int64_t> fetch() const
 172         {
 173                 if (time_ == duration_)
 174                         return dest_;
 175
 176                 double source   = boost::rational_cast<double>(source_);
 177                 double delta    = boost::rational_cast<double>(dest_) - source;
 178                 double result   = tweener_(time_, source, delta, duration_);
 179
 180                 return boost::rational<int64_t>(static_cast<int64_t>(result * 1000000.0), 1000000);
 181         }
 182
 183         boost::rational<int64_t> fetch_and_tick()
 184         {
 185                 time_ = std::min(time_ + 1, duration_);
 186                 return fetch();
 187         }
 188 };
 189
 190 class framerate_producer : public frame_producer_base
 191 {
 192         spl::shared_ptr<frame_producer>                                         source_;
 193         std::function<boost::rational<int>()>                           get_source_framerate_;
 194         boost::rational<int>                                                            source_framerate_                               = -1;
 195         audio_channel_layout                                                            source_channel_layout_                  = audio_channel_layout::invalid();
 196         const boost::rational<int>                                                      original_destination_framerate_;
 197         const field_mode                                                                        original_destination_fieldmode_;
 198         field_mode                                                                                      destination_fieldmode_                  = field_mode::empty;
 199         std::vector<int>                                                                        destination_audio_cadence_;
 200         boost::rational<std::int64_t>                                           speed_;
 201         speed_tweener                                                                           user_speed_;
 202         std::function<draw_frame (
 203                         const draw_frame& source,
 204                         const draw_frame& destination,
 205                         const boost::rational<int64_t>& distance)>      interpolator_                                   = drop_or_repeat;
 206
 207         boost::rational<std::int64_t>                                           current_frame_number_                   = 0;
 208         draw_frame                                                                                      previous_frame_                                 = draw_frame::empty();
 209         draw_frame                                                                                      next_frame_                                             = draw_frame::empty();
 210         mutable_audio_buffer                                                            audio_samples_;
 211
 212         unsigned int                                                                            output_repeat_                                  = 0;
 213         unsigned int                                                                            output_frame_                                   = 0;
 214         draw_frame                                                                                      last_frame_                                             = draw_frame::empty();
 215 public:
 216         framerate_producer(
 217                         spl::shared_ptr<frame_producer> source,
 218                         std::function<boost::rational<int> ()> get_source_framerate,
 219                         boost::rational<int> destination_framerate,
 220                         field_mode destination_fieldmode,
 221                         std::vector<int> destination_audio_cadence)
 222                 : source_(std::move(source))
 223                 , get_source_framerate_(std::move(get_source_framerate))
 224                 , original_destination_framerate_(std::move(destination_framerate))
 225                 , original_destination_fieldmode_(destination_fieldmode)
 226                 , destination_audio_cadence_(std::move(destination_audio_cadence))
 227         {
 228                 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
 229                 // This cadence fills the audio mixer most optimally.
 230                 boost::range::rotate(destination_audio_cadence_, std::end(destination_audio_cadence_) - 1);
 231         }
 232
 233         draw_frame receive_impl() override
 234         {
 235                 // destination field mode initially unknown but known after the first update_source_framerate().
 236                 auto field1 = do_render_progressive_frame(true);
 237
 238                 if (destination_fieldmode_ == field_mode::progressive)
 239                 {
 240                         return field1;
 241                 }
 242                 else
 243                 {
 244                         auto field2 = do_render_progressive_frame(false);
 245
 246                         return draw_frame::interlace(field1, field2, destination_fieldmode_);
 247                 }
 248         }
 249
 250         std::future<std::wstring> call(const std::vector<std::wstring>& params) override
 251         {
 252                 if (!boost::iequals(params.at(0), L"framerate"))
 253                         return source_->call(params);
 254
 255                 if (boost::iequals(params.at(1), L"speed"))
 256                 {
 257                         auto destination_user_speed = boost::rational<std::int64_t>(
 258                                         static_cast<std::int64_t>(std::stod(params.at(2)) * 1000000.0),
 259                                         1000000);
 260                         auto frames = params.size() > 3 ? std::stoi(params.at(3)) : 0;
 261                         auto easing = params.size() > 4 ? params.at(4) : L"linear";
 262
 263                         user_speed_ = speed_tweener(user_speed_.fetch(), destination_user_speed, frames, tweener(easing));
 264                 }
 265                 else if (boost::iequals(params.at(1), L"interpolation"))
 266                 {
 267                         if (boost::iequals(params.at(2), L"blend2"))
 268                                 interpolator_ = &blend2;
 269                         else if (boost::iequals(params.at(2), L"blend3"))
 270                                 interpolator_ = blend3();
 271                         else if (boost::iequals(params.at(2), L"drop_or_repeat"))
 272                                 interpolator_ = &drop_or_repeat;
 273                         else
 274                                 CASPAR_THROW_EXCEPTION(user_error() << msg_info("Valid interpolations are DROP_OR_REPEAT, BLEND2 and BLEND3"));
 275                 }
 276                 else if (boost::iequals(params.at(1), L"output_repeat")) // Only for debugging purposes
 277                 {
 278                         output_repeat_ = boost::lexical_cast<unsigned int>(params.at(2));
 279                 }
 280
 281                 return make_ready_future<std::wstring>(L"");
 282         }
 283
 284         monitor::subject& monitor_output() override
 285         {
 286                 return source_->monitor_output();
 287         }
 288
 289         std::wstring print() const override
 290         {
 291                 return source_->print();
 292         }
 293
 294         std::wstring name() const override
 295         {
 296                 return source_->name();
 297         }
 298
 299         boost::property_tree::wptree info() const override
 300         {
 301                 auto info = source_->info();
 302
 303                 auto incorrect_frame_number = info.get_child_optional(L"frame-number");
 304                 if (incorrect_frame_number)
 305                         incorrect_frame_number->put_value(frame_number());
 306
 307                 auto incorrect_nb_frames = info.get_child_optional(L"nb-frames");
 308                 if (incorrect_nb_frames)
 309                         incorrect_nb_frames->put_value(nb_frames());
 310
 311                 return info;
 312         }
 313
 314         uint32_t nb_frames() const override
 315         {
 316                 if (!is_initialized())
 317                         return std::numeric_limits<uint32_t>::max();
 318
 319                 auto source_nb_frames = source_->nb_frames();
 320                 auto multiple = boost::rational_cast<double>(1 / get_speed() * (output_repeat_ != 0 ? 2 : 1));
 321
 322                 return static_cast<uint32_t>(source_nb_frames * multiple);
 323         }
 324
 325         uint32_t frame_number() const override
 326         {
 327                 if (!is_initialized())
 328                         return 0;
 329
 330                 auto source_frame_number = source_->frame_number() - 1; // next frame already received
 331                 auto multiple = boost::rational_cast<double>(1 / get_speed() * (output_repeat_ != 0 ? 2 : 1));
 332
 333                 return static_cast<uint32_t>(source_frame_number * multiple);
 334         }
 335
 336         constraints& pixel_constraints() override
 337         {
 338                 return source_->pixel_constraints();
 339         }
 340 private:
 341         bool is_initialized() const
 342         {
 343                 return source_framerate_ != -1;
 344         }
 345
 346         draw_frame do_render_progressive_frame(bool sound)
 347         {
 348                 user_speed_.fetch_and_tick();
 349
 350                 if (output_repeat_ && output_frame_++ % output_repeat_)
 351                 {
 352                         auto frame = last_frame_;
 353
 354                         frame.transform().audio_transform.volume = 0.0;
 355
 356                         return attach_sound(frame);
 357                 }
 358
 359                 if (previous_frame_ == draw_frame::empty())
 360                         previous_frame_ = pop_frame_from_source();
 361
 362                 auto current_frame_number       = current_frame_number_;
 363                 auto distance                           = current_frame_number_ - boost::rational_cast<int64_t>(current_frame_number_);
 364                 bool needs_next                         = distance > 0 || !enough_sound();
 365
 366                 if (needs_next && next_frame_ == draw_frame::empty())
 367                         next_frame_ = pop_frame_from_source();
 368
 369                 auto result = interpolator_(previous_frame_, next_frame_, distance);
 370
 371                 auto next_frame_number          = current_frame_number_ += get_speed();
 372                 auto integer_current_frame      = boost::rational_cast<std::int64_t>(current_frame_number);
 373                 auto integer_next_frame         = boost::rational_cast<std::int64_t>(next_frame_number);
 374
 375                 fast_forward_integer_frames(integer_next_frame - integer_current_frame);
 376
 377                 last_frame_ = result;
 378
 379                 if (sound)
 380                         return attach_sound(result);
 381                 else
 382                         return result;
 383         }
 384
 385         void fast_forward_integer_frames(std::int64_t num_frames)
 386         {
 387                 if (num_frames == 0)
 388                         return;
 389
 390                 for (std::int64_t i = 0; i < num_frames; ++i)
 391                 {
 392                         if (next_frame_ == draw_frame::empty())
 393                                 previous_frame_ = pop_frame_from_source();
 394                         else
 395                         {
 396                                 previous_frame_ = std::move(next_frame_);
 397
 398                                 next_frame_ = pop_frame_from_source();
 399                         }
 400                 }
 401         }
 402
 403         boost::rational<std::int64_t> get_speed() const
 404         {
 405                 return speed_ * user_speed_.fetch();
 406         }
 407
 408         draw_frame pop_frame_from_source()
 409         {
 410                 auto frame = source_->receive();
 411                 update_source_framerate();
 412
 413                 if (user_speed_.fetch() == 1)
 414                 {
 415                         audio_extractor extractor([this](const const_frame& frame)
 416                         {
 417                                 if (source_channel_layout_ != frame.audio_channel_layout())
 418                                 {
 419                                         source_channel_layout_ = frame.audio_channel_layout();
 420
 421                                         // Insert silence samples so that the audio mixer is guaranteed to be filled.
 422                                         auto min_num_samples_per_frame  = *boost::min_element(destination_audio_cadence_);
 423                                         auto max_num_samples_per_frame  = *boost::max_element(destination_audio_cadence_);
 424                                         auto cadence_safety_samples             = max_num_samples_per_frame - min_num_samples_per_frame;
 425                                         audio_samples_.resize(source_channel_layout_.num_channels * cadence_safety_samples, 0);
 426                                 }
 427
 428                                 auto& buffer = frame.audio_data();
 429                                 audio_samples_.insert(audio_samples_.end(), buffer.begin(), buffer.end());
 430                         });
 431
 432                         frame.accept(extractor);
 433                 }
 434                 else
 435                 {
 436                         source_channel_layout_ = audio_channel_layout::invalid();
 437                         audio_samples_.clear();
 438                 }
 439
 440                 frame.transform().audio_transform.volume = 0.0;
 441
 442                 return frame;
 443         }
 444
 445         draw_frame attach_sound(draw_frame frame)
 446         {
 447                 if (user_speed_.fetch() != 1 || source_channel_layout_ == audio_channel_layout::invalid())
 448                         return frame;
 449
 450                 mutable_audio_buffer buffer;
 451
 452                 if (destination_audio_cadence_.front() * source_channel_layout_.num_channels == audio_samples_.size())
 453                 {
 454                         buffer.swap(audio_samples_);
 455                 }
 456                 else if (audio_samples_.size() >= destination_audio_cadence_.front() * source_channel_layout_.num_channels)
 457                 {
 458                         auto begin      = audio_samples_.begin();
 459                         auto end        = begin + destination_audio_cadence_.front() * source_channel_layout_.num_channels;
 460
 461                         buffer.insert(buffer.begin(), begin, end);
 462                         audio_samples_.erase(begin, end);
 463                 }
 464                 else
 465                 {
 466                         auto needed = destination_audio_cadence_.front();
 467                         auto got = audio_samples_.size() / source_channel_layout_.num_channels;
 468                         if (got != 0) // If at end of stream we don't care
 469                                 CASPAR_LOG(debug) << print() << L" Too few audio samples. Needed " << needed << L" but got " << got;
 470                         buffer.swap(audio_samples_);
 471                         buffer.resize(needed * source_channel_layout_.num_channels, 0);
 472                 }
 473
 474                 boost::range::rotate(destination_audio_cadence_, std::begin(destination_audio_cadence_) + 1);
 475
 476                 auto audio_frame = mutable_frame(
 477                                 {},
 478                                 std::move(buffer),
 479                                 this,
 480                                 pixel_format_desc(),
 481                                 source_channel_layout_);
 482                 return draw_frame::over(frame, draw_frame(std::move(audio_frame)));
 483         }
 484
 485         bool enough_sound() const
 486         {
 487                 return source_channel_layout_ == core::audio_channel_layout::invalid()
 488                                 || user_speed_.fetch() != 1
 489                                 || audio_samples_.size() / source_channel_layout_.num_channels >= destination_audio_cadence_.at(0);
 490         }
 491
 492         void update_source_framerate()
 493         {
 494                 auto source_framerate = get_source_framerate_();
 495
 496                 if (source_framerate_ == source_framerate)
 497                         return;
 498
 499                 output_repeat_                          = 0;
 500                 output_frame_                           = 0;
 501                 source_framerate_                       = source_framerate;
 502                 auto destination_framerate      = original_destination_framerate_;
 503                 destination_fieldmode_          = original_destination_fieldmode_;
 504
 505                 // Coarse adjustment to correct fps family (23.98 - 30 vs 47.95 - 60)
 506                 if (destination_fieldmode_ != field_mode::progressive)  // Interlaced output
 507                 {
 508                         auto diff_double        = boost::abs(source_framerate_ - destination_framerate * 2);
 509                         auto diff_keep          = boost::abs(source_framerate_ - destination_framerate);
 510
 511                         if (diff_double < diff_keep)                                            // Double rate interlaced
 512                         {
 513                                 destination_framerate *= 2;
 514                         }
 515                         else                                                                                            // Progressive non interlaced
 516                         {
 517                                 destination_fieldmode_ = field_mode::progressive;
 518                         }
 519                 }
 520                 else                                                                                                    // Progressive
 521                 {
 522                         auto diff_halve = boost::abs(source_framerate_ * 2      - destination_framerate);
 523                         auto diff_keep  = boost::abs(source_framerate_          - destination_framerate);
 524
 525                         if (diff_halve < diff_keep)                                                     // Repeat every frame two times
 526                         {
 527                                 destination_framerate   /= 2;
 528                                 output_repeat_                  = 2;
 529                         }
 530                 }
 531
 532                 speed_ = boost::rational<int64_t>(source_framerate_ / destination_framerate);
 533
 534                 // drop_or_repeat will only be used by default for exact framerate multiples (half, same and double)
 535                 // for all other framerates a frame interpolator will be chosen.
 536                 if (speed_ != 1 && speed_ * 2 != 1 && speed_ != 2)
 537                 {
 538                         auto high_source_framerate              = source_framerate_ > 47;
 539                         auto high_destination_framerate = destination_framerate > 47
 540                                         || destination_fieldmode_ != field_mode::progressive;
 541
 542                         if (high_source_framerate && high_destination_framerate)        // The bluriness of blend3 is acceptable on high framerates.
 543                                 interpolator_   = blend3();
 544                         else                                                                                                            // blend3 is mostly too blurry on low framerates. blend2 provides a compromise.
 545                                 interpolator_   = &blend2;
 546
 547                         CASPAR_LOG(warning) << source_->print() << L" Frame blending frame rate conversion required to conform to channel frame rate.";
 548                 }
 549                 else
 550                         interpolator_           = &drop_or_repeat;
 551         }
 552 };
 553
 554 void describe_framerate_producer(help_sink& sink)
 555 {
 556         sink.para()->text(L"Framerate conversion control / Slow motion examples:");
 557         sink.example(L">> CALL 1-10 FRAMERATE INTERPOLATION BLEND2", L"enables 2 frame blend interpolation.");
 558         sink.example(L">> CALL 1-10 FRAMERATE INTERPOLATION BLEND3", L"enables 3 frame blend interpolation.");
 559         sink.example(L">> CALL 1-10 FRAMERATE INTERPOLATION DROP_OR_REPEAT", L"disables frame interpolation.");
 560         sink.example(L">> CALL 1-10 FRAMERATE SPEED 0.25", L"immediately changes the speed to 25%. Sound will be disabled.");
 561         sink.example(L">> CALL 1-10 FRAMERATE SPEED 0.25 50", L"changes the speed to 25% linearly over 50 frames. Sound will be disabled.");
 562         sink.example(L">> CALL 1-10 FRAMERATE SPEED 0.25 50 easeinoutsine", L"changes the speed to 25% over 50 frames using specified easing curve. Sound will be disabled.");
 563         sink.example(L">> CALL 1-10 FRAMERATE SPEED 1 50", L"changes the speed to 100% linearly over 50 frames. Sound will be enabled when the destination speed of 100% has been reached.");
 564 }
 565
 566 spl::shared_ptr<frame_producer> create_framerate_producer(
 567                 spl::shared_ptr<frame_producer> source,
 568                 std::function<boost::rational<int> ()> get_source_framerate,
 569                 boost::rational<int> destination_framerate,
 570                 field_mode destination_fieldmode,
 571                 std::vector<int> destination_audio_cadence)
 572 {
 573         return spl::make_shared<framerate_producer>(
 574                         std::move(source),
 575                         std::move(get_source_framerate),
 576                         std::move(destination_framerate),
 577                         destination_fieldmode,
 578                         std::move(destination_audio_cadence));
 579 }
 580
 581 }}