git.sesse.net Git - casparcg/blob - core/producer/framerate/framerate_producer.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Helge Norberg, helge.norberg@svt.se
  20 */
  21
  22 #include "../../StdAfx.h"
  23
  24 #include "framerate_producer.h"
  25
  26 #include "../frame_producer.h"
  27 #include "../../frame/audio_channel_layout.h"
  28 #include "../../frame/draw_frame.h"
  29 #include "../../frame/frame.h"
  30 #include "../../frame/frame_transform.h"
  31 #include "../../frame/pixel_format.h"
  32 #include "../../monitor/monitor.h"
  33
  34 #include <common/future.h>
  35 #include <common/tweener.h>
  36
  37 #include <functional>
  38 #include <queue>
  39 #include <future>
  40
  41 namespace caspar { namespace core {
  42
  43 draw_frame drop_and_skip(const draw_frame& source, const draw_frame&, const boost::rational<int64_t>&)
  44 {
  45         return source;
  46 }
  47
  48 // Blends next frame with current frame when the distance is not 0.
  49 // Completely sharp when distance is 0 but blurry when in between.
  50 draw_frame blend(const draw_frame& source, const draw_frame& destination, const boost::rational<int64_t>& distance)
  51 {
  52         if (destination == draw_frame::empty())
  53                 return source;
  54
  55         auto under                                      = source;
  56         auto over                                       = destination;
  57         double float_distance           = static_cast<double>(distance.numerator()) / static_cast<double>(distance.denominator());
  58
  59         under.transform().image_transform.is_mix        = true;
  60         under.transform().image_transform.opacity       = 1 - float_distance;
  61         over.transform().image_transform.is_mix         = true;
  62         over.transform().image_transform.opacity        = float_distance;
  63
  64         return draw_frame::over(under, over);
  65 }
  66
  67 // Blends a moving window with a width of 1 frame duration.
  68 // * A distance of 0.0 gives 50% previous, 50% current and 0% next.
  69 // * A distance of 0.5 gives 25% previous, 50% current and 25% next.
  70 // * A distance of 0.75 gives 12.5% previous, 50% current and 37.5% next.
  71 // This is blurrier than blend, but gives a more even bluriness, instead of sharp, blurry, sharp, blurry.
  72 struct blend_all
  73 {
  74         draw_frame previous_frame       = draw_frame::empty();
  75         draw_frame last_source          = draw_frame::empty();
  76         draw_frame last_destination     = draw_frame::empty();
  77
  78         draw_frame operator()(const draw_frame& source, const draw_frame& destination, const boost::rational<int64_t>& distance)
  79         {
  80                 if (last_source != draw_frame::empty() && last_source != source)
  81                 {
  82                         if (last_destination == source)
  83                                 previous_frame = last_source;
  84                         else // A two frame jump
  85                                 previous_frame = last_destination;
  86                 }
  87
  88                 last_source                     = source;
  89                 last_destination        = destination;
  90
  91                 bool has_previous = previous_frame != draw_frame::empty();
  92
  93                 if (!has_previous)
  94                         return blend(source, destination, distance);
  95
  96                 auto middle                                                                                     = last_source;
  97                 auto next_frame                                                                         = destination;
  98                 previous_frame.transform().image_transform.is_mix       = true;
  99                 middle.transform().image_transform.is_mix                       = true;
 100                 next_frame.transform().image_transform.is_mix           = true;
 101
 102                 double float_distance                                                           = static_cast<double>(distance.numerator()) / static_cast<double>(distance.denominator());
 103                 previous_frame.transform().image_transform.opacity      = std::max(0.0, 0.5 - float_distance * 0.5);
 104                 middle.transform().image_transform.opacity                      = 0.5;
 105                 next_frame.transform().image_transform.opacity          = 1.0 - previous_frame.transform().image_transform.opacity - middle.transform().image_transform.opacity;
 106
 107                 std::vector<draw_frame> combination { previous_frame, middle, next_frame };
 108
 109                 return draw_frame(std::move(combination));
 110         }
 111 };
 112
 113 struct audio_extractor : public frame_visitor
 114 {
 115         std::function<void(const const_frame& frame)> on_frame_;
 116
 117         audio_extractor(std::function<void(const const_frame& frame)> on_frame)
 118                 : on_frame_(std::move(on_frame))
 119         {
 120         }
 121
 122         void push(const frame_transform& transform) override    { }
 123         void pop() override                                                                             { }
 124         void visit(const const_frame& frame) override
 125         {
 126                 if (!frame.audio_data().empty())
 127                         on_frame_(frame);
 128         }
 129 };
 130
 131 // Like tweened_transform but for framerates
 132 class speed_tweener
 133 {
 134         boost::rational<int64_t>        source_         = 1LL;
 135         boost::rational<int64_t>        dest_           = 1LL;
 136         int                                                     duration_       = 0;
 137         int                                                     time_           = 0;
 138         tweener                                         tweener_;
 139 public:
 140         speed_tweener() = default;
 141         speed_tweener(
 142                         const boost::rational<int64_t>& source,
 143                         const boost::rational<int64_t>& dest,
 144                         int duration,
 145                         const tweener& tween)
 146                 : source_(source)
 147                 , dest_(dest)
 148                 , duration_(duration)
 149                 , time_(0)
 150                 , tweener_(tween)
 151         {
 152         }
 153
 154         const boost::rational<int64_t>& dest() const
 155         {
 156                 return dest_;
 157         }
 158
 159         boost::rational<int64_t> fetch() const
 160         {
 161                 if (time_ == duration_)
 162                         return dest_;
 163                 double source   = static_cast<double>(source_.numerator()) / static_cast<double>(source_.denominator());
 164                 double delta    = static_cast<double>(dest_.numerator()) / static_cast<double>(dest_.denominator()) - source;
 165                 double result = tweener_(time_, source, delta, duration_);
 166
 167                 return boost::rational<int64_t>(static_cast<int64_t>(result * 1000000.0), 1000000);
 168         }
 169
 170         boost::rational<int64_t> fetch_and_tick()
 171         {
 172                 time_ = std::min(time_ + 1, duration_);
 173                 return fetch();
 174         }
 175 };
 176
 177 class framerate_producer : public frame_producer_base
 178 {
 179         spl::shared_ptr<frame_producer>                                         source_;
 180         boost::rational<int>                                                            source_framerate_;
 181         audio_channel_layout                                                            source_channel_layout_          = audio_channel_layout::invalid();
 182         boost::rational<int>                                                            destination_framerate_;
 183         field_mode                                                                                      destination_fieldmode_;
 184         std::vector<int>                                                                        destination_audio_cadence_;
 185         boost::rational<std::int64_t>                                           speed_;
 186         speed_tweener                                                                           user_speed_;
 187         std::function<draw_frame (
 188                         const draw_frame& source,
 189                         const draw_frame& destination,
 190                         const boost::rational<int64_t>& distance)>      interpolator_                           = drop_and_skip;
 191
 192         boost::rational<std::int64_t>                                           current_frame_number_           = 0;
 193         draw_frame                                                                                      previous_frame_                         = draw_frame::empty();
 194         draw_frame                                                                                      next_frame_                                     = draw_frame::empty();
 195         mutable_audio_buffer                                                            audio_samples_;
 196
 197         unsigned int                                                                            output_repeat_                          = 0;
 198         unsigned int                                                                            output_frame_                           = 0;
 199 public:
 200         framerate_producer(
 201                         spl::shared_ptr<frame_producer> source,
 202                         boost::rational<int> source_framerate,
 203                         boost::rational<int> destination_framerate,
 204                         field_mode destination_fieldmode,
 205                         std::vector<int> destination_audio_cadence)
 206                 : source_(std::move(source))
 207                 , source_framerate_(std::move(source_framerate))
 208                 , destination_framerate_(std::move(destination_framerate))
 209                 , destination_fieldmode_(destination_fieldmode)
 210                 , destination_audio_cadence_(std::move(destination_audio_cadence))
 211         {
 212                 // Coarse adjustment to correct fps family (23.98 - 30 vs 47.95 - 60)
 213                 if (destination_fieldmode_ != field_mode::progressive)  // Interlaced output
 214                 {
 215                         auto diff_double        = boost::abs(source_framerate_ - destination_framerate_ * 2);
 216                         auto diff_keep          = boost::abs(source_framerate_ - destination_framerate_);
 217
 218                         if (diff_double < diff_keep)                                            // Double rate interlaced
 219                         {
 220                                 destination_framerate_ *= 2;
 221                         }
 222                         else                                                                                            // Progressive non interlaced
 223                         {
 224                                 destination_fieldmode_ = field_mode::progressive;
 225                         }
 226                 }
 227                 else                                                                                                    // Progressive
 228                 {
 229                         auto diff_halve = boost::abs(source_framerate_ * 2      - destination_framerate_);
 230                         auto diff_keep  = boost::abs(source_framerate_          - destination_framerate_);
 231
 232                         if (diff_halve < diff_keep)                                                     // Repeat every frame two times
 233                         {
 234                                 destination_framerate_  /= 2;
 235                                 output_repeat_                  = 2;
 236                         }
 237                 }
 238
 239                 speed_ = boost::rational<int64_t>(source_framerate_ / destination_framerate_);
 240
 241                 // drop_and_skip will only be used by default for exact framerate multiples (half, same and double)
 242                 // for all other framerates a frame interpolator will be chosen.
 243                 if (speed_ != 1 && speed_ * 2 != 1 && speed_ != 2)
 244                 {
 245                         if (source_framerate_ > 47)             // The bluriness of blend_all is acceptable on high framerates.
 246                                 interpolator_ = blend_all();
 247                         else                                                    // blend_all is mostly too blurry on low framerates. blend provides a compromise.
 248                                 interpolator_ = &blend;
 249
 250                         CASPAR_LOG(warning) << source_->print() << L" Frame blending frame rate conversion required to conform to channel frame rate.";
 251                 }
 252
 253                 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
 254                 // This cadence fills the audio mixer most optimally.
 255                 boost::range::rotate(destination_audio_cadence_, std::end(destination_audio_cadence_) - 1);
 256         }
 257
 258         draw_frame receive_impl() override
 259         {
 260                 if (destination_fieldmode_ == field_mode::progressive)
 261                 {
 262                         return do_render_progressive_frame(true);
 263                 }
 264                 else
 265                 {
 266                         auto field1 = do_render_progressive_frame(true);
 267                         auto field2 = do_render_progressive_frame(false);
 268
 269                         return draw_frame::interlace(field1, field2, destination_fieldmode_);
 270                 }
 271         }
 272
 273         std::future<std::wstring> call(const std::vector<std::wstring>& params) override
 274         {
 275                 if (!boost::iequals(params.at(0), L"framerate"))
 276                         return source_->call(params);
 277
 278                 if (boost::iequals(params.at(1), L"speed"))
 279                 {
 280                         auto destination_user_speed = boost::rational<std::int64_t>(
 281                                         static_cast<std::int64_t>(boost::lexical_cast<double>(params.at(2)) * 1000000.0),
 282                                         1000000);
 283                         auto frames = params.size() > 3 ? boost::lexical_cast<int>(params.at(3)) : 0;
 284                         auto easing = params.size() > 4 ? params.at(4) : L"linear";
 285
 286                         user_speed_ = speed_tweener(user_speed_.fetch(), destination_user_speed, frames, tweener(easing));
 287                 }
 288                 else if (boost::iequals(params.at(1), L"interpolation"))
 289                 {
 290                         if (boost::iequals(params.at(2), L"blend"))
 291                                 interpolator_ = &blend;
 292                         else if (boost::iequals(params.at(2), L"blend_all"))
 293                                 interpolator_ = blend_all();
 294                         else
 295                                 interpolator_ = &drop_and_skip;
 296                 }
 297                 else if (boost::iequals(params.at(1), L"output_repeat")) // Only for debugging purposes
 298                 {
 299                         output_repeat_ = boost::lexical_cast<unsigned int>(params.at(2));
 300                 }
 301
 302                 return make_ready_future<std::wstring>(L"");
 303         }
 304
 305         monitor::subject& monitor_output() override
 306         {
 307                 return source_->monitor_output();
 308         }
 309
 310         std::wstring print() const override
 311         {
 312                 return source_->print();
 313         }
 314
 315         std::wstring name() const override
 316         {
 317                 return source_->name();
 318         }
 319
 320         boost::property_tree::wptree info() const override
 321         {
 322                 return source_->info();
 323         }
 324
 325         constraints& pixel_constraints() override
 326         {
 327                 return source_->pixel_constraints();
 328         }
 329 private:
 330         draw_frame do_render_progressive_frame(bool sound)
 331         {
 332                 user_speed_.fetch_and_tick();
 333
 334                 if (output_repeat_ && ++output_frame_ % output_repeat_)
 335                 {
 336                         auto frame = draw_frame::still(last_frame());
 337
 338                         frame.transform().audio_transform.volume = 0.0;
 339
 340                         return attach_sound(frame);
 341                 }
 342
 343                 if (previous_frame_ == draw_frame::empty())
 344                         previous_frame_ = pop_frame_from_source();
 345
 346                 auto current_frame_number       = current_frame_number_;
 347                 auto distance                           = current_frame_number_ - boost::rational_cast<int64_t>(current_frame_number_);
 348                 bool needs_next                         = distance > 0 || !enough_sound();
 349
 350                 if (needs_next && next_frame_ == draw_frame::empty())
 351                         next_frame_ = pop_frame_from_source();
 352
 353                 auto result = interpolator_(previous_frame_, next_frame_, distance);
 354
 355                 auto next_frame_number          = current_frame_number_ += get_speed();
 356                 auto integer_current_frame      = boost::rational_cast<std::int64_t>(current_frame_number);
 357                 auto integer_next_frame         = boost::rational_cast<std::int64_t>(next_frame_number);
 358
 359                 fast_forward_integer_frames(integer_next_frame - integer_current_frame);
 360
 361                 if (sound)
 362                         return attach_sound(result);
 363                 else
 364                         return result;
 365         }
 366
 367         void fast_forward_integer_frames(std::int64_t num_frames)
 368         {
 369                 if (num_frames == 0)
 370                         return;
 371
 372                 for (std::int64_t i = 0; i < num_frames; ++i)
 373                 {
 374                         previous_frame_ = std::move(next_frame_);
 375
 376                         next_frame_ = pop_frame_from_source();
 377                 }
 378         }
 379
 380         boost::rational<std::int64_t> get_speed() const
 381         {
 382                 return speed_ * user_speed_.fetch();
 383         }
 384
 385         draw_frame pop_frame_from_source()
 386         {
 387                 auto frame = source_->receive();
 388
 389                 if (user_speed_.fetch() == 1)
 390                 {
 391                         audio_extractor extractor([this](const const_frame& frame)
 392                         {
 393                                 if (source_channel_layout_ != frame.audio_channel_layout())
 394                                 {
 395                                         source_channel_layout_ = frame.audio_channel_layout();
 396
 397                                         // Insert silence samples so that the audio mixer is guaranteed to be filled.
 398                                         auto min_num_samples_per_frame  = *boost::min_element(destination_audio_cadence_);
 399                                         auto max_num_samples_per_frame  = *boost::max_element(destination_audio_cadence_);
 400                                         auto cadence_safety_samples             = max_num_samples_per_frame - min_num_samples_per_frame;
 401                                         audio_samples_.resize(source_channel_layout_.num_channels * cadence_safety_samples, 0);
 402                                 }
 403
 404                                 auto& buffer = frame.audio_data();
 405                                 audio_samples_.insert(audio_samples_.end(), buffer.begin(), buffer.end());
 406                         });
 407
 408                         frame.accept(extractor);
 409                 }
 410                 else
 411                 {
 412                         source_channel_layout_ = audio_channel_layout::invalid();
 413                         audio_samples_.clear();
 414                 }
 415
 416                 frame.transform().audio_transform.volume = 0.0;
 417
 418                 return frame;
 419         }
 420
 421         draw_frame attach_sound(draw_frame frame)
 422         {
 423                 if (user_speed_.fetch() != 1 || source_channel_layout_ == audio_channel_layout::invalid())
 424                         return frame;
 425
 426                 mutable_audio_buffer buffer;
 427
 428                 if (destination_audio_cadence_.front() * source_channel_layout_.num_channels == audio_samples_.size())
 429                 {
 430                         buffer.swap(audio_samples_);
 431                 }
 432                 else if (audio_samples_.size() >= destination_audio_cadence_.front() * source_channel_layout_.num_channels)
 433                 {
 434                         auto begin      = audio_samples_.begin();
 435                         auto end        = begin + destination_audio_cadence_.front() * source_channel_layout_.num_channels;
 436
 437                         buffer.insert(buffer.begin(), begin, end);
 438                         audio_samples_.erase(begin, end);
 439                 }
 440                 else
 441                 {
 442                         auto needed = destination_audio_cadence_.front();
 443                         auto got = audio_samples_.size() / source_channel_layout_.num_channels;
 444                         CASPAR_LOG(debug) << print() << L" Too few audio samples. Needed " << needed << L" but got " << got;
 445                         buffer.swap(audio_samples_);
 446                         buffer.resize(needed * source_channel_layout_.num_channels, 0);
 447                 }
 448
 449                 boost::range::rotate(destination_audio_cadence_, std::begin(destination_audio_cadence_) + 1);
 450
 451                 auto audio_frame = mutable_frame(
 452                                 {},
 453                                 std::move(buffer),
 454                                 this,
 455                                 pixel_format_desc(),
 456                                 source_channel_layout_);
 457                 return draw_frame::over(frame, draw_frame(std::move(audio_frame)));
 458         }
 459
 460         bool enough_sound() const
 461         {
 462                 return source_channel_layout_ == core::audio_channel_layout::invalid()
 463                                 || user_speed_.fetch() != 1
 464                                 || audio_samples_.size() / source_channel_layout_.num_channels >= destination_audio_cadence_.at(0);
 465         }
 466 };
 467
 468 spl::shared_ptr<frame_producer> create_framerate_producer(
 469                 spl::shared_ptr<frame_producer> source,
 470                 boost::rational<int> source_framerate,
 471                 boost::rational<int> destination_framerate,
 472                 field_mode destination_fieldmode,
 473                 std::vector<int> destination_audio_cadence)
 474 {
 475         return spl::make_shared<framerate_producer>(
 476                         std::move(source),
 477                         std::move(source_framerate),
 478                         std::move(destination_framerate),
 479                         destination_fieldmode,
 480                         std::move(destination_audio_cadence));
 481 }
 482
 483 }}
 484