git.sesse.net Git - casparcg/blob - core/producer/framerate/framerate_producer.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Helge Norberg, helge.norberg@svt.se
  20 */
  21
  22 #include "../../StdAfx.h"
  23
  24 #include "framerate_producer.h"
  25
  26 #include "../frame_producer.h"
  27 #include "../../frame/audio_channel_layout.h"
  28 #include "../../frame/draw_frame.h"
  29 #include "../../frame/frame.h"
  30 #include "../../frame/frame_transform.h"
  31 #include "../../frame/pixel_format.h"
  32 #include "../../monitor/monitor.h"
  33
  34 #include <common/future.h>
  35
  36 #include <functional>
  37 #include <queue>
  38 #include <future>
  39
  40 namespace caspar { namespace core {
  41
  42 draw_frame drop_and_skip(const draw_frame& source, const draw_frame&, const boost::rational<int64_t>&)
  43 {
  44         return source;
  45 }
  46
  47 // Blends next frame with current frame when the distance is not 0.
  48 // Completely sharp when distance is 0 but blurry when in between.
  49 draw_frame blend(const draw_frame& source, const draw_frame& destination, const boost::rational<int64_t>& distance)
  50 {
  51         if (destination == draw_frame::empty())
  52                 return source;
  53
  54         auto under                                      = source;
  55         auto over                                       = destination;
  56         double float_distance           = static_cast<double>(distance.numerator()) / static_cast<double>(distance.denominator());
  57
  58         under.transform().image_transform.is_mix        = true;
  59         under.transform().image_transform.opacity       = 1 - float_distance;
  60         over.transform().image_transform.is_mix         = true;
  61         over.transform().image_transform.opacity        = float_distance;
  62
  63         return draw_frame::over(under, over);
  64 }
  65
  66 // Blends a moving window with a width of 1 frame duration.
  67 // * A distance of 0.0 gives 50% previous, 50% current and 0% next.
  68 // * A distance of 0.5 gives 25% previous, 50% current and 25% next.
  69 // * A distance of 0.75 gives 12.5% previous, 50% current and 37.5% next.
  70 // This is blurrier than blend, but gives a more even bluriness, instead of sharp, blurry, sharp, blurry.
  71 struct blend_all
  72 {
  73         draw_frame previous_frame       = draw_frame::empty();
  74         draw_frame last_source          = draw_frame::empty();
  75         draw_frame last_destination     = draw_frame::empty();
  76
  77         draw_frame operator()(const draw_frame& source, const draw_frame& destination, const boost::rational<int64_t>& distance)
  78         {
  79                 if (last_source != draw_frame::empty() && last_source != source)
  80                 {
  81                         if (last_destination == source)
  82                                 previous_frame = last_source;
  83                         else // A two frame jump
  84                                 previous_frame = last_destination;
  85                 }
  86
  87                 last_source                     = source;
  88                 last_destination        = destination;
  89
  90                 bool has_previous = previous_frame != draw_frame::empty();
  91
  92                 if (!has_previous)
  93                         return blend(source, destination, distance);
  94
  95                 auto middle                                                                                     = last_source;
  96                 auto next_frame                                                                         = destination;
  97                 previous_frame.transform().image_transform.is_mix       = true;
  98                 middle.transform().image_transform.is_mix                       = true;
  99                 next_frame.transform().image_transform.is_mix           = true;
 100
 101                 double float_distance                                                           = static_cast<double>(distance.numerator()) / static_cast<double>(distance.denominator());
 102                 previous_frame.transform().image_transform.opacity      = std::max(0.0, 0.5 - float_distance * 0.5);
 103                 middle.transform().image_transform.opacity                      = 0.5;
 104                 next_frame.transform().image_transform.opacity          = 1.0 - previous_frame.transform().image_transform.opacity - middle.transform().image_transform.opacity;
 105
 106                 std::vector<draw_frame> combination { previous_frame, middle, next_frame };
 107
 108                 return draw_frame(std::move(combination));
 109         }
 110 };
 111
 112 struct audio_extractor : public frame_visitor
 113 {
 114         std::function<void(const const_frame& frame)> on_frame_;
 115
 116         audio_extractor(std::function<void(const const_frame& frame)> on_frame)
 117                 : on_frame_(std::move(on_frame))
 118         {
 119         }
 120
 121         void push(const frame_transform& transform) override    { }
 122         void pop() override                                                                             { }
 123         void visit(const const_frame& frame) override
 124         {
 125                 if (!frame.audio_data().empty())
 126                         on_frame_(frame);
 127         }
 128 };
 129
 130 class framerate_producer : public frame_producer_base
 131 {
 132         spl::shared_ptr<frame_producer>                                         source_;
 133         boost::rational<int>                                                            source_framerate_;
 134         audio_channel_layout                                                            source_channel_layout_          = audio_channel_layout::invalid();
 135         boost::rational<int>                                                            destination_framerate_;
 136         field_mode                                                                                      destination_fieldmode_;
 137         std::vector<int>                                                                        destination_audio_cadence_;
 138         boost::rational<std::int64_t>                                           speed_;
 139         std::function<draw_frame (
 140                         const draw_frame& source,
 141                         const draw_frame& destination,
 142                         const boost::rational<int64_t>& distance)>      interpolator_                           = drop_and_skip;
 143
 144         boost::rational<std::int64_t>                                           current_frame_number_           = 0;
 145         draw_frame                                                                                      previous_frame_                         = draw_frame::empty();
 146         draw_frame                                                                                      next_frame_                                     = draw_frame::empty();
 147         mutable_audio_buffer                                                            audio_samples_;
 148
 149         unsigned int                                                                            output_repeat_                          = 0;
 150         unsigned int                                                                            output_frame_                           = 0;
 151 public:
 152         framerate_producer(
 153                         spl::shared_ptr<frame_producer> source,
 154                         boost::rational<int> source_framerate,
 155                         boost::rational<int> destination_framerate,
 156                         field_mode destination_fieldmode,
 157                         std::vector<int> destination_audio_cadence)
 158                 : source_(std::move(source))
 159                 , source_framerate_(std::move(source_framerate))
 160                 , destination_framerate_(std::move(destination_framerate))
 161                 , destination_fieldmode_(destination_fieldmode)
 162                 , destination_audio_cadence_(std::move(destination_audio_cadence))
 163         {
 164                 // Coarse adjustment to correct fps family (23.98 - 30 vs 47.95 - 60)
 165                 if (destination_fieldmode_ != field_mode::progressive)  // Interlaced output
 166                 {
 167                         auto diff_double        = boost::abs(source_framerate_ - destination_framerate_ * 2);
 168                         auto diff_keep          = boost::abs(source_framerate_ - destination_framerate_);
 169
 170                         if (diff_double < diff_keep)                                            // Double rate interlaced
 171                         {
 172                                 destination_framerate_ *= 2;
 173                         }
 174                         else                                                                                            // Progressive non interlaced
 175                         {
 176                                 destination_fieldmode_ = field_mode::progressive;
 177                         }
 178                 }
 179                 else                                                                                                    // Progressive
 180                 {
 181                         auto diff_halve = boost::abs(source_framerate_ * 2      - destination_framerate_);
 182                         auto diff_keep  = boost::abs(source_framerate_          - destination_framerate_);
 183
 184                         if (diff_halve < diff_keep)                                                     // Repeat every frame two times
 185                         {
 186                                 destination_framerate_  /= 2;
 187                                 output_repeat_                  = 2;
 188                         }
 189                 }
 190
 191                 speed_ = boost::rational<int64_t>(source_framerate_ / destination_framerate_);
 192
 193                 // drop_and_skip will only be used by default for exact framerate multiples (half, same and double)
 194                 // for all other framerates a frame interpolator will be chosen.
 195                 if (speed_ != 1 && speed_ * 2 != 1 && speed_ != 2)
 196                 {
 197                         if (source_framerate_ > 47)             // The bluriness of blend_all is acceptable on high framerates.
 198                                 interpolator_ = blend_all();
 199                         else                                                    // blend_all is mostly too blurry on low framerates. blend provides a compromise.
 200                                 interpolator_ = &blend;
 201
 202                         CASPAR_LOG(warning) << source_->print() << L" Frame blending frame rate conversion required to conform to channel frame rate.";
 203                 }
 204
 205                 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
 206                 // This cadence fills the audio mixer most optimally.
 207                 boost::range::rotate(destination_audio_cadence_, std::end(destination_audio_cadence_) - 1);
 208         }
 209
 210         draw_frame receive_impl() override
 211         {
 212                 if (destination_fieldmode_ == field_mode::progressive)
 213                 {
 214                         return do_render_progressive_frame(true);
 215                 }
 216                 else
 217                 {
 218                         auto field1 = do_render_progressive_frame(true);
 219                         auto field2 = do_render_progressive_frame(false);
 220
 221                         return draw_frame::interlace(field1, field2, destination_fieldmode_);
 222                 }
 223         }
 224
 225         std::future<std::wstring> call(const std::vector<std::wstring>& params) override
 226         {
 227                 if (!boost::iequals(params.at(0), L"framerate") || params.size() != 3)
 228                         return source_->call(params);
 229
 230                 if (boost::iequals(params.at(1), L"interpolation"))
 231                 {
 232                         if (boost::iequals(params.at(2), L"blend"))
 233                                 interpolator_ = &blend;
 234                         else if (boost::iequals(params.at(2), L"blend_all"))
 235                                 interpolator_ = blend_all();
 236                         else
 237                                 interpolator_ = &drop_and_skip;
 238                 }
 239                 else if (boost::iequals(params.at(1), L"output_repeat")) // Only for debugging purposes
 240                 {
 241                         output_repeat_ = boost::lexical_cast<unsigned int>(params.at(2));
 242                 }
 243
 244                 return make_ready_future<std::wstring>(L"");
 245         }
 246
 247         monitor::subject& monitor_output() override
 248         {
 249                 return source_->monitor_output();
 250         }
 251
 252         std::wstring print() const override
 253         {
 254                 return source_->print();
 255         }
 256
 257         std::wstring name() const override
 258         {
 259                 return source_->name();
 260         }
 261
 262         boost::property_tree::wptree info() const override
 263         {
 264                 return source_->info();
 265         }
 266
 267         constraints& pixel_constraints() override
 268         {
 269                 return source_->pixel_constraints();
 270         }
 271 private:
 272         draw_frame do_render_progressive_frame(bool sound)
 273         {
 274                 if (output_repeat_ && ++output_frame_ % output_repeat_)
 275                 {
 276                         auto frame = draw_frame::still(last_frame());
 277
 278                         frame.transform().audio_transform.volume = 0.0;
 279
 280                         return attach_sound(frame);
 281                 }
 282
 283                 if (previous_frame_ == draw_frame::empty())
 284                         previous_frame_ = pop_frame_from_source();
 285
 286                 auto current_frame_number       = current_frame_number_;
 287                 auto distance                           = current_frame_number_ - boost::rational_cast<int64_t>(current_frame_number_);
 288                 bool needs_next                         = distance > 0 || !enough_sound();
 289
 290                 if (needs_next && next_frame_ == draw_frame::empty())
 291                         next_frame_ = pop_frame_from_source();
 292
 293                 auto result = interpolator_(previous_frame_, next_frame_, distance);
 294
 295                 auto next_frame_number          = current_frame_number_ += get_speed();
 296                 auto integer_current_frame      = boost::rational_cast<std::int64_t>(current_frame_number);
 297                 auto integer_next_frame         = boost::rational_cast<std::int64_t>(next_frame_number);
 298
 299                 fast_forward_integer_frames(integer_next_frame - integer_current_frame);
 300
 301                 if (sound)
 302                         return attach_sound(result);
 303                 else
 304                         return result;
 305         }
 306
 307         void fast_forward_integer_frames(std::int64_t num_frames)
 308         {
 309                 if (num_frames == 0)
 310                         return;
 311
 312                 for (std::int64_t i = 0; i < num_frames; ++i)
 313                 {
 314                         previous_frame_ = std::move(next_frame_);
 315
 316                         next_frame_ = pop_frame_from_source();
 317                 }
 318         }
 319
 320         boost::rational<std::int64_t> get_speed() const
 321         {
 322                 return speed_;
 323         }
 324
 325         draw_frame pop_frame_from_source()
 326         {
 327                 auto frame = source_->receive();
 328
 329                 audio_extractor extractor([this](const const_frame& frame)
 330                 {
 331                         if (source_channel_layout_ != frame.audio_channel_layout())
 332                         {
 333                                 source_channel_layout_ = frame.audio_channel_layout();
 334
 335                                 // Insert silence samples so that the audio mixer is guaranteed to be filled.
 336                                 auto min_num_samples_per_frame  = *boost::min_element(destination_audio_cadence_);
 337                                 auto max_num_samples_per_frame  = *boost::max_element(destination_audio_cadence_);
 338                                 auto cadence_safety_samples             = max_num_samples_per_frame - min_num_samples_per_frame;
 339                                 audio_samples_.resize(source_channel_layout_.num_channels * cadence_safety_samples, 0);
 340                         }
 341
 342                         auto& buffer = frame.audio_data();
 343                         audio_samples_.insert(audio_samples_.end(), buffer.begin(), buffer.end());
 344                 });
 345
 346                 frame.accept(extractor);
 347                 frame.transform().audio_transform.volume = 0.0;
 348
 349                 return frame;
 350         }
 351
 352         draw_frame attach_sound(draw_frame frame)
 353         {
 354                 if (source_channel_layout_ == audio_channel_layout::invalid())
 355                         return frame;
 356
 357                 mutable_audio_buffer buffer;
 358
 359                 if (destination_audio_cadence_.front() * source_channel_layout_.num_channels == audio_samples_.size())
 360                 {
 361                         buffer.swap(audio_samples_);
 362                 }
 363                 else if (audio_samples_.size() >= destination_audio_cadence_.front() * source_channel_layout_.num_channels)
 364                 {
 365                         auto begin      = audio_samples_.begin();
 366                         auto end        = begin + destination_audio_cadence_.front() * source_channel_layout_.num_channels;
 367
 368                         buffer.insert(buffer.begin(), begin, end);
 369                         audio_samples_.erase(begin, end);
 370                 }
 371                 else
 372                 {
 373                         auto needed = destination_audio_cadence_.front();
 374                         auto got = audio_samples_.size() / source_channel_layout_.num_channels;
 375                         CASPAR_LOG(debug) << print() << L" Too few audio samples. Needed " << needed << L" but got " << got;
 376                         buffer.swap(audio_samples_);
 377                         buffer.resize(needed * source_channel_layout_.num_channels, 0);
 378                 }
 379
 380                 boost::range::rotate(destination_audio_cadence_, std::begin(destination_audio_cadence_) + 1);
 381
 382                 auto audio_frame = mutable_frame(
 383                                 {},
 384                                 std::move(buffer),
 385                                 this,
 386                                 pixel_format_desc(),
 387                                 source_channel_layout_);
 388                 return draw_frame::over(frame, draw_frame(std::move(audio_frame)));
 389         }
 390
 391         bool enough_sound() const
 392         {
 393                 return source_channel_layout_ == core::audio_channel_layout::invalid()
 394                                 || audio_samples_.size() / source_channel_layout_.num_channels >= destination_audio_cadence_.at(0);
 395         }
 396 };
 397
 398 spl::shared_ptr<frame_producer> create_framerate_producer(
 399                 spl::shared_ptr<frame_producer> source,
 400                 boost::rational<int> source_framerate,
 401                 boost::rational<int> destination_framerate,
 402                 field_mode destination_fieldmode,
 403                 std::vector<int> destination_audio_cadence)
 404 {
 405         return spl::make_shared<framerate_producer>(
 406                         std::move(source),
 407                         std::move(source_framerate),
 408                         std::move(destination_framerate),
 409                         destination_fieldmode,
 410                         std::move(destination_audio_cadence));
 411 }
 412
 413 }}
 414