git.sesse.net Git - casparcg/blob - modules/ffmpeg/producer/muxer/frame_muxer.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Robert Nagy, ronag89@gmail.com
  20 */
  21
  22 #include "../../StdAfx.h"
  23
  24 #include "frame_muxer.h"
  25
  26 #include "../filter/filter.h"
  27 #include "../filter/audio_filter.h"
  28 #include "../util/util.h"
  29 #include "../../ffmpeg.h"
  30
  31 #include <core/producer/frame_producer.h>
  32 #include <core/frame/draw_frame.h>
  33 #include <core/frame/frame_transform.h>
  34 #include <core/frame/pixel_format.h>
  35 #include <core/frame/frame_factory.h>
  36 #include <core/frame/frame.h>
  37 #include <core/frame/audio_channel_layout.h>
  38
  39 #include <common/env.h>
  40 #include <common/except.h>
  41 #include <common/log.h>
  42
  43 #if defined(_MSC_VER)
  44 #pragma warning (push)
  45 #pragma warning (disable : 4244)
  46 #endif
  47 extern "C"
  48 {
  49         #define __STDC_CONSTANT_MACROS
  50         #define __STDC_LIMIT_MACROS
  51         #include <libavcodec/avcodec.h>
  52         #include <libavformat/avformat.h>
  53 }
  54 #if defined(_MSC_VER)
  55 #pragma warning (pop)
  56 #endif
  57
  58 #include <common/assert.h>
  59 #include <boost/range/algorithm_ext/push_back.hpp>
  60 #include <boost/algorithm/string/predicate.hpp>
  61 #include <boost/thread/mutex.hpp>
  62 #include <boost/optional.hpp>
  63
  64 #include <deque>
  65 #include <queue>
  66 #include <vector>
  67
  68 using namespace caspar::core;
  69
  70 namespace caspar { namespace ffmpeg {
  71
  72 struct av_frame_format
  73 {
  74         int                                                                             pix_format;
  75         std::array<int, AV_NUM_DATA_POINTERS>   line_sizes;
  76         int                                                                             width;
  77         int                                                                             height;
  78
  79         av_frame_format(const AVFrame& frame)
  80                 : pix_format(frame.format)
  81                 , width(frame.width)
  82                 , height(frame.height)
  83         {
  84                 boost::copy(frame.linesize, line_sizes.begin());
  85         }
  86
  87         bool operator==(const av_frame_format& other) const
  88         {
  89                 return pix_format == other.pix_format
  90                         && line_sizes == other.line_sizes
  91                         && width == other.width
  92                         && height == other.height;
  93         }
  94
  95         bool operator!=(const av_frame_format& other) const
  96         {
  97                 return !(*this == other);
  98         }
  99 };
 100
 101 std::unique_ptr<audio_filter> create_amerge_filter(std::vector<audio_input_pad> input_pads, const core::audio_channel_layout& layout)
 102 {
 103         std::vector<audio_output_pad> output_pads;
 104         std::wstring amerge;
 105
 106         output_pads.emplace_back(
 107                         std::vector<int>                        { 48000 },
 108                         std::vector<AVSampleFormat>     { AVSampleFormat::AV_SAMPLE_FMT_S32 },
 109                         std::vector<uint64_t>           { static_cast<uint64_t>(av_get_default_channel_layout(layout.num_channels)) });
 110
 111         if (input_pads.size() > 1)
 112         {
 113                 for (int i = 0; i < input_pads.size(); ++i)
 114                         amerge += L"[a:" + boost::lexical_cast<std::wstring>(i) + L"]";
 115
 116                 amerge += L"amerge=inputs=" + boost::lexical_cast<std::wstring>(input_pads.size());
 117         }
 118
 119         std::wstring afilter;
 120
 121         if (!amerge.empty())
 122         {
 123                 afilter = amerge;
 124                 afilter += L"[aout:0]";
 125         }
 126
 127         return std::unique_ptr<audio_filter>(new audio_filter(input_pads, output_pads, u8(afilter)));
 128 }
 129
 130 struct frame_muxer::impl : boost::noncopyable
 131 {
 132         static constexpr std::size_t                                    max_stream_size                         = 120;
 133         std::queue<std::queue<core::mutable_frame>>             video_streams_;
 134         std::queue<core::mutable_audio_buffer>                  audio_streams_;
 135         std::queue<core::draw_frame>                                    frame_buffer_;
 136         display_mode                                                                    display_mode_                           = display_mode::invalid;
 137         const boost::rational<int>                                              in_framerate_;
 138         const video_format_desc                                                 format_desc_;
 139         const audio_channel_layout                                              audio_channel_layout_;
 140
 141         std::vector<int>                                                                audio_cadence_                          = format_desc_.audio_cadence;
 142
 143         spl::shared_ptr<core::frame_factory>                    frame_factory_;
 144         boost::optional<av_frame_format>                                previously_filtered_frame_;
 145
 146         std::unique_ptr<filter>                                                 filter_;
 147         const std::wstring                                                              filter_str_;
 148         std::unique_ptr<audio_filter>                                   audio_filter_;
 149         const bool                                                                              multithreaded_filter_;
 150         bool                                                                                    force_deinterlacing_            = env::properties().get(L"configuration.force-deinterlace", false);
 151
 152         mutable boost::mutex                                                    out_framerate_mutex_;
 153         boost::rational<int>                                                    out_framerate_;
 154
 155         impl(
 156                         boost::rational<int> in_framerate,
 157                         std::vector<audio_input_pad> audio_input_pads,
 158                         const spl::shared_ptr<core::frame_factory>& frame_factory,
 159                         const core::video_format_desc& format_desc,
 160                         const core::audio_channel_layout& channel_layout,
 161                         const std::wstring& filter_str,
 162                         bool multithreaded_filter)
 163                 : in_framerate_(in_framerate)
 164                 , format_desc_(format_desc)
 165                 , audio_channel_layout_(channel_layout)
 166                 , frame_factory_(frame_factory)
 167                 , filter_str_(filter_str)
 168                 , multithreaded_filter_(multithreaded_filter)
 169         {
 170                 video_streams_.push(std::queue<core::mutable_frame>());
 171                 audio_streams_.push(core::mutable_audio_buffer());
 172
 173                 set_out_framerate(in_framerate_);
 174
 175                 if (!audio_input_pads.empty())
 176                 {
 177                         audio_filter_ = create_amerge_filter(std::move(audio_input_pads), audio_channel_layout_);
 178                 }
 179         }
 180
 181         void push(const std::shared_ptr<AVFrame>& video_frame)
 182         {
 183                 if (!video_frame)
 184                         return;
 185
 186                 av_frame_format current_frame_format(*video_frame);
 187
 188                 if (previously_filtered_frame_ && video_frame->data[0] && *previously_filtered_frame_ != current_frame_format)
 189                 {
 190                         // Fixes bug where avfilter crashes server on some DV files (starts in YUV420p but changes to YUV411p after the first frame).
 191                         if (ffmpeg::is_logging_quiet_for_thread())
 192                                 CASPAR_LOG(debug) << L"[frame_muxer] Frame format has changed. Resetting display mode.";
 193                         else
 194                                 CASPAR_LOG(info) << L"[frame_muxer] Frame format has changed. Resetting display mode.";
 195
 196                         display_mode_ = display_mode::invalid;
 197                         filter_.reset();
 198                         previously_filtered_frame_ = boost::none;
 199                 }
 200
 201                 if (video_frame == flush_video())
 202                 {
 203                         video_streams_.push(std::queue<core::mutable_frame>());
 204                 }
 205                 else if (video_frame == empty_video())
 206                 {
 207                         video_streams_.back().push(frame_factory_->create_frame(this, core::pixel_format::invalid, audio_channel_layout_));
 208                         display_mode_ = display_mode::simple;
 209                 }
 210                 else
 211                 {
 212                         if (!filter_ || display_mode_ == display_mode::invalid)
 213                                 update_display_mode(video_frame);
 214
 215                         if (filter_)
 216                         {
 217                                 filter_->push(video_frame);
 218                                 previously_filtered_frame_ = current_frame_format;
 219
 220                                 for (auto& av_frame : filter_->poll_all())
 221                                         video_streams_.back().push(make_frame(this, av_frame, *frame_factory_, audio_channel_layout_));
 222                         }
 223                 }
 224
 225                 if (video_streams_.back().size() > max_stream_size)
 226                         CASPAR_THROW_EXCEPTION(invalid_operation() << source_info("frame_muxer") << msg_info("video-stream overflow. This can be caused by incorrect frame-rate. Check clip meta-data."));
 227         }
 228
 229         void push(const std::vector<std::shared_ptr<core::mutable_audio_buffer>>& audio_samples_per_stream)
 230         {
 231                 if (audio_samples_per_stream.empty())
 232                         return;
 233
 234                 bool is_flush = boost::count_if(
 235                                 audio_samples_per_stream,
 236                                 [](std::shared_ptr<core::mutable_audio_buffer> a) { return a == flush_audio(); }) > 0;
 237
 238                 if (is_flush)
 239                 {
 240                         audio_streams_.push(core::mutable_audio_buffer());
 241                 }
 242                 else if (audio_samples_per_stream.at(0) == empty_audio())
 243                 {
 244                         boost::range::push_back(audio_streams_.back(), core::mutable_audio_buffer(audio_cadence_.front() * audio_channel_layout_.num_channels, 0));
 245                 }
 246                 else
 247                 {
 248                         for (int i = 0; i < audio_samples_per_stream.size(); ++i)
 249                         {
 250                                 auto range = boost::make_iterator_range_n(
 251                                                 audio_samples_per_stream.at(i)->data(),
 252                                                 audio_samples_per_stream.at(i)->size());
 253
 254                                 audio_filter_->push(i, range);
 255                         }
 256
 257                         for (auto frame : audio_filter_->poll_all(0))
 258                         {
 259                                 auto audio = boost::make_iterator_range_n(
 260                                                 reinterpret_cast<std::int32_t*>(frame->extended_data[0]),
 261                                                 frame->nb_samples * frame->channels);
 262
 263                                 boost::range::push_back(audio_streams_.back(), audio);
 264                         }
 265                 }
 266
 267                 if (audio_streams_.back().size() > max_stream_size * audio_cadence_.front() * audio_channel_layout_.num_channels)
 268                         CASPAR_THROW_EXCEPTION(invalid_operation() << source_info("frame_muxer") << msg_info("audio-stream overflow. This can be caused by incorrect frame-rate. Check clip meta-data."));
 269         }
 270
 271         bool video_ready() const
 272         {
 273                 return video_streams_.size() > 1 || (video_streams_.size() >= audio_streams_.size() && video_ready2());
 274         }
 275
 276         bool audio_ready() const
 277         {
 278                 return audio_streams_.size() > 1 || (audio_streams_.size() >= video_streams_.size() && audio_ready2());
 279         }
 280
 281         bool video_ready2() const
 282         {
 283                 return video_streams_.front().size() >= 1;
 284         }
 285
 286         bool audio_ready2() const
 287         {
 288                 return audio_streams_.front().size() >= audio_cadence_.front() * audio_channel_layout_.num_channels;
 289         }
 290
 291         core::draw_frame poll()
 292         {
 293                 if (!frame_buffer_.empty())
 294                 {
 295                         auto frame = frame_buffer_.front();
 296                         frame_buffer_.pop();
 297                         return frame;
 298                 }
 299
 300                 if (video_streams_.size() > 1 && audio_streams_.size() > 1 && (!video_ready2() || !audio_ready2()))
 301                 {
 302                         if (!video_streams_.front().empty() || !audio_streams_.front().empty())
 303                                 CASPAR_LOG(trace) << "Truncating: " << video_streams_.front().size() << L" video-frames, " << audio_streams_.front().size() << L" audio-samples.";
 304
 305                         video_streams_.pop();
 306                         audio_streams_.pop();
 307                 }
 308
 309                 if (!video_ready2() || !audio_ready2() || display_mode_ == display_mode::invalid)
 310                         return core::draw_frame::empty();
 311
 312                 auto frame                      = pop_video();
 313                 frame.audio_data()      = pop_audio();
 314
 315                 frame_buffer_.push(core::draw_frame(std::move(frame)));
 316
 317                 return poll();
 318         }
 319
 320         core::mutable_frame pop_video()
 321         {
 322                 auto frame = std::move(video_streams_.front().front());
 323                 video_streams_.front().pop();
 324                 return frame;
 325         }
 326
 327         core::mutable_audio_buffer pop_audio()
 328         {
 329                 CASPAR_VERIFY(audio_streams_.front().size() >= audio_cadence_.front() * audio_channel_layout_.num_channels);
 330
 331                 auto begin      = audio_streams_.front().begin();
 332                 auto end        = begin + (audio_cadence_.front() * audio_channel_layout_.num_channels);
 333
 334                 core::mutable_audio_buffer samples(begin, end);
 335                 audio_streams_.front().erase(begin, end);
 336
 337                 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_) + 1);
 338
 339                 return samples;
 340         }
 341
 342         uint32_t calc_nb_frames(uint32_t nb_frames) const
 343         {
 344                 uint64_t nb_frames2 = nb_frames;
 345
 346                 if(filter_ && filter_->is_double_rate()) // Take into account transformations in filter.
 347                         nb_frames2 *= 2;
 348
 349                 return static_cast<uint32_t>(nb_frames2);
 350         }
 351
 352         boost::rational<int> out_framerate() const
 353         {
 354                 boost::lock_guard<boost::mutex> lock(out_framerate_mutex_);
 355
 356                 return out_framerate_;
 357         }
 358 private:
 359         void update_display_mode(const std::shared_ptr<AVFrame>& frame)
 360         {
 361                 std::wstring filter_str = filter_str_;
 362
 363                 display_mode_ = display_mode::simple;
 364
 365                 auto mode = get_mode(*frame);
 366
 367                 if (filter::is_deinterlacing(filter_str_))
 368                 {
 369                         display_mode_ = display_mode::simple;
 370                 }
 371                 else if (mode != core::field_mode::progressive)
 372                 {
 373                         if (force_deinterlacing_)
 374                         {
 375                                 display_mode_ = display_mode::deinterlace_bob;
 376                         }
 377                         else
 378                         {
 379                                 bool output_also_interlaced = format_desc_.field_mode != core::field_mode::progressive;
 380                                 bool interlaced_output_compatible =
 381                                                 output_also_interlaced
 382                                                 && (
 383                                                                 (frame->height == 480 && format_desc_.height == 486) // don't deinterlace for NTSC DV
 384                                                                 || frame->height == format_desc_.height
 385                                                 )
 386                                                 && in_framerate_ == format_desc_.framerate;
 387
 388                                 display_mode_ = interlaced_output_compatible ? display_mode::simple : display_mode::deinterlace_bob;
 389                         }
 390                 }
 391
 392                 if (display_mode_ == display_mode::deinterlace_bob)
 393                         filter_str = append_filter(filter_str, L"YADIF=1:-1");
 394
 395                 auto out_framerate = in_framerate_;
 396
 397                 if (filter::is_double_rate(filter_str))
 398                         out_framerate *= 2;
 399
 400                 if (frame->height == 480) // NTSC DV
 401                 {
 402                         auto pad_str = L"PAD=" + boost::lexical_cast<std::wstring>(frame->width) + L":486:0:2:black";
 403                         filter_str = append_filter(filter_str, pad_str);
 404                 }
 405
 406                 filter_.reset (new filter(
 407                                 frame->width,
 408                                 frame->height,
 409                                 1 / in_framerate_,
 410                                 in_framerate_,
 411                                 boost::rational<int>(frame->sample_aspect_ratio.num, frame->sample_aspect_ratio.den),
 412                                 static_cast<AVPixelFormat>(frame->format),
 413                                 std::vector<AVPixelFormat>(),
 414                                 u8(filter_str)));
 415
 416                 set_out_framerate(out_framerate);
 417
 418                 auto in_fps = static_cast<double>(in_framerate_.numerator()) / static_cast<double>(in_framerate_.denominator());
 419
 420                 if (ffmpeg::is_logging_quiet_for_thread())
 421                         CASPAR_LOG(debug) << L"[frame_muxer] " << display_mode_ << L" " << print_mode(frame->width, frame->height, in_fps, frame->interlaced_frame > 0);
 422                 else
 423                         CASPAR_LOG(info) << L"[frame_muxer] " << display_mode_ << L" " << print_mode(frame->width, frame->height, in_fps, frame->interlaced_frame > 0);
 424         }
 425
 426         void merge()
 427         {
 428                 while (video_ready() && audio_ready() && display_mode_ != display_mode::invalid)
 429                 {
 430                         auto frame1 = pop_video();
 431                         frame1.audio_data() = pop_audio();
 432
 433                         frame_buffer_.push(core::draw_frame(std::move(frame1)));
 434                 }
 435         }
 436
 437         void set_out_framerate(boost::rational<int> out_framerate)
 438         {
 439                 boost::lock_guard<boost::mutex> lock(out_framerate_mutex_);
 440
 441                 bool changed = out_framerate != out_framerate_;
 442                 out_framerate_ = std::move(out_framerate);
 443
 444                 if (changed)
 445                         update_audio_cadence();
 446         }
 447
 448         void update_audio_cadence()
 449         {
 450                 audio_cadence_ = find_audio_cadence(out_framerate_);
 451
 452                 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
 453                 // This cadence fills the audio mixer most optimally.
 454                 boost::range::rotate(audio_cadence_, std::end(audio_cadence_) - 1);
 455         }
 456 };
 457
 458 frame_muxer::frame_muxer(
 459                 boost::rational<int> in_framerate,
 460                 std::vector<audio_input_pad> audio_input_pads,
 461                 const spl::shared_ptr<core::frame_factory>& frame_factory,
 462                 const core::video_format_desc& format_desc,
 463                 const core::audio_channel_layout& channel_layout,
 464                 const std::wstring& filter,
 465                 bool multithreaded_filter)
 466         : impl_(new impl(std::move(in_framerate), std::move(audio_input_pads), frame_factory, format_desc, channel_layout, filter, multithreaded_filter)){}
 467 void frame_muxer::push(const std::shared_ptr<AVFrame>& video){impl_->push(video);}
 468 void frame_muxer::push(const std::vector<std::shared_ptr<core::mutable_audio_buffer>>& audio_samples_per_stream){impl_->push(audio_samples_per_stream);}
 469 core::draw_frame frame_muxer::poll(){return impl_->poll();}
 470 uint32_t frame_muxer::calc_nb_frames(uint32_t nb_frames) const {return impl_->calc_nb_frames(nb_frames);}
 471 bool frame_muxer::video_ready() const{return impl_->video_ready();}
 472 bool frame_muxer::audio_ready() const{return impl_->audio_ready();}
 473 boost::rational<int> frame_muxer::out_framerate() const { return impl_->out_framerate(); }
 474 }}