git.sesse.net Git - casparcg/blob - modules/ffmpeg/producer/muxer/frame_muxer.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Robert Nagy, ronag89@gmail.com
  20 */
  21
  22 #include "../../StdAfx.h"
  23
  24 #include "frame_muxer.h"
  25
  26 #include "../filter/filter.h"
  27 #include "../util/util.h"
  28 #include "../../ffmpeg.h"
  29
  30 #include <core/producer/frame_producer.h>
  31 #include <core/frame/draw_frame.h>
  32 #include <core/frame/frame_transform.h>
  33 #include <core/frame/pixel_format.h>
  34 #include <core/frame/frame_factory.h>
  35 #include <core/frame/frame.h>
  36 #include <core/frame/audio_channel_layout.h>
  37
  38 #include <common/env.h>
  39 #include <common/except.h>
  40 #include <common/log.h>
  41
  42 #if defined(_MSC_VER)
  43 #pragma warning (push)
  44 #pragma warning (disable : 4244)
  45 #endif
  46 extern "C"
  47 {
  48         #define __STDC_CONSTANT_MACROS
  49         #define __STDC_LIMIT_MACROS
  50         #include <libavcodec/avcodec.h>
  51         #include <libavformat/avformat.h>
  52 }
  53 #if defined(_MSC_VER)
  54 #pragma warning (pop)
  55 #endif
  56
  57 #include <common/assert.h>
  58 #include <boost/range/algorithm_ext/push_back.hpp>
  59 #include <boost/algorithm/string/predicate.hpp>
  60 #include <boost/thread/mutex.hpp>
  61 #include <boost/optional.hpp>
  62
  63 #include <deque>
  64 #include <queue>
  65 #include <vector>
  66
  67 using namespace caspar::core;
  68
  69 namespace caspar { namespace ffmpeg {
  70 struct av_frame_format
  71 {
  72         int                                                                             pix_format;
  73         std::array<int, AV_NUM_DATA_POINTERS>   line_sizes;
  74         int                                                                             width;
  75         int                                                                             height;
  76
  77         av_frame_format(const AVFrame& frame)
  78                 : pix_format(frame.format)
  79                 , width(frame.width)
  80                 , height(frame.height)
  81         {
  82                 boost::copy(frame.linesize, line_sizes.begin());
  83         }
  84
  85         bool operator==(const av_frame_format& other) const
  86         {
  87                 return pix_format == other.pix_format
  88                         && line_sizes == other.line_sizes
  89                         && width == other.width
  90                         && height == other.height;
  91         }
  92
  93         bool operator!=(const av_frame_format& other) const
  94         {
  95                 return !(*this == other);
  96         }
  97 };
  98
  99 struct frame_muxer::impl : boost::noncopyable
 100 {
 101         std::queue<std::queue<core::mutable_frame>>             video_streams_;
 102         std::queue<core::mutable_audio_buffer>                  audio_streams_;
 103         std::queue<core::draw_frame>                                    frame_buffer_;
 104         display_mode                                                                    display_mode_                   = display_mode::invalid;
 105         const boost::rational<int>                                              in_framerate_;
 106         const video_format_desc                                                 format_desc_;
 107         const audio_channel_layout                                              audio_channel_layout_;
 108
 109         std::vector<int>                                                                audio_cadence_                  = format_desc_.audio_cadence;
 110
 111         spl::shared_ptr<core::frame_factory>                    frame_factory_;
 112         boost::optional<av_frame_format>                                previously_filtered_frame_;
 113
 114         std::unique_ptr<filter>                                                 filter_;
 115         const std::wstring                                                              filter_str_;
 116         const bool                                                                              multithreaded_filter_;
 117         bool                                                                                    force_deinterlacing_    = env::properties().get(L"configuration.force-deinterlace", false);
 118
 119         mutable boost::mutex                                                    out_framerate_mutex_;
 120         boost::rational<int>                                                    out_framerate_;
 121
 122         impl(
 123                         boost::rational<int> in_framerate,
 124                         const spl::shared_ptr<core::frame_factory>& frame_factory,
 125                         const core::video_format_desc& format_desc,
 126                         const core::audio_channel_layout& channel_layout,
 127                         const std::wstring& filter_str,
 128                         bool multithreaded_filter)
 129                 : in_framerate_(in_framerate)
 130                 , format_desc_(format_desc)
 131                 , audio_channel_layout_(channel_layout)
 132                 , frame_factory_(frame_factory)
 133                 , filter_str_(filter_str)
 134                 , multithreaded_filter_(multithreaded_filter)
 135         {
 136                 video_streams_.push(std::queue<core::mutable_frame>());
 137                 audio_streams_.push(core::mutable_audio_buffer());
 138
 139                 set_out_framerate(in_framerate_);
 140         }
 141
 142         void push(const std::shared_ptr<AVFrame>& video_frame)
 143         {
 144                 if (!video_frame)
 145                         return;
 146
 147                 av_frame_format current_frame_format(*video_frame);
 148
 149                 if (previously_filtered_frame_ && video_frame->data[0] && *previously_filtered_frame_ != current_frame_format)
 150                 {
 151                         // Fixes bug where avfilter crashes server on some DV files (starts in YUV420p but changes to YUV411p after the first frame).
 152                         if (ffmpeg::is_logging_quiet_for_thread())
 153                                 CASPAR_LOG(debug) << L"[frame_muxer] Frame format has changed. Resetting display mode.";
 154                         else
 155                                 CASPAR_LOG(info) << L"[frame_muxer] Frame format has changed. Resetting display mode.";
 156
 157                         display_mode_ = display_mode::invalid;
 158                         filter_.reset();
 159                         previously_filtered_frame_ = boost::none;
 160                 }
 161
 162                 if (video_frame == flush_video())
 163                 {
 164                         video_streams_.push(std::queue<core::mutable_frame>());
 165                 }
 166                 else if (video_frame == empty_video())
 167                 {
 168                         video_streams_.back().push(frame_factory_->create_frame(this, core::pixel_format::invalid, audio_channel_layout_));
 169                         display_mode_ = display_mode::simple;
 170                 }
 171                 else
 172                 {
 173                         if (!filter_ || display_mode_ == display_mode::invalid)
 174                                 update_display_mode(video_frame);
 175
 176                         if (filter_)
 177                         {
 178                                 filter_->push(video_frame);
 179                                 previously_filtered_frame_ = current_frame_format;
 180
 181                                 for (auto& av_frame : filter_->poll_all())
 182                                         video_streams_.back().push(make_frame(this, av_frame, *frame_factory_, audio_channel_layout_));
 183                         }
 184                 }
 185
 186                 if (video_streams_.back().size() > 32)
 187                         CASPAR_THROW_EXCEPTION(invalid_operation() << source_info("frame_muxer") << msg_info("video-stream overflow. This can be caused by incorrect frame-rate. Check clip meta-data."));
 188         }
 189
 190         void push(const std::shared_ptr<core::mutable_audio_buffer>& audio)
 191         {
 192                 if (!audio)
 193                         return;
 194
 195                 if (audio == flush_audio())
 196                 {
 197                         audio_streams_.push(core::mutable_audio_buffer());
 198                 }
 199                 else if (audio == empty_audio())
 200                 {
 201                         boost::range::push_back(audio_streams_.back(), core::mutable_audio_buffer(audio_cadence_.front() * audio_channel_layout_.num_channels, 0));
 202                 }
 203                 else
 204                 {
 205                         boost::range::push_back(audio_streams_.back(), *audio);
 206                 }
 207
 208                 if (audio_streams_.back().size() > 32 * audio_cadence_.front() * audio_channel_layout_.num_channels)
 209                         BOOST_THROW_EXCEPTION(invalid_operation() << source_info("frame_muxer") << msg_info("audio-stream overflow. This can be caused by incorrect frame-rate. Check clip meta-data."));
 210         }
 211
 212         bool video_ready() const
 213         {
 214                 return video_streams_.size() > 1 || (video_streams_.size() >= audio_streams_.size() && video_ready2());
 215         }
 216
 217         bool audio_ready() const
 218         {
 219                 return audio_streams_.size() > 1 || (audio_streams_.size() >= video_streams_.size() && audio_ready2());
 220         }
 221
 222         bool video_ready2() const
 223         {
 224                 return video_streams_.front().size() >= 1;
 225         }
 226
 227         bool audio_ready2() const
 228         {
 229                 return audio_streams_.front().size() >= audio_cadence_.front() * audio_channel_layout_.num_channels;
 230         }
 231
 232         core::draw_frame poll()
 233         {
 234                 if (!frame_buffer_.empty())
 235                 {
 236                         auto frame = frame_buffer_.front();
 237                         frame_buffer_.pop();
 238                         return frame;
 239                 }
 240
 241                 if (video_streams_.size() > 1 && audio_streams_.size() > 1 && (!video_ready2() || !audio_ready2()))
 242                 {
 243                         if (!video_streams_.front().empty() || !audio_streams_.front().empty())
 244                                 CASPAR_LOG(trace) << "Truncating: " << video_streams_.front().size() << L" video-frames, " << audio_streams_.front().size() << L" audio-samples.";
 245
 246                         video_streams_.pop();
 247                         audio_streams_.pop();
 248                 }
 249
 250                 if (!video_ready2() || !audio_ready2() || display_mode_ == display_mode::invalid)
 251                         return core::draw_frame::empty();
 252
 253                 auto frame                      = pop_video();
 254                 frame.audio_data()      = pop_audio();
 255
 256                 frame_buffer_.push(core::draw_frame(std::move(frame)));
 257
 258                 return frame_buffer_.empty() ? core::draw_frame::empty() : poll();
 259         }
 260
 261         core::mutable_frame pop_video()
 262         {
 263                 auto frame = std::move(video_streams_.front().front());
 264                 video_streams_.front().pop();
 265                 return frame;
 266         }
 267
 268         core::mutable_audio_buffer pop_audio()
 269         {
 270                 CASPAR_VERIFY(audio_streams_.front().size() >= audio_cadence_.front() * audio_channel_layout_.num_channels);
 271
 272                 auto begin      = audio_streams_.front().begin();
 273                 auto end        = begin + (audio_cadence_.front() * audio_channel_layout_.num_channels);
 274
 275                 core::mutable_audio_buffer samples(begin, end);
 276                 audio_streams_.front().erase(begin, end);
 277
 278                 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_) + 1);
 279
 280                 return samples;
 281         }
 282
 283         uint32_t calc_nb_frames(uint32_t nb_frames) const
 284         {
 285                 uint64_t nb_frames2 = nb_frames;
 286
 287                 if(filter_ && filter_->is_double_rate()) // Take into account transformations in filter.
 288                         nb_frames2 *= 2;
 289
 290                 return static_cast<uint32_t>(nb_frames2);
 291         }
 292
 293         boost::rational<int> out_framerate() const
 294         {
 295                 boost::lock_guard<boost::mutex> lock(out_framerate_mutex_);
 296
 297                 return out_framerate_;
 298         }
 299 private:
 300         void update_display_mode(const std::shared_ptr<AVFrame>& frame)
 301         {
 302                 std::wstring filter_str = filter_str_;
 303
 304                 display_mode_ = display_mode::simple;
 305
 306                 auto mode = get_mode(*frame);
 307                 if (mode == core::field_mode::progressive && frame->height < 720 && in_framerate_ < 50) // SD frames are interlaced. Probably incorrect meta-data. Fix it.
 308                         mode = core::field_mode::upper;
 309
 310                 if (filter::is_deinterlacing(filter_str_))
 311                 {
 312                         display_mode_ = display_mode::simple;
 313                 }
 314                 else if (mode != core::field_mode::progressive)
 315                 {
 316                         if (force_deinterlacing_)
 317                         {
 318                                 display_mode_ = display_mode::deinterlace_bob;
 319                         }
 320                         else
 321                         {
 322                                 bool output_also_interlaced = format_desc_.field_mode != core::field_mode::progressive;
 323                                 bool interlaced_output_compatible =
 324                                                 output_also_interlaced
 325                                                 && (
 326                                                                 (frame->height == 480 && format_desc_.height == 486) // don't deinterlace for NTSC DV
 327                                                                 || frame->height == format_desc_.height
 328                                                 )
 329                                                 && in_framerate_ == format_desc_.framerate;
 330
 331                                 display_mode_ = interlaced_output_compatible ? display_mode::simple : display_mode::deinterlace_bob;
 332                         }
 333                 }
 334
 335                 if (display_mode_ == display_mode::deinterlace_bob)
 336                         filter_str = append_filter(filter_str, L"YADIF=1:-1");
 337
 338                 auto out_framerate = in_framerate_;
 339
 340                 if (filter::is_double_rate(filter_str))
 341                         out_framerate *= 2;
 342
 343                 if (frame->height == 480) // NTSC DV
 344                 {
 345                         auto pad_str = L"PAD=" + boost::lexical_cast<std::wstring>(frame->width) + L":486:0:2:black";
 346                         filter_str = append_filter(filter_str, pad_str);
 347                 }
 348
 349                 filter_.reset (new filter(
 350                                 frame->width,
 351                                 frame->height,
 352                                 1 / in_framerate_,
 353                                 in_framerate_,
 354                                 boost::rational<int>(frame->sample_aspect_ratio.num, frame->sample_aspect_ratio.den),
 355                                 static_cast<AVPixelFormat>(frame->format),
 356                                 std::vector<AVPixelFormat>(),
 357                                 u8(filter_str)));
 358
 359                 set_out_framerate(out_framerate);
 360
 361                 auto in_fps = static_cast<double>(in_framerate_.numerator()) / static_cast<double>(in_framerate_.denominator());
 362
 363                 if (ffmpeg::is_logging_quiet_for_thread())
 364                         CASPAR_LOG(debug) << L"[frame_muxer] " << display_mode_ << L" " << print_mode(frame->width, frame->height, in_fps, frame->interlaced_frame > 0);
 365                 else
 366                         CASPAR_LOG(info) << L"[frame_muxer] " << display_mode_ << L" " << print_mode(frame->width, frame->height, in_fps, frame->interlaced_frame > 0);
 367         }
 368
 369         void merge()
 370         {
 371                 while (video_ready() && audio_ready() && display_mode_ != display_mode::invalid)
 372                 {
 373                         auto frame1 = pop_video();
 374                         frame1.audio_data() = pop_audio();
 375
 376                         frame_buffer_.push(core::draw_frame(std::move(frame1)));
 377                 }
 378         }
 379
 380         void set_out_framerate(boost::rational<int> out_framerate)
 381         {
 382                 boost::lock_guard<boost::mutex> lock(out_framerate_mutex_);
 383
 384                 bool changed = out_framerate != out_framerate_;
 385                 out_framerate_ = std::move(out_framerate);
 386
 387                 if (changed)
 388                         update_audio_cadence();
 389         }
 390
 391         void update_audio_cadence()
 392         {
 393                 audio_cadence_ = find_audio_cadence(out_framerate_);
 394
 395                 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
 396                 // This cadence fills the audio mixer most optimally.
 397                 boost::range::rotate(audio_cadence_, std::end(audio_cadence_) - 1);
 398         }
 399 };
 400
 401 frame_muxer::frame_muxer(
 402                 boost::rational<int> in_framerate,
 403                 const spl::shared_ptr<core::frame_factory>& frame_factory,
 404                 const core::video_format_desc& format_desc,
 405                 const core::audio_channel_layout& channel_layout,
 406                 const std::wstring& filter,
 407                 bool multithreaded_filter)
 408         : impl_(new impl(in_framerate, frame_factory, format_desc, channel_layout, filter, multithreaded_filter)){}
 409 void frame_muxer::push(const std::shared_ptr<AVFrame>& video){impl_->push(video);}
 410 void frame_muxer::push(const std::shared_ptr<core::mutable_audio_buffer>& audio){impl_->push(audio);}
 411 core::draw_frame frame_muxer::poll(){return impl_->poll();}
 412 uint32_t frame_muxer::calc_nb_frames(uint32_t nb_frames) const {return impl_->calc_nb_frames(nb_frames);}
 413 bool frame_muxer::video_ready() const{return impl_->video_ready();}
 414 bool frame_muxer::audio_ready() const{return impl_->audio_ready();}
 415 boost::rational<int> frame_muxer::out_framerate() const { return impl_->out_framerate(); }
 416 }}