git.sesse.net Git - casparcg/blob - modules/ffmpeg/producer/muxer/frame_muxer.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Robert Nagy, ronag89@gmail.com
  20 */
  21
  22 #include "../../StdAfx.h"
  23
  24 #include "frame_muxer.h"
  25
  26 #include "../filter/filter.h"
  27 #include "../util/util.h"
  28 #include "../../ffmpeg.h"
  29
  30 #include <core/producer/frame_producer.h>
  31 #include <core/frame/draw_frame.h>
  32 #include <core/frame/frame_transform.h>
  33 #include <core/frame/pixel_format.h>
  34 #include <core/frame/frame_factory.h>
  35 #include <core/frame/frame.h>
  36 #include <core/frame/audio_channel_layout.h>
  37
  38 #include <common/env.h>
  39 #include <common/except.h>
  40 #include <common/log.h>
  41
  42 #if defined(_MSC_VER)
  43 #pragma warning (push)
  44 #pragma warning (disable : 4244)
  45 #endif
  46 extern "C"
  47 {
  48         #define __STDC_CONSTANT_MACROS
  49         #define __STDC_LIMIT_MACROS
  50         #include <libavcodec/avcodec.h>
  51         #include <libavformat/avformat.h>
  52 }
  53 #if defined(_MSC_VER)
  54 #pragma warning (pop)
  55 #endif
  56
  57 #include <common/assert.h>
  58 #include <boost/range/algorithm_ext/push_back.hpp>
  59 #include <boost/algorithm/string/predicate.hpp>
  60
  61 #include <deque>
  62 #include <queue>
  63 #include <vector>
  64
  65 using namespace caspar::core;
  66
  67 namespace caspar { namespace ffmpeg {
  68
  69 bool is_frame_format_changed(const AVFrame& lhs, const AVFrame& rhs)
  70 {
  71         if (lhs.format != rhs.format)
  72                 return true;
  73
  74         for (int i = 0; i < AV_NUM_DATA_POINTERS; ++i)
  75         {
  76                 if (lhs.linesize[i] != rhs.linesize[i])
  77                         return true;
  78         }
  79
  80         return false;
  81 }
  82
  83 struct frame_muxer::impl : boost::noncopyable
  84 {
  85         std::queue<core::mutable_frame>                                 video_stream_;
  86         core::mutable_audio_buffer                                              audio_stream_;
  87         std::queue<draw_frame>                                                  frame_buffer_;
  88         display_mode                                                                    display_mode_                   = display_mode::invalid;
  89         const double                                                                    in_fps_;
  90         const video_format_desc                                                 format_desc_;
  91         audio_channel_layout                                                    channel_layout_;
  92
  93         std::vector<int>                                                                audio_cadence_                  = format_desc_.audio_cadence;
  94
  95         spl::shared_ptr<core::frame_factory>                    frame_factory_;
  96         std::shared_ptr<AVFrame>                                                previous_frame_;
  97
  98         std::unique_ptr<filter>                                                 filter_;
  99         const std::wstring                                                              filter_str_;
 100         bool                                                                                    force_deinterlacing_    = env::properties().get(L"configuration.force-deinterlace", true);
 101
 102         impl(
 103                         double in_fps,
 104                         const spl::shared_ptr<core::frame_factory>& frame_factory,
 105                         const core::video_format_desc& format_desc,
 106                         const core::audio_channel_layout& channel_layout,
 107                         const std::wstring& filter_str)
 108                 : in_fps_(in_fps)
 109                 , format_desc_(format_desc)
 110                 , channel_layout_(channel_layout)
 111                 , frame_factory_(frame_factory)
 112                 , filter_str_(filter_str)
 113         {
 114                 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
 115                 // This cadence fills the audio mixer most optimally.
 116                 boost::range::rotate(audio_cadence_, std::end(audio_cadence_)-1);
 117         }
 118
 119         void push_video(const std::shared_ptr<AVFrame>& video)
 120         {
 121                 if(!video)
 122                         return;
 123
 124                 if (previous_frame_ && video->data[0] && is_frame_format_changed(*previous_frame_, *video))
 125                 {
 126                         // Fixes bug where avfilter crashes server on some DV files (starts in YUV420p but changes to YUV411p after the first frame).
 127                         if (ffmpeg::is_logging_quiet_for_thread())
 128                                 CASPAR_LOG(debug) << L"[frame_muxer] Frame format has changed. Resetting display mode.";
 129                         else
 130                                 CASPAR_LOG(info) << L"[frame_muxer] Frame format has changed. Resetting display mode.";
 131
 132                         display_mode_ = display_mode::invalid;
 133                 }
 134
 135                 if(!video->data[0])
 136                 {
 137                         auto empty_frame = frame_factory_->create_frame(this, core::pixel_format_desc(core::pixel_format::invalid), channel_layout_);
 138                         video_stream_.push(std::move(empty_frame));
 139                         display_mode_ = display_mode::simple;
 140                 }
 141                 else
 142                 {
 143                         if(!filter_ || display_mode_ == display_mode::invalid)
 144                                 update_display_mode(video);
 145
 146                         filter_->push(video);
 147                         previous_frame_ = video;
 148                         for (auto& av_frame : filter_->poll_all())
 149                                 video_stream_.push(make_frame(this, av_frame, *frame_factory_, channel_layout_));
 150                 }
 151
 152                 merge();
 153         }
 154
 155         void push_audio(const std::shared_ptr<AVFrame>& audio)
 156         {
 157                 if(!audio)
 158                         return;
 159
 160                 if(!audio->data[0])
 161                 {
 162                         if (channel_layout_ == core::audio_channel_layout::invalid())
 163                                 channel_layout_ = *core::audio_channel_layout_repository::get_default()->get_layout(L"stereo");
 164
 165                         boost::range::push_back(audio_stream_, core::mutable_audio_buffer(audio_cadence_.front() * channel_layout_.num_channels, 0));
 166                 }
 167                 else
 168                 {
 169                         auto ptr = reinterpret_cast<int32_t*>(audio->data[0]);
 170                         audio_stream_.insert(audio_stream_.end(), ptr, ptr + audio->linesize[0]/sizeof(int32_t));
 171                 }
 172
 173                 merge();
 174         }
 175
 176         bool video_ready() const
 177         {
 178                 switch(display_mode_)
 179                 {
 180                 case display_mode::deinterlace_bob_reinterlace:
 181                 case display_mode::interlace:
 182                 case display_mode::half:
 183                         return video_stream_.size() >= 2;
 184                 default:
 185                         return video_stream_.size() >= 1;
 186                 }
 187         }
 188
 189         bool audio_ready() const
 190         {
 191                 switch(display_mode_)
 192                 {
 193                 case display_mode::duplicate:
 194                         return audio_stream_.size() >= static_cast<size_t>(audio_cadence_[0] + audio_cadence_[1 % audio_cadence_.size()]) * channel_layout_.num_channels;
 195                 default:
 196                         return audio_stream_.size() >= static_cast<size_t>(audio_cadence_.front()) * channel_layout_.num_channels;
 197                 }
 198         }
 199
 200         bool empty() const
 201         {
 202                 return frame_buffer_.empty();
 203         }
 204
 205         core::draw_frame front() const
 206         {
 207                 return frame_buffer_.front();
 208         }
 209
 210         void pop()
 211         {
 212                 frame_buffer_.pop();
 213         }
 214
 215         void merge()
 216         {
 217                 while(video_ready() && audio_ready() && display_mode_ != display_mode::invalid)
 218                 {
 219                         auto frame1                     = pop_video();
 220                         frame1.audio_data()     = pop_audio();
 221
 222                         switch(display_mode_)
 223                         {
 224                         case display_mode::simple:
 225                         case display_mode::deinterlace_bob:
 226                         case display_mode::deinterlace:
 227                                 {
 228                                         frame_buffer_.push(core::draw_frame(std::move(frame1)));
 229                                         break;
 230                                 }
 231                         case display_mode::interlace:
 232                         case display_mode::deinterlace_bob_reinterlace:
 233                                 {
 234                                         auto frame2 = pop_video();
 235
 236                                         frame_buffer_.push(core::draw_frame::interlace(
 237                                                 core::draw_frame(std::move(frame1)),
 238                                                 core::draw_frame(std::move(frame2)),
 239                                                 format_desc_.field_mode));
 240                                         break;
 241                                 }
 242                         case display_mode::duplicate:
 243                                 {
 244                                         //boost::range::push_back(frame1.audio_data(), pop_audio());
 245
 246                                         auto second_audio_frame = core::mutable_frame(
 247                                                         std::vector<array<std::uint8_t>>(),
 248                                                         pop_audio(),
 249                                                         frame1.stream_tag(),
 250                                                         core::pixel_format_desc(),
 251                                                         channel_layout_);
 252                                         auto first_frame = core::draw_frame(std::move(frame1));
 253                                         auto muted_first_frame = core::draw_frame(first_frame);
 254                                         muted_first_frame.transform().audio_transform.volume = 0;
 255                                         auto second_frame = core::draw_frame({ core::draw_frame(std::move(second_audio_frame)), muted_first_frame });
 256
 257                                         // Same video but different audio.
 258                                         frame_buffer_.push(first_frame);
 259                                         frame_buffer_.push(second_frame);
 260                                         break;
 261                                 }
 262                         case display_mode::half:
 263                                 {
 264                                         pop_video(); // Throw away
 265
 266                                         frame_buffer_.push(core::draw_frame(std::move(frame1)));
 267                                         break;
 268                                 }
 269                         default:
 270                                 CASPAR_THROW_EXCEPTION(invalid_operation());
 271                         }
 272                 }
 273         }
 274
 275         core::mutable_frame pop_video()
 276         {
 277                 auto frame = std::move(video_stream_.front());
 278                 video_stream_.pop();
 279                 return std::move(frame);
 280         }
 281
 282         core::mutable_audio_buffer pop_audio()
 283         {
 284                 if (audio_stream_.size() < audio_cadence_.front() * channel_layout_.num_channels)
 285                         CASPAR_THROW_EXCEPTION(out_of_range());
 286
 287                 auto begin = audio_stream_.begin();
 288                 auto end   = begin + audio_cadence_.front() * channel_layout_.num_channels;
 289
 290                 core::mutable_audio_buffer samples(begin, end);
 291                 audio_stream_.erase(begin, end);
 292
 293                 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1);
 294
 295                 return samples;
 296         }
 297
 298         void update_display_mode(const std::shared_ptr<AVFrame>& frame)
 299         {
 300                 std::wstring filter_str = filter_str_;
 301
 302                 display_mode_ = display_mode::simple;
 303
 304                 auto mode = get_mode(*frame);
 305                 if(mode == core::field_mode::progressive && frame->height < 720 && in_fps_ < 50.0) // SD frames are interlaced. Probably incorrect meta-data. Fix it.
 306                         mode = core::field_mode::upper;
 307
 308                 auto fps  = in_fps_;
 309
 310                 if(filter::is_deinterlacing(filter_str_))
 311                         mode = core::field_mode::progressive;
 312
 313                 if(filter::is_double_rate(filter_str_))
 314                         fps *= 2;
 315
 316                 display_mode_ = get_display_mode(mode, fps, format_desc_.field_mode, format_desc_.fps);
 317
 318                 if((frame->height != 480 || format_desc_.height != 486) && // don't deinterlace for NTSC DV
 319                                 display_mode_ == display_mode::simple && mode != core::field_mode::progressive && format_desc_.field_mode != core::field_mode::progressive &&
 320                                 frame->height != format_desc_.height)
 321                 {
 322                         display_mode_ = display_mode::deinterlace_bob_reinterlace; // The frame will most likely be scaled, we need to deinterlace->reinterlace
 323                 }
 324
 325                 // ALWAYS de-interlace, until we have GPU de-interlacing.
 326                 if(force_deinterlacing_ && frame->interlaced_frame && display_mode_ != display_mode::deinterlace_bob && display_mode_ != display_mode::deinterlace)
 327                         display_mode_ = display_mode::deinterlace_bob_reinterlace;
 328
 329                 if(display_mode_ == display_mode::deinterlace)
 330                         filter_str = append_filter(filter_str, L"YADIF=0:-1");
 331                 else if(display_mode_ == display_mode::deinterlace_bob || display_mode_ == display_mode::deinterlace_bob_reinterlace)
 332                         filter_str = append_filter(filter_str, L"YADIF=1:-1");
 333
 334                 if(display_mode_ == display_mode::invalid)
 335                 {
 336                         if (ffmpeg::is_logging_quiet_for_thread())
 337                                 CASPAR_LOG(debug) << L"[frame_muxer] Auto-transcode: Failed to detect display-mode.";
 338                         else
 339                                 CASPAR_LOG(warning) << L"[frame_muxer] Auto-transcode: Failed to detect display-mode.";
 340
 341                         display_mode_ = display_mode::simple;
 342                 }
 343
 344                 if(frame->height == 480) // NTSC DV
 345                 {
 346                         auto pad_str = L"PAD=" + boost::lexical_cast<std::wstring>(frame->width) + L":486:0:2:black";
 347                         filter_str = append_filter(filter_str, pad_str);
 348                 }
 349
 350                 filter_.reset (new filter(
 351                         frame->width,
 352                         frame->height,
 353                         boost::rational<int>(1000000, static_cast<int>(in_fps_ * 1000000)),
 354                         boost::rational<int>(static_cast<int>(in_fps_ * 1000000), 1000000),
 355                         boost::rational<int>(frame->sample_aspect_ratio.num, frame->sample_aspect_ratio.den),
 356                         static_cast<AVPixelFormat>(frame->format),
 357                         std::vector<AVPixelFormat>(),
 358                         u8(filter_str)));
 359
 360                 if (ffmpeg::is_logging_quiet_for_thread())
 361                         CASPAR_LOG(debug) << L"[frame_muxer] " << display_mode_ << L" " << print_mode(frame->width, frame->height, in_fps_, frame->interlaced_frame > 0);
 362                 else
 363                         CASPAR_LOG(info) << L"[frame_muxer] " << display_mode_ << L" " << print_mode(frame->width, frame->height, in_fps_, frame->interlaced_frame > 0);
 364         }
 365
 366         uint32_t calc_nb_frames(uint32_t nb_frames) const
 367         {
 368                 uint64_t nb_frames2 = nb_frames;
 369
 370                 if(filter_ && filter_->is_double_rate()) // Take into account transformations in filter.
 371                         nb_frames2 *= 2;
 372
 373                 switch(display_mode_) // Take into account transformation in run.
 374                 {
 375                 case display_mode::deinterlace_bob_reinterlace:
 376                 case display_mode::interlace:
 377                 case display_mode::half:
 378                         nb_frames2 /= 2;
 379                         break;
 380                 case display_mode::duplicate:
 381                         nb_frames2 *= 2;
 382                         break;
 383                 }
 384
 385                 return static_cast<uint32_t>(nb_frames2);
 386         }
 387
 388         void clear()
 389         {
 390                 while(!video_stream_.empty())
 391                         video_stream_.pop();
 392
 393                 audio_stream_.clear();
 394
 395                 while(!frame_buffer_.empty())
 396                         frame_buffer_.pop();
 397
 398                 filter_.reset();
 399         }
 400 };
 401
 402 frame_muxer::frame_muxer(
 403                 double in_fps,
 404                 const spl::shared_ptr<core::frame_factory>& frame_factory,
 405                 const core::video_format_desc& format_desc,
 406                 const core::audio_channel_layout& channel_layout,
 407                 const std::wstring& filter)
 408         : impl_(new impl(in_fps, frame_factory, format_desc, channel_layout, filter)){}
 409 void frame_muxer::push_video(const std::shared_ptr<AVFrame>& frame){impl_->push_video(frame);}
 410 void frame_muxer::push_audio(const std::shared_ptr<AVFrame>& frame){impl_->push_audio(frame);}
 411 bool frame_muxer::empty() const{return impl_->empty();}
 412 core::draw_frame frame_muxer::front() const{return impl_->front();}
 413 void frame_muxer::pop(){return impl_->pop();}
 414 void frame_muxer::clear(){impl_->clear();}
 415 uint32_t frame_muxer::calc_nb_frames(uint32_t nb_frames) const {return impl_->calc_nb_frames(nb_frames);}
 416 bool frame_muxer::video_ready() const{return impl_->video_ready();}
 417 bool frame_muxer::audio_ready() const{return impl_->audio_ready();}
 418
 419 }}