git.sesse.net Git - casparcg/blob - modules/ffmpeg/consumer/ffmpeg_consumer.cpp

   1 #include "../StdAfx.h"
   2
   3 #include "ffmpeg_consumer.h"
   4
   5 #include "../ffmpeg_error.h"
   6 #include "../producer/util/util.h"
   7 #include "../producer/filter/filter.h"
   8 #include "../producer/filter/audio_filter.h"
   9
  10 #include <common/except.h>
  11 #include <common/executor.h>
  12 #include <common/assert.h>
  13 #include <common/utf.h>
  14 #include <common/future.h>
  15 #include <common/diagnostics/graph.h>
  16 #include <common/env.h>
  17 #include <common/scope_exit.h>
  18 #include <common/ptree.h>
  19 #include <common/param.h>
  20 #include <common/semaphore.h>
  21
  22 #include <core/consumer/frame_consumer.h>
  23 #include <core/frame/frame.h>
  24 #include <core/frame/audio_channel_layout.h>
  25 #include <core/video_format.h>
  26 #include <core/monitor/monitor.h>
  27 #include <core/help/help_repository.h>
  28 #include <core/help/help_sink.h>
  29
  30 #include <boost/noncopyable.hpp>
  31 #include <boost/rational.hpp>
  32 #include <boost/format.hpp>
  33 #include <boost/algorithm/string/predicate.hpp>
  34 #include <boost/property_tree/ptree.hpp>
  35
  36 #pragma warning(push)
  37 #pragma warning(disable: 4244)
  38 #pragma warning(disable: 4245)
  39 #include <boost/crc.hpp>
  40 #pragma warning(pop)
  41
  42 #include <tbb/atomic.h>
  43 #include <tbb/concurrent_queue.h>
  44 #include <tbb/parallel_invoke.h>
  45 #include <tbb/parallel_for.h>
  46
  47 #include <numeric>
  48
  49 #pragma warning(push)
  50 #pragma warning(disable: 4244)
  51
  52 extern "C"
  53 {
  54         #define __STDC_CONSTANT_MACROS
  55         #define __STDC_LIMIT_MACROS
  56         #include <libavformat/avformat.h>
  57         #include <libavcodec/avcodec.h>
  58         #include <libavutil/avutil.h>
  59         #include <libavutil/frame.h>
  60         #include <libavutil/opt.h>
  61         #include <libavutil/imgutils.h>
  62         #include <libavutil/parseutils.h>
  63         #include <libavfilter/avfilter.h>
  64         #include <libavfilter/buffersink.h>
  65         #include <libavfilter/buffersrc.h>
  66 }
  67
  68 #pragma warning(pop)
  69
  70 namespace caspar { namespace ffmpeg {
  71
  72 void set_pixel_format(AVFilterContext* sink, AVPixelFormat pix_fmt)
  73 {
  74 #pragma warning (push)
  75 #pragma warning (disable : 4245)
  76
  77         FF(av_opt_set_int_list(
  78                 sink,
  79                 "pix_fmts",
  80                 std::vector<AVPixelFormat>({ pix_fmt, AVPixelFormat::AV_PIX_FMT_NONE }).data(),
  81                 -1,
  82                 AV_OPT_SEARCH_CHILDREN));
  83
  84 #pragma warning (pop)
  85 }
  86
  87 void adjust_video_filter(const AVCodec& codec, const core::video_format_desc& in_format, AVFilterContext* sink, std::string& filter)
  88 {
  89         switch (codec.id)
  90         {
  91         case AV_CODEC_ID_DVVIDEO:
  92                 // Crop
  93                 if (in_format.format == core::video_format::ntsc)
  94                         filter = u8(append_filter(u16(filter), L"crop=720:480:0:2"));
  95
  96                 // Pixel format selection
  97                 if (in_format.format == core::video_format::ntsc)
  98                         set_pixel_format(sink, AVPixelFormat::AV_PIX_FMT_YUV411P);
  99                 else if (in_format.format == core::video_format::pal)
 100                         set_pixel_format(sink, AVPixelFormat::AV_PIX_FMT_YUV420P);
 101                 else
 102                         set_pixel_format(sink, AVPixelFormat::AV_PIX_FMT_YUV422P);
 103
 104                 // Scale
 105                 if (in_format.height == 1080)
 106                         filter = u8(append_filter(u16(filter), in_format.duration == 1001
 107                                 ? L"scale=1280:1080"
 108                                 : L"scale=1440:1080"));
 109                 else if (in_format.height == 720)
 110                         filter = u8(append_filter(u16(filter), L"scale=960:720"));
 111
 112                 break;
 113         }
 114 }
 115
 116 void setup_codec_defaults(AVCodecContext& encoder)
 117 {
 118         static const int MEGABIT = 1000000;
 119
 120         switch (encoder.codec_id)
 121         {
 122         case AV_CODEC_ID_DNXHD:
 123                 encoder.bit_rate = 220 * MEGABIT;
 124
 125                 break;
 126         case AV_CODEC_ID_PRORES:
 127                 encoder.bit_rate = encoder.width < 1280
 128                                 ?  63 * MEGABIT
 129                                 : 220 * MEGABIT;
 130
 131                 break;
 132         case AV_CODEC_ID_H264:
 133                 av_opt_set(encoder.priv_data,   "preset",       "ultrafast",    0);
 134                 av_opt_set(encoder.priv_data,   "tune",         "fastdecode",   0);
 135                 av_opt_set(encoder.priv_data,   "crf",          "5",                    0);
 136
 137                 break;
 138         }
 139 }
 140
 141 bool is_pcm_s24le_not_supported(const AVFormatContext& container)
 142 {
 143         auto name = std::string(container.oformat->name);
 144
 145         if (name == "mp4" || name == "dv")
 146                 return true;
 147
 148         return false;
 149 }
 150
 151 template<typename Out, typename In>
 152 std::vector<Out> from_terminated_array(const In* array, In terminator)
 153 {
 154         std::vector<Out> result;
 155
 156         while (array != nullptr && *array != terminator)
 157         {
 158                 In val          = *array;
 159                 Out casted      = static_cast<Out>(val);
 160
 161                 result.push_back(casted);
 162
 163                 ++array;
 164         }
 165
 166         return result;
 167 }
 168
 169 class ffmpeg_consumer
 170 {
 171 private:
 172         const spl::shared_ptr<diagnostics::graph>       graph_;
 173         core::monitor::subject                                          subject_;
 174         std::string                                                                     path_;
 175         boost::filesystem::path                                         full_path_;
 176
 177         std::map<std::string, std::string>                      options_;
 178         bool                                                                            mono_streams_;
 179
 180         core::video_format_desc                                         in_video_format_;
 181         core::audio_channel_layout                                      in_channel_layout_                      = core::audio_channel_layout::invalid();
 182
 183         std::shared_ptr<AVFormatContext>                        oc_;
 184         tbb::atomic<bool>                                                       abort_request_;
 185
 186         std::shared_ptr<AVStream>                                       video_st_;
 187         std::vector<std::shared_ptr<AVStream>>          audio_sts_;
 188
 189         std::int64_t                                                            video_pts_                                      = 0;
 190         std::int64_t                                                            audio_pts_                                      = 0;
 191
 192         std::unique_ptr<audio_filter>                           audio_filter_;
 193
 194         // TODO: make use of already existent avfilter abstraction for video also
 195     AVFilterContext*                                                    video_graph_in_;
 196     AVFilterContext*                                                    video_graph_out_;
 197     std::shared_ptr<AVFilterGraph>                              video_graph_;
 198
 199         executor                                                                        video_encoder_executor_;
 200         executor                                                                        audio_encoder_executor_;
 201
 202         semaphore                                                                       tokens_                                         { 0 };
 203
 204         tbb::atomic<int64_t>                                            current_encoding_delay_;
 205
 206         executor                                                                        write_executor_;
 207
 208 public:
 209
 210         ffmpeg_consumer(
 211                         std::string path,
 212                         std::string options,
 213                         bool mono_streams)
 214                 : path_(path)
 215                 , full_path_(path)
 216                 , mono_streams_(mono_streams)
 217                 , audio_encoder_executor_(print() + L" audio_encoder")
 218                 , video_encoder_executor_(print() + L" video_encoder")
 219                 , write_executor_(print() + L" io")
 220         {
 221                 abort_request_ = false;
 222                 current_encoding_delay_ = 0;
 223
 224                 for(auto it =
 225                                 boost::sregex_iterator(
 226                                         options.begin(),
 227                                         options.end(),
 228                                         boost::regex("-(?<NAME>[^-\\s]+)(\\s+(?<VALUE>[^\\s]+))?"));
 229                         it != boost::sregex_iterator();
 230                         ++it)
 231                 {
 232                         options_[(*it)["NAME"].str()] = (*it)["VALUE"].matched ? (*it)["VALUE"].str() : "";
 233                 }
 234
 235         if (options_.find("threads") == options_.end())
 236             options_["threads"] = "auto";
 237
 238                 tokens_.release(
 239                         std::max(
 240                                 1,
 241                                 try_remove_arg<int>(
 242                                         options_,
 243                                         boost::regex("tokens")).get_value_or(2)));
 244         }
 245
 246         ~ffmpeg_consumer()
 247         {
 248                 if(oc_)
 249                 {
 250                         video_encoder_executor_.begin_invoke([&] { encode_video(core::const_frame::empty(), nullptr); });
 251                         audio_encoder_executor_.begin_invoke([&] { encode_audio(core::const_frame::empty(), nullptr); });
 252
 253                         video_encoder_executor_.stop();
 254                         audio_encoder_executor_.stop();
 255                         video_encoder_executor_.join();
 256                         audio_encoder_executor_.join();
 257
 258                         video_graph_.reset();
 259                         audio_filter_.reset();
 260                         video_st_.reset();
 261                         audio_sts_.clear();
 262
 263                         write_packet(nullptr, nullptr);
 264
 265                         write_executor_.stop();
 266                         write_executor_.join();
 267
 268                         FF(av_write_trailer(oc_.get()));
 269
 270                         if (!(oc_->oformat->flags & AVFMT_NOFILE) && oc_->pb)
 271                                 avio_close(oc_->pb);
 272
 273                         oc_.reset();
 274                 }
 275         }
 276
 277         void initialize(
 278                         const core::video_format_desc& format_desc,
 279                         const core::audio_channel_layout& channel_layout)
 280         {
 281                 try
 282                 {
 283                         static boost::regex prot_exp("^.+:.*" );
 284
 285                         if(!boost::regex_match(
 286                                         path_,
 287                                         prot_exp))
 288                         {
 289                                 if(!full_path_.is_complete())
 290                                 {
 291                                         full_path_ =
 292                                                 u8(
 293                                                         env::media_folder()) +
 294                                                         path_;
 295                                 }
 296
 297                                 if(boost::filesystem::exists(full_path_))
 298                                         boost::filesystem::remove(full_path_);
 299
 300                                 boost::filesystem::create_directories(full_path_.parent_path());
 301                         }
 302
 303                         graph_->set_color("frame-time", diagnostics::color(0.1f, 1.0f, 0.1f));
 304                         graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
 305                         graph_->set_text(print());
 306                         diagnostics::register_graph(graph_);
 307
 308                         const auto oformat_name =
 309                                 try_remove_arg<std::string>(
 310                                         options_,
 311                                         boost::regex("^f|format$"));
 312
 313                         AVFormatContext* oc;
 314
 315                         FF(avformat_alloc_output_context2(
 316                                 &oc,
 317                                 nullptr,
 318                                 oformat_name && !oformat_name->empty() ? oformat_name->c_str() : nullptr,
 319                                 full_path_.string().c_str()));
 320
 321                         oc_.reset(
 322                                 oc,
 323                                 avformat_free_context);
 324
 325                         CASPAR_VERIFY(oc_->oformat);
 326
 327                         oc_->interrupt_callback.callback = ffmpeg_consumer::interrupt_cb;
 328                         oc_->interrupt_callback.opaque   = this;
 329
 330                         CASPAR_VERIFY(format_desc.format != core::video_format::invalid);
 331
 332                         in_video_format_ = format_desc;
 333                         in_channel_layout_ = channel_layout;
 334
 335                         CASPAR_VERIFY(oc_->oformat);
 336
 337                         const auto video_codec_name =
 338                                 try_remove_arg<std::string>(
 339                                         options_,
 340                                         boost::regex("^c:v|codec:v|vcodec$"));
 341
 342                         const auto video_codec =
 343                                 video_codec_name
 344                                         ? avcodec_find_encoder_by_name(video_codec_name->c_str())
 345                                         : avcodec_find_encoder(oc_->oformat->video_codec);
 346
 347                         const auto audio_codec_name =
 348                                 try_remove_arg<std::string>(
 349                                         options_,
 350                                          boost::regex("^c:a|codec:a|acodec$"));
 351
 352                         const auto audio_codec =
 353                                 audio_codec_name
 354                                         ? avcodec_find_encoder_by_name(audio_codec_name->c_str())
 355                                         : (is_pcm_s24le_not_supported(*oc_)
 356                                                 ? avcodec_find_encoder(oc_->oformat->audio_codec)
 357                                                 : avcodec_find_encoder_by_name("pcm_s24le"));
 358
 359                         if (!video_codec)
 360                                 CASPAR_THROW_EXCEPTION(user_error() << msg_info(
 361                                                 "Failed to find video codec " + (video_codec_name
 362                                                                 ? *video_codec_name
 363                                                                 : "with id " + boost::lexical_cast<std::string>(
 364                                                                                 oc_->oformat->video_codec))));
 365                         if (!audio_codec)
 366                                 CASPAR_THROW_EXCEPTION(user_error() << msg_info(
 367                                                 "Failed to find audio codec " + (audio_codec_name
 368                                                                 ? *audio_codec_name
 369                                                                 : "with id " + boost::lexical_cast<std::string>(
 370                                                                                 oc_->oformat->audio_codec))));
 371
 372                         // Filters
 373
 374                         {
 375                                 configure_video_filters(
 376                                         *video_codec,
 377                                         try_remove_arg<std::string>(options_,
 378                                         boost::regex("vf|f:v|filter:v")).get_value_or(""));
 379
 380                                 configure_audio_filters(
 381                                         *audio_codec,
 382                                         try_remove_arg<std::string>(options_,
 383                                         boost::regex("af|f:a|filter:a")).get_value_or(""));
 384                         }
 385
 386                         // Encoders
 387
 388                         {
 389                                 auto video_options = options_;
 390                                 auto audio_options = options_;
 391
 392                                 video_st_ = open_encoder(
 393                                         *video_codec,
 394                                         video_options,
 395                                         0);
 396
 397                                 for (int i = 0; i < audio_filter_->get_num_output_pads(); ++i)
 398                                         audio_sts_.push_back(open_encoder(
 399                                                         *audio_codec,
 400                                                         audio_options,
 401                                                         i));
 402
 403                                 auto it = options_.begin();
 404                                 while(it != options_.end())
 405                                 {
 406                                         if(video_options.find(it->first) == video_options.end() || audio_options.find(it->first) == audio_options.end())
 407                                                 it = options_.erase(it);
 408                                         else
 409                                                 ++it;
 410                                 }
 411                         }
 412
 413                         // Output
 414                         {
 415                                 AVDictionary* av_opts = nullptr;
 416
 417                                 to_dict(
 418                                         &av_opts,
 419                                         std::move(options_));
 420
 421                                 CASPAR_SCOPE_EXIT
 422                                 {
 423                                         av_dict_free(&av_opts);
 424                                 };
 425
 426                                 if (!(oc_->oformat->flags & AVFMT_NOFILE))
 427                                 {
 428                                         FF(avio_open2(
 429                                                 &oc_->pb,
 430                                                 full_path_.string().c_str(),
 431                                                 AVIO_FLAG_WRITE,
 432                                                 &oc_->interrupt_callback,
 433                                                 &av_opts));
 434                                 }
 435
 436                                 FF(avformat_write_header(
 437                                         oc_.get(),
 438                                         &av_opts));
 439
 440                                 options_ = to_map(av_opts);
 441                         }
 442
 443                         // Dump Info
 444
 445                         av_dump_format(
 446                                 oc_.get(),
 447                                 0,
 448                                 oc_->filename,
 449                                 1);
 450
 451                         for (const auto& option : options_)
 452                         {
 453                                 CASPAR_LOG(warning)
 454                                         << L"Invalid option: -"
 455                                         << u16(option.first)
 456                                         << L" "
 457                                         << u16(option.second);
 458                         }
 459                 }
 460                 catch(...)
 461                 {
 462                         video_st_.reset();
 463                         audio_sts_.clear();
 464                         oc_.reset();
 465                         throw;
 466                 }
 467         }
 468
 469         core::monitor::subject& monitor_output()
 470         {
 471                 return subject_;
 472         }
 473
 474         void send(core::const_frame frame)
 475         {
 476                 CASPAR_VERIFY(in_video_format_.format != core::video_format::invalid);
 477
 478                 auto frame_timer = spl::make_shared<caspar::timer>();
 479
 480                 std::shared_ptr<void> token(
 481                         nullptr,
 482                         [this, frame, frame_timer](void*)
 483                         {
 484                                 tokens_.release();
 485                                 current_encoding_delay_ = frame.get_age_millis();
 486                                 graph_->set_value("frame-time", frame_timer->elapsed() * in_video_format_.fps * 0.5);
 487                         });
 488                 tokens_.acquire();
 489
 490                 video_encoder_executor_.begin_invoke([=]() mutable
 491                 {
 492                         encode_video(
 493                                 frame,
 494                                 token);
 495                 });
 496
 497                 audio_encoder_executor_.begin_invoke([=]() mutable
 498                 {
 499                         encode_audio(
 500                                 frame,
 501                                 token);
 502                 });
 503         }
 504
 505         bool ready_for_frame() const
 506         {
 507                 return tokens_.permits() > 0;
 508         }
 509
 510         void mark_dropped()
 511         {
 512                 graph_->set_tag(diagnostics::tag_severity::WARNING, "dropped-frame");
 513         }
 514
 515         std::wstring print() const
 516         {
 517                 return L"ffmpeg_consumer[" + u16(path_) + L"]";
 518         }
 519
 520         int64_t presentation_frame_age_millis() const
 521         {
 522                 return current_encoding_delay_;
 523         }
 524
 525 private:
 526
 527         static int interrupt_cb(void* ctx)
 528         {
 529                 CASPAR_ASSERT(ctx);
 530                 return reinterpret_cast<ffmpeg_consumer*>(ctx)->abort_request_;
 531         }
 532
 533         std::shared_ptr<AVStream> open_encoder(
 534                         const AVCodec& codec,
 535                         std::map<std::string,
 536                         std::string>& options,
 537                         int stream_number_for_media_type)
 538         {
 539                 auto st =
 540                         avformat_new_stream(
 541                                 oc_.get(),
 542                                 &codec);
 543
 544                 if (!st)
 545                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("Could not allocate video-stream.") << boost::errinfo_api_function("avformat_new_stream"));
 546
 547                 auto enc = st->codec;
 548
 549                 CASPAR_VERIFY(enc);
 550
 551                 switch(enc->codec_type)
 552                 {
 553                         case AVMEDIA_TYPE_VIDEO:
 554                         {
 555                                 enc->time_base                          = video_graph_out_->inputs[0]->time_base;
 556                                 enc->pix_fmt                                    = static_cast<AVPixelFormat>(video_graph_out_->inputs[0]->format);
 557                                 enc->sample_aspect_ratio                = st->sample_aspect_ratio = video_graph_out_->inputs[0]->sample_aspect_ratio;
 558                                 enc->width                                      = video_graph_out_->inputs[0]->w;
 559                                 enc->height                                     = video_graph_out_->inputs[0]->h;
 560                                 enc->bit_rate_tolerance         = 400 * 1000000;
 561
 562                                 break;
 563                         }
 564                         case AVMEDIA_TYPE_AUDIO:
 565                         {
 566                                 enc->time_base                          = audio_filter_->get_output_pad_info(stream_number_for_media_type).time_base;
 567                                 enc->sample_fmt                         = static_cast<AVSampleFormat>(audio_filter_->get_output_pad_info(stream_number_for_media_type).format);
 568                                 enc->sample_rate                                = audio_filter_->get_output_pad_info(stream_number_for_media_type).sample_rate;
 569                                 enc->channel_layout                     = audio_filter_->get_output_pad_info(stream_number_for_media_type).channel_layout;
 570                                 enc->channels                           = audio_filter_->get_output_pad_info(stream_number_for_media_type).channels;
 571
 572                                 break;
 573                         }
 574                 }
 575
 576                 setup_codec_defaults(*enc);
 577
 578                 if(oc_->oformat->flags & AVFMT_GLOBALHEADER)
 579                         enc->flags |= CODEC_FLAG_GLOBAL_HEADER;
 580
 581                 static const std::array<std::string, 4> char_id_map = {{"v", "a", "d", "s"}};
 582
 583                 const auto char_id = char_id_map.at(enc->codec_type);
 584
 585                 const auto codec_opts =
 586                         remove_options(
 587                                 options,
 588                                 boost::regex("^(" + char_id + "?[^:]+):" + char_id + "$"));
 589
 590                 AVDictionary* av_codec_opts = nullptr;
 591
 592                 to_dict(
 593                         &av_codec_opts,
 594                         options);
 595
 596                 to_dict(
 597                         &av_codec_opts,
 598                         codec_opts);
 599
 600                 options.clear();
 601
 602                 FF(avcodec_open2(
 603                         enc,
 604                         &codec,
 605                         av_codec_opts ? &av_codec_opts : nullptr));
 606
 607                 if(av_codec_opts)
 608                 {
 609                         auto t =
 610                                 av_dict_get(
 611                                         av_codec_opts,
 612                                         "",
 613                                          nullptr,
 614                                         AV_DICT_IGNORE_SUFFIX);
 615
 616                         while(t)
 617                         {
 618                                 options[t->key + (codec_opts.find(t->key) != codec_opts.end() ? ":" + char_id : "")] = t->value;
 619
 620                                 t = av_dict_get(
 621                                                 av_codec_opts,
 622                                                 "",
 623                                                 t,
 624                                                 AV_DICT_IGNORE_SUFFIX);
 625                         }
 626
 627                         av_dict_free(&av_codec_opts);
 628                 }
 629
 630                 if(enc->codec_type == AVMEDIA_TYPE_AUDIO && !(codec.capabilities & CODEC_CAP_VARIABLE_FRAME_SIZE))
 631                 {
 632                         CASPAR_ASSERT(enc->frame_size > 0);
 633                         audio_filter_->set_guaranteed_output_num_samples_per_frame(
 634                                         stream_number_for_media_type,
 635                                         enc->frame_size);
 636                 }
 637
 638                 return std::shared_ptr<AVStream>(st, [this](AVStream* st)
 639                 {
 640                         avcodec_close(st->codec);
 641                 });
 642         }
 643
 644         void configure_video_filters(
 645                         const AVCodec& codec,
 646                         std::string filtergraph)
 647         {
 648                 video_graph_.reset(
 649                                 avfilter_graph_alloc(),
 650                                 [](AVFilterGraph* p)
 651                                 {
 652                                         avfilter_graph_free(&p);
 653                                 });
 654
 655                 video_graph_->nb_threads  = boost::thread::hardware_concurrency()/2;
 656                 video_graph_->thread_type = AVFILTER_THREAD_SLICE;
 657
 658                 const auto sample_aspect_ratio =
 659                         boost::rational<int>(
 660                                         in_video_format_.square_width,
 661                                         in_video_format_.square_height) /
 662                         boost::rational<int>(
 663                                         in_video_format_.width,
 664                                         in_video_format_.height);
 665
 666                 const auto vsrc_options = (boost::format("video_size=%1%x%2%:pix_fmt=%3%:time_base=%4%/%5%:pixel_aspect=%6%/%7%:frame_rate=%8%/%9%")
 667                         % in_video_format_.width % in_video_format_.height
 668                         % AVPixelFormat::AV_PIX_FMT_BGRA
 669                         % in_video_format_.duration     % in_video_format_.time_scale
 670                         % sample_aspect_ratio.numerator() % sample_aspect_ratio.denominator()
 671                         % in_video_format_.time_scale % in_video_format_.duration).str();
 672
 673                 AVFilterContext* filt_vsrc = nullptr;
 674                 FF(avfilter_graph_create_filter(
 675                                 &filt_vsrc,
 676                                 avfilter_get_by_name("buffer"),
 677                                 "ffmpeg_consumer_buffer",
 678                                 vsrc_options.c_str(),
 679                                 nullptr,
 680                                 video_graph_.get()));
 681
 682                 AVFilterContext* filt_vsink = nullptr;
 683                 FF(avfilter_graph_create_filter(
 684                                 &filt_vsink,
 685                                 avfilter_get_by_name("buffersink"),
 686                                 "ffmpeg_consumer_buffersink",
 687                                 nullptr,
 688                                 nullptr,
 689                                 video_graph_.get()));
 690
 691 #pragma warning (push)
 692 #pragma warning (disable : 4245)
 693
 694                 FF(av_opt_set_int_list(
 695                                 filt_vsink,
 696                                 "pix_fmts",
 697                                 codec.pix_fmts,
 698                                 -1,
 699                                 AV_OPT_SEARCH_CHILDREN));
 700
 701 #pragma warning (pop)
 702
 703                 adjust_video_filter(codec, in_video_format_, filt_vsink, filtergraph);
 704
 705                 if (in_video_format_.width < 1280)
 706                         video_graph_->scale_sws_opts = "out_color_matrix=bt601";
 707                 else
 708                         video_graph_->scale_sws_opts = "out_color_matrix=bt709";
 709
 710                 configure_filtergraph(
 711                                 *video_graph_,
 712                                 filtergraph,
 713                                 *filt_vsrc,
 714                                 *filt_vsink);
 715
 716                 video_graph_in_  = filt_vsrc;
 717                 video_graph_out_ = filt_vsink;
 718
 719                 CASPAR_LOG(info)
 720                         <<      u16(std::string("\n")
 721                                 + avfilter_graph_dump(
 722                                                 video_graph_.get(),
 723                                                 nullptr));
 724         }
 725
 726         void configure_audio_filters(
 727                         const AVCodec& codec,
 728                         std::string filtergraph)
 729         {
 730                 int num_output_pads = 1;
 731
 732                 if (mono_streams_)
 733                 {
 734                         num_output_pads = in_channel_layout_.num_channels;
 735                 }
 736
 737                 if (num_output_pads > 1)
 738                 {
 739                         std::string splitfilter = "[a:0]channelsplit=channel_layout=";
 740
 741                         splitfilter += (boost::format("0x%|1$x|") % create_channel_layout_bitmask(in_channel_layout_.num_channels)).str();
 742
 743                         for (int i = 0; i < num_output_pads; ++i)
 744                                 splitfilter += "[aout:" + boost::lexical_cast<std::string>(i) + "]";
 745
 746                         filtergraph = u8(append_filter(u16(filtergraph), u16(splitfilter)));
 747                 }
 748
 749                 std::vector<audio_output_pad> output_pads(
 750                                 num_output_pads,
 751                                 audio_output_pad(
 752                                                 from_terminated_array<int>(                             codec.supported_samplerates,    0),
 753                                                 from_terminated_array<AVSampleFormat>(  codec.sample_fmts,                              AVSampleFormat::AV_SAMPLE_FMT_NONE),
 754                                                 from_terminated_array<uint64_t>(                codec.channel_layouts,                  static_cast<uint64_t>(0))));
 755
 756                 audio_filter_.reset(new audio_filter(
 757                                 { audio_input_pad(
 758                                                 boost::rational<int>(1, in_video_format_.audio_sample_rate),
 759                                                 in_video_format_.audio_sample_rate,
 760                                                 AVSampleFormat::AV_SAMPLE_FMT_S32,
 761                                                 create_channel_layout_bitmask(in_channel_layout_.num_channels)) },
 762                                                 output_pads,
 763                                                 filtergraph));
 764         }
 765
 766         void configure_filtergraph(
 767                         AVFilterGraph& graph,
 768                         const std::string& filtergraph,
 769                         AVFilterContext& source_ctx,
 770                         AVFilterContext& sink_ctx)
 771         {
 772                 AVFilterInOut* outputs = nullptr;
 773                 AVFilterInOut* inputs = nullptr;
 774
 775                 if(!filtergraph.empty())
 776                 {
 777                         outputs = avfilter_inout_alloc();
 778                         inputs  = avfilter_inout_alloc();
 779
 780                         try
 781                         {
 782                                 CASPAR_VERIFY(outputs && inputs);
 783
 784                                 outputs->name           = av_strdup("in");
 785                                 outputs->filter_ctx     = &source_ctx;
 786                                 outputs->pad_idx                = 0;
 787                                 outputs->next           = nullptr;
 788
 789                                 inputs->name                    = av_strdup("out");
 790                                 inputs->filter_ctx      = &sink_ctx;
 791                                 inputs->pad_idx         = 0;
 792                                 inputs->next                    = nullptr;
 793                         }
 794                         catch (...)
 795                         {
 796                                 avfilter_inout_free(&outputs);
 797                                 avfilter_inout_free(&inputs);
 798                                 throw;
 799                         }
 800
 801                         FF(avfilter_graph_parse(
 802                                         &graph,
 803                                         filtergraph.c_str(),
 804                                         inputs,
 805                                         outputs,
 806                                         nullptr));
 807                 }
 808                 else
 809                 {
 810                         FF(avfilter_link(
 811                                         &source_ctx,
 812                                         0,
 813                                         &sink_ctx,
 814                                         0));
 815                 }
 816
 817                 FF(avfilter_graph_config(
 818                                 &graph,
 819                                 nullptr));
 820         }
 821
 822         void encode_video(core::const_frame frame_ptr, std::shared_ptr<void> token)
 823         {
 824                 if(!video_st_)
 825                         return;
 826
 827                 auto enc = video_st_->codec;
 828
 829                 if(frame_ptr != core::const_frame::empty())
 830                 {
 831                         auto src_av_frame = create_frame();
 832
 833                         const auto sample_aspect_ratio =
 834                                 boost::rational<int>(
 835                                         in_video_format_.square_width,
 836                                         in_video_format_.square_height) /
 837                                 boost::rational<int>(
 838                                         in_video_format_.width,
 839                                         in_video_format_.height);
 840
 841                         src_av_frame->format                                            = AVPixelFormat::AV_PIX_FMT_BGRA;
 842                         src_av_frame->width                                             = in_video_format_.width;
 843                         src_av_frame->height                                            = in_video_format_.height;
 844                         src_av_frame->sample_aspect_ratio.num   = sample_aspect_ratio.numerator();
 845                         src_av_frame->sample_aspect_ratio.den   = sample_aspect_ratio.denominator();
 846                         src_av_frame->pts                                               = video_pts_;
 847
 848                         video_pts_ += 1;
 849
 850                         subject_
 851                                         << core::monitor::message("/frame")     % video_pts_
 852                                         << core::monitor::message("/path")      % path_
 853                                         << core::monitor::message("/fps")       % in_video_format_.fps;
 854
 855                         FF(av_image_fill_arrays(
 856                                 src_av_frame->data,
 857                                 src_av_frame->linesize,
 858                                 frame_ptr.image_data().begin(),
 859                                 static_cast<AVPixelFormat>(src_av_frame->format),
 860                                 in_video_format_.width,
 861                                 in_video_format_.height,
 862                                 1));
 863
 864                         FF(av_buffersrc_add_frame(
 865                                 video_graph_in_,
 866                                 src_av_frame.get()));
 867                 }
 868
 869                 int ret = 0;
 870
 871                 while(ret >= 0)
 872                 {
 873                         auto filt_frame = create_frame();
 874
 875                         ret = av_buffersink_get_frame(
 876                                 video_graph_out_,
 877                                 filt_frame.get());
 878
 879                         video_encoder_executor_.begin_invoke([=]
 880                         {
 881                                 if(ret == AVERROR_EOF)
 882                                 {
 883                                         if(enc->codec->capabilities & CODEC_CAP_DELAY)
 884                                         {
 885                                                 while(encode_av_frame(
 886                                                                 *video_st_,
 887                                                                 avcodec_encode_video2,
 888                                                                 nullptr, token))
 889                                                 {
 890                                                         boost::this_thread::yield(); // TODO:
 891                                                 }
 892                                         }
 893                                 }
 894                                 else if(ret != AVERROR(EAGAIN))
 895                                 {
 896                                         FF_RET(ret, "av_buffersink_get_frame");
 897
 898                                         if (filt_frame->interlaced_frame)
 899                                         {
 900                                                 if (enc->codec->id == AV_CODEC_ID_MJPEG)
 901                                                         enc->field_order = filt_frame->top_field_first ? AV_FIELD_TT : AV_FIELD_BB;
 902                                                 else
 903                                                         enc->field_order = filt_frame->top_field_first ? AV_FIELD_TB : AV_FIELD_BT;
 904                                         }
 905                                         else
 906                                                 enc->field_order = AV_FIELD_PROGRESSIVE;
 907
 908                                         filt_frame->quality = enc->global_quality;
 909
 910                                         if (!enc->me_threshold)
 911                                                 filt_frame->pict_type = AV_PICTURE_TYPE_NONE;
 912
 913                                         encode_av_frame(
 914                                                 *video_st_,
 915                                                 avcodec_encode_video2,
 916                                                 filt_frame,
 917                                                 token);
 918
 919                                         boost::this_thread::yield(); // TODO:
 920                                 }
 921                         });
 922                 }
 923         }
 924
 925         void encode_audio(core::const_frame frame_ptr, std::shared_ptr<void> token)
 926         {
 927                 if(audio_sts_.empty())
 928                         return;
 929
 930                 if(frame_ptr != core::const_frame::empty())
 931                 {
 932                         auto src_av_frame = create_frame();
 933
 934                         src_av_frame->channels                  = in_channel_layout_.num_channels;
 935                         src_av_frame->channel_layout            = create_channel_layout_bitmask(in_channel_layout_.num_channels);
 936                         src_av_frame->sample_rate               = in_video_format_.audio_sample_rate;
 937                         src_av_frame->nb_samples                        = static_cast<int>(frame_ptr.audio_data().size()) / src_av_frame->channels;
 938                         src_av_frame->format                            = AV_SAMPLE_FMT_S32;
 939                         src_av_frame->pts                               = audio_pts_;
 940
 941                         audio_pts_ += src_av_frame->nb_samples;
 942
 943                         FF(av_samples_fill_arrays(
 944                                         src_av_frame->extended_data,
 945                                         src_av_frame->linesize,
 946                                         reinterpret_cast<const std::uint8_t*>(&*frame_ptr.audio_data().begin()),
 947                                         src_av_frame->channels,
 948                                         src_av_frame->nb_samples,
 949                                         static_cast<AVSampleFormat>(src_av_frame->format),
 950                                         16));
 951
 952                         audio_filter_->push(0, src_av_frame);
 953                 }
 954
 955                 for (int pad_id = 0; pad_id < audio_filter_->get_num_output_pads(); ++pad_id)
 956                 {
 957                         for (auto filt_frame : audio_filter_->poll_all(pad_id))
 958                         {
 959                                 audio_encoder_executor_.begin_invoke([=]
 960                                 {
 961                                         encode_av_frame(
 962                                                         *audio_sts_.at(pad_id),
 963                                                         avcodec_encode_audio2,
 964                                                         filt_frame,
 965                                                         token);
 966
 967                                         boost::this_thread::yield(); // TODO:
 968                                 });
 969                         }
 970                 }
 971
 972                 bool eof = frame_ptr == core::const_frame::empty();
 973
 974                 if (eof)
 975                 {
 976                         audio_encoder_executor_.begin_invoke([=]
 977                         {
 978                                 for (int pad_id = 0; pad_id < audio_filter_->get_num_output_pads(); ++pad_id)
 979                                 {
 980                                         auto enc = audio_sts_.at(pad_id)->codec;
 981
 982                                         if (enc->codec->capabilities & CODEC_CAP_DELAY)
 983                                         {
 984                                                 while (encode_av_frame(
 985                                                                 *audio_sts_.at(pad_id),
 986                                                                 avcodec_encode_audio2,
 987                                                                 nullptr,
 988                                                                 token))
 989                                                 {
 990                                                         boost::this_thread::yield(); // TODO:
 991                                                 }
 992                                         }
 993                                 }
 994                         });
 995                 }
 996         }
 997
 998         template<typename F>
 999         bool encode_av_frame(
1000                         AVStream& st,
1001                         const F& func,
1002                         const std::shared_ptr<AVFrame>& src_av_frame,
1003                         std::shared_ptr<void> token)
1004         {
1005                 AVPacket pkt = {};
1006                 av_init_packet(&pkt);
1007
1008                 int got_packet = 0;
1009
1010                 FF(func(
1011                         st.codec,
1012                         &pkt,
1013                         src_av_frame.get(),
1014                         &got_packet));
1015
1016                 if(!got_packet || pkt.size <= 0)
1017                         return false;
1018
1019                 pkt.stream_index = st.index;
1020
1021                 if (pkt.pts != AV_NOPTS_VALUE)
1022                 {
1023                         pkt.pts =
1024                                 av_rescale_q(
1025                                         pkt.pts,
1026                                         st.codec->time_base,
1027                                         st.time_base);
1028                 }
1029
1030                 if (pkt.dts != AV_NOPTS_VALUE)
1031                 {
1032                         pkt.dts =
1033                                 av_rescale_q(
1034                                         pkt.dts,
1035                                         st.codec->time_base,
1036                                         st.time_base);
1037                 }
1038
1039                 pkt.duration =
1040                         static_cast<int>(
1041                                 av_rescale_q(
1042                                         pkt.duration,
1043                                         st.codec->time_base, st.time_base));
1044
1045                 write_packet(
1046                         std::shared_ptr<AVPacket>(
1047                                 new AVPacket(pkt),
1048                                 [](AVPacket* p)
1049                                 {
1050                                         av_free_packet(p);
1051                                         delete p;
1052                                 }), token);
1053
1054                 return true;
1055         }
1056
1057         void write_packet(
1058                         const std::shared_ptr<AVPacket>& pkt_ptr,
1059                         std::shared_ptr<void> token)
1060         {
1061                 write_executor_.begin_invoke([this, pkt_ptr, token]() mutable
1062                 {
1063                         FF(av_interleaved_write_frame(
1064                                 oc_.get(),
1065                                 pkt_ptr.get()));
1066                 });
1067         }
1068
1069         template<typename T>
1070         static boost::optional<T> try_remove_arg(
1071                         std::map<std::string, std::string>& options,
1072                         const boost::regex& expr)
1073         {
1074                 for(auto it = options.begin(); it != options.end(); ++it)
1075                 {
1076                         if(boost::regex_search(it->first, expr))
1077                         {
1078                                 auto arg = it->second;
1079                                 options.erase(it);
1080                                 return boost::lexical_cast<T>(arg);
1081                         }
1082                 }
1083
1084                 return boost::optional<T>();
1085         }
1086
1087         static std::map<std::string, std::string> remove_options(
1088                         std::map<std::string, std::string>& options,
1089                         const boost::regex& expr)
1090         {
1091                 std::map<std::string, std::string> result;
1092
1093                 auto it = options.begin();
1094                 while(it != options.end())
1095                 {
1096                         boost::smatch what;
1097                         if(boost::regex_search(it->first, what, expr))
1098                         {
1099                                 result[
1100                                         what.size() > 0 && what[1].matched
1101                                                 ? what[1].str()
1102                                                 : it->first] = it->second;
1103                                 it = options.erase(it);
1104                         }
1105                         else
1106                                 ++it;
1107                 }
1108
1109                 return result;
1110         }
1111
1112         static void to_dict(AVDictionary** dest, const std::map<std::string, std::string>& c)
1113         {
1114                 for (const auto& entry : c)
1115                 {
1116                         av_dict_set(
1117                                 dest,
1118                                 entry.first.c_str(),
1119                                 entry.second.c_str(), 0);
1120                 }
1121         }
1122
1123         static std::map<std::string, std::string> to_map(AVDictionary* dict)
1124         {
1125                 std::map<std::string, std::string> result;
1126
1127                 for(auto t = dict
1128                                 ? av_dict_get(
1129                                         dict,
1130                                         "",
1131                                         nullptr,
1132                                         AV_DICT_IGNORE_SUFFIX)
1133                                 : nullptr;
1134                         t;
1135                         t = av_dict_get(
1136                                 dict,
1137                                 "",
1138                                 t,
1139                                 AV_DICT_IGNORE_SUFFIX))
1140                 {
1141                         result[t->key] = t->value;
1142                 }
1143
1144                 return result;
1145         }
1146 };
1147
1148 int crc16(const std::string& str)
1149 {
1150         boost::crc_16_type result;
1151
1152         result.process_bytes(str.data(), str.length());
1153
1154         return result.checksum();
1155 }
1156
1157 struct ffmpeg_consumer_proxy : public core::frame_consumer
1158 {
1159         const std::string                                       path_;
1160         const std::string                                       options_;
1161         const bool                                                      separate_key_;
1162         const bool                                                      mono_streams_;
1163         const bool                                                      compatibility_mode_;
1164         int                                                                     consumer_index_offset_;
1165
1166         std::unique_ptr<ffmpeg_consumer>        consumer_;
1167         std::unique_ptr<ffmpeg_consumer>        key_only_consumer_;
1168
1169 public:
1170
1171         ffmpeg_consumer_proxy(const std::string& path, const std::string& options, bool separate_key, bool mono_streams, bool compatibility_mode)
1172                 : path_(path)
1173                 , options_(options)
1174                 , separate_key_(separate_key)
1175                 , mono_streams_(mono_streams)
1176                 , compatibility_mode_(compatibility_mode)
1177                 , consumer_index_offset_(crc16(path))
1178         {
1179         }
1180
1181         void initialize(const core::video_format_desc& format_desc, const core::audio_channel_layout& channel_layout, int) override
1182         {
1183                 if (consumer_)
1184                         CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info("Cannot reinitialize ffmpeg-consumer."));
1185
1186                 consumer_.reset(new ffmpeg_consumer(path_, options_, mono_streams_));
1187                 consumer_->initialize(format_desc, channel_layout);
1188
1189                 if (separate_key_)
1190                 {
1191                         boost::filesystem::path fill_file(path_);
1192                         auto without_extension = u16(fill_file.parent_path().string() + "/" + fill_file.stem().string());
1193                         auto key_file = without_extension + L"_A" + u16(fill_file.extension().string());
1194
1195                         key_only_consumer_.reset(new ffmpeg_consumer(u8(key_file), options_, mono_streams_));
1196                         key_only_consumer_->initialize(format_desc, channel_layout);
1197                 }
1198         }
1199
1200         int64_t presentation_frame_age_millis() const override
1201         {
1202                 return consumer_ ? static_cast<int64_t>(consumer_->presentation_frame_age_millis()) : 0;
1203         }
1204
1205         std::future<bool> send(core::const_frame frame) override
1206         {
1207                 bool ready_for_frame = consumer_->ready_for_frame();
1208
1209                 if (ready_for_frame && separate_key_)
1210                         ready_for_frame = ready_for_frame && key_only_consumer_->ready_for_frame();
1211
1212                 if (ready_for_frame)
1213                 {
1214                         consumer_->send(frame);
1215
1216                         if (separate_key_)
1217                                 key_only_consumer_->send(frame.key_only());
1218                 }
1219                 else
1220                 {
1221                         consumer_->mark_dropped();
1222
1223                         if (separate_key_)
1224                                 key_only_consumer_->mark_dropped();
1225                 }
1226
1227                 return make_ready_future(true);
1228         }
1229
1230         std::wstring print() const override
1231         {
1232                 return consumer_ ? consumer_->print() : L"[ffmpeg_consumer]";
1233         }
1234
1235         std::wstring name() const override
1236         {
1237                 return L"ffmpeg";
1238         }
1239
1240         boost::property_tree::wptree info() const override
1241         {
1242                 boost::property_tree::wptree info;
1243
1244                 info.add(L"type",                       L"ffmpeg");
1245                 info.add(L"path",                       u16(path_));
1246                 info.add(L"separate_key",       separate_key_);
1247                 info.add(L"mono_streams",       mono_streams_);
1248
1249                 return info;
1250         }
1251
1252         bool has_synchronization_clock() const override
1253         {
1254                 return false;
1255         }
1256
1257         int buffer_depth() const override
1258         {
1259                 return -1;
1260         }
1261
1262         int index() const override
1263         {
1264                 return compatibility_mode_ ? 200 : 100000 + consumer_index_offset_;
1265         }
1266
1267         core::monitor::subject& monitor_output() override
1268         {
1269                 return consumer_->monitor_output();
1270         }
1271 };
1272
1273 void describe_ffmpeg_consumer(core::help_sink& sink, const core::help_repository& repo)
1274 {
1275         sink.short_description(L"For streaming/recording the contents of a channel using FFmpeg.");
1276         sink.syntax(L"FILE,STREAM [filename:string],[url:string] {-[ffmpeg_param1:string] [value1:string] {-[ffmpeg_param2:string] [value2:string] {...}}} {[separate_key:SEPARATE_KEY]} {[mono_streams:MONO_STREAMS]}");
1277         sink.para()->text(L"For recording or streaming the contents of a channel using FFmpeg");
1278         sink.definitions()
1279                 ->item(L"filename",                     L"The filename under the media folder including the extension (decides which kind of container format that will be used).")
1280                 ->item(L"url",                          L"If the filename is given in the form of an URL a network stream will be created instead of a file on disk.")
1281                 ->item(L"ffmpeg_paramX",                L"A parameter supported by FFmpeg. For example vcodec or acodec etc.")
1282                 ->item(L"separate_key",         L"If defined will create two files simultaneously -- One for fill and one for key (_A will be appended).")
1283                 ->item(L"mono_streams",         L"If defined every audio channel will be written to its own audio stream.");
1284         sink.para()->text(L"Examples:");
1285         sink.example(L">> ADD 1 FILE output.mov -vcodec dnxhd");
1286         sink.example(L">> ADD 1 FILE output.mov -vcodec prores");
1287         sink.example(L">> ADD 1 FILE output.mov -vcodec dvvideo");
1288         sink.example(L">> ADD 1 FILE output.mov -vcodec libx264 -preset ultrafast -tune fastdecode -crf 25");
1289         sink.example(L">> ADD 1 FILE output.mov -vcodec dnxhd SEPARATE_KEY", L"for creating output.mov with fill and output_A.mov with key/alpha");
1290         sink.example(L">> ADD 1 FILE output.mxf -vcodec dnxhd MONO_STREAMS", L"for creating output.mxf with every audio channel encoded in its own mono stream.");
1291         sink.example(L">> ADD 1 STREAM udp://<client_ip_address>:9250 -format mpegts -vcodec libx264 -crf 25 -tune zerolatency -preset ultrafast",
1292                 L"for streaming over UDP instead of creating a local file.");
1293 }
1294
1295 spl::shared_ptr<core::frame_consumer> create_ffmpeg_consumer(
1296                 const std::vector<std::wstring>& params, core::interaction_sink*, std::vector<spl::shared_ptr<core::video_channel>> channels)
1297 {
1298         if (params.size() < 1 || (!boost::iequals(params.at(0), L"STREAM") && !boost::iequals(params.at(0), L"FILE")))
1299                 return core::frame_consumer::empty();
1300
1301         auto params2                    = params;
1302         bool separate_key               = get_and_consume_flag(L"SEPARATE_KEY", params2);
1303         bool mono_streams               = get_and_consume_flag(L"MONO_STREAMS", params2);
1304         auto compatibility_mode = boost::iequals(params.at(0), L"FILE");
1305         auto path                               = u8(params2.size() > 1 ? params2.at(1) : L"");
1306         auto args                               = u8(boost::join(params2, L" "));
1307
1308         return spl::make_shared<ffmpeg_consumer_proxy>(path, args, separate_key, mono_streams, compatibility_mode);
1309 }
1310
1311 spl::shared_ptr<core::frame_consumer> create_preconfigured_ffmpeg_consumer(
1312                 const boost::property_tree::wptree& ptree, core::interaction_sink*, std::vector<spl::shared_ptr<core::video_channel>> channels)
1313 {
1314         return spl::make_shared<ffmpeg_consumer_proxy>(
1315                         u8(ptree_get<std::wstring>(ptree, L"path")),
1316                         u8(ptree.get<std::wstring>(L"args", L"")),
1317                         ptree.get<bool>(L"separate-key", false),
1318                         ptree.get<bool>(L"mono-streams", false),
1319                         false);
1320 }
1321
1322 }}