git.sesse.net Git - casparcg/blob - modules/ffmpeg/consumer/streaming_consumer.cpp

   1 #include "../StdAfx.h"
   2
   3 #include "ffmpeg_consumer.h"
   4
   5 #include "../ffmpeg_error.h"
   6
   7 #include <common/except.h>
   8 #include <common/executor.h>
   9 #include <common/assert.h>
  10 #include <common/utf.h>
  11 #include <common/future.h>
  12 #include <common/env.h>
  13 #include <common/scope_exit.h>
  14
  15 #include <core/consumer/frame_consumer.h>
  16 #include <core/frame/frame.h>
  17 #include <core/video_format.h>
  18 #include <core/monitor/monitor.h>
  19
  20 #include <boost/noncopyable.hpp>
  21 #include <boost/rational.hpp>
  22 #include <boost/format.hpp>
  23 #include <boost/algorithm/string/predicate.hpp>
  24 #include <boost/property_tree/ptree.hpp>
  25
  26 #pragma warning(push)
  27 #pragma warning(disable: 4244)
  28 #pragma warning(disable: 4245)
  29 #include <boost/crc.hpp>
  30 #pragma warning(pop)
  31
  32 #include <tbb/atomic.h>
  33 #include <tbb/concurrent_queue.h>
  34 #include <tbb/parallel_invoke.h>
  35 #include <tbb/parallel_for.h>
  36
  37 #include <numeric>
  38
  39 #pragma warning(push)
  40 #pragma warning(disable: 4244)
  41
  42 extern "C"
  43 {
  44         #define __STDC_CONSTANT_MACROS
  45         #define __STDC_LIMIT_MACROS
  46         #include <libavformat/avformat.h>
  47         #include <libavcodec/avcodec.h>
  48         #include <libavutil/avutil.h>
  49         #include <libavutil/frame.h>
  50         #include <libavutil/opt.h>
  51         #include <libavutil/imgutils.h>
  52         #include <libavutil/parseutils.h>
  53         #include <libavfilter/avfilter.h>
  54         #include <libavfilter/buffersink.h>
  55         #include <libavfilter/buffersrc.h>
  56 }
  57
  58 #pragma warning(pop)
  59
  60 namespace caspar { namespace ffmpeg {
  61
  62 int crc16(const std::string& str)
  63 {
  64         boost::crc_16_type result;
  65
  66         result.process_bytes(str.data(), str.length());
  67
  68         return result.checksum();
  69 }
  70
  71 class streaming_consumer final : public core::frame_consumer
  72 {
  73 public:
  74         // Static Members
  75
  76 private:
  77         core::monitor::subject                                          subject_;
  78         boost::filesystem::path                                         path_;
  79         int                                                                                     consumer_index_offset_;
  80
  81         std::map<std::string, std::string>                      options_;
  82
  83         core::video_format_desc                                         in_video_format_;
  84
  85         std::shared_ptr<AVFormatContext>                        oc_;
  86         tbb::atomic<bool>                                                       abort_request_;
  87
  88         std::shared_ptr<AVStream>                                       video_st_;
  89         std::shared_ptr<AVStream>                                       audio_st_;
  90
  91         std::int64_t                                                            video_pts_;
  92         std::int64_t                                                            audio_pts_;
  93
  94     AVFilterContext*                                                    audio_graph_in_;
  95     AVFilterContext*                                                    audio_graph_out_;
  96     std::shared_ptr<AVFilterGraph>                              audio_graph_;
  97         std::shared_ptr<AVBitStreamFilterContext>       audio_bitstream_filter_;
  98
  99     AVFilterContext*                                                    video_graph_in_;
 100     AVFilterContext*                                                    video_graph_out_;
 101     std::shared_ptr<AVFilterGraph>                              video_graph_;
 102         std::shared_ptr<AVBitStreamFilterContext>       video_bitstream_filter_;
 103
 104         executor                                                                        executor_;
 105
 106         executor                                                                        video_encoder_executor_;
 107         executor                                                                        audio_encoder_executor_;
 108
 109         tbb::atomic<int>                                                        tokens_;
 110         boost::mutex                                                            tokens_mutex_;
 111         boost::condition_variable                                       tokens_cond_;
 112
 113         executor                                                                        write_executor_;
 114
 115 public:
 116
 117         streaming_consumer(
 118                         std::string path,
 119                         std::string options)
 120                 : path_(path)
 121                 , consumer_index_offset_(crc16(path))
 122                 , video_pts_(0)
 123                 , audio_pts_(0)
 124                 , executor_(print())
 125                 , audio_encoder_executor_(print() + L" audio_encoder")
 126                 , video_encoder_executor_(print() + L" video_encoder")
 127                 , write_executor_(print() + L" io")
 128         {
 129                 abort_request_ = false;
 130
 131                 for(auto it =
 132                                 boost::sregex_iterator(
 133                                         options.begin(),
 134                                         options.end(),
 135                                         boost::regex("-(?<NAME>[^-\\s]+)(\\s+(?<VALUE>[^\\s]+))?"));
 136                         it != boost::sregex_iterator();
 137                         ++it)
 138                 {
 139                         options_[(*it)["NAME"].str()] = (*it)["VALUE"].matched ? (*it)["VALUE"].str() : "";
 140                 }
 141
 142         if (options_.find("threads") == options_.end())
 143             options_["threads"] = "auto";
 144
 145                 tokens_ =
 146                         std::max(
 147                                 1,
 148                                 try_remove_arg<int>(
 149                                         options_,
 150                                         boost::regex("tokens")).get_value_or(2));
 151         }
 152
 153         ~streaming_consumer()
 154         {
 155                 if(oc_)
 156                 {
 157                         video_encoder_executor_.begin_invoke([&] { encode_video(core::const_frame::empty(), nullptr); });
 158                         audio_encoder_executor_.begin_invoke([&] { encode_audio(core::const_frame::empty(), nullptr); });
 159
 160                         video_encoder_executor_.stop();
 161                         audio_encoder_executor_.stop();
 162                         video_encoder_executor_.join();
 163                         audio_encoder_executor_.join();
 164
 165                         video_graph_.reset();
 166                         audio_graph_.reset();
 167                         video_st_.reset();
 168                         audio_st_.reset();
 169
 170                         write_packet(nullptr, nullptr);
 171
 172                         write_executor_.stop();
 173                         write_executor_.join();
 174
 175                         FF(av_write_trailer(oc_.get()));
 176
 177                         if (!(oc_->oformat->flags & AVFMT_NOFILE) && oc_->pb)
 178                                 avio_close(oc_->pb);
 179
 180                         oc_.reset();
 181                 }
 182         }
 183
 184         void initialize(
 185                         const core::video_format_desc& format_desc,
 186                         int channel_index) override
 187         {
 188                 try
 189                 {
 190                         static boost::regex prot_exp("^.+:.*" );
 191
 192                         const auto overwrite =
 193                                 try_remove_arg<std::string>(
 194                                         options_,
 195                                         boost::regex("y")) != nullptr;
 196
 197                         if(!boost::regex_match(
 198                                         path_.string(),
 199                                         prot_exp))
 200                         {
 201                                 if(!path_.is_complete())
 202                                 {
 203                                         path_ =
 204                                                 u8(
 205                                                         env::media_folder()) +
 206                                                         path_.string();
 207                                 }
 208
 209                                 if(boost::filesystem::exists(path_))
 210                                 {
 211                                         if(!overwrite)
 212                                                 BOOST_THROW_EXCEPTION(invalid_argument() << msg_info("File exists"));
 213
 214                                         boost::filesystem::remove(path_);
 215                                 }
 216                         }
 217
 218                         const auto oformat_name =
 219                                 try_remove_arg<std::string>(
 220                                         options_,
 221                                         boost::regex("^f|format$"));
 222
 223                         AVFormatContext* oc;
 224
 225                         FF(avformat_alloc_output_context2(
 226                                 &oc,
 227                                 nullptr,
 228                                 oformat_name && !oformat_name->empty() ? oformat_name->c_str() : nullptr,
 229                                 path_.string().c_str()));
 230
 231                         oc_.reset(
 232                                 oc,
 233                                 avformat_free_context);
 234
 235                         CASPAR_VERIFY(oc_->oformat);
 236
 237                         oc_->interrupt_callback.callback = streaming_consumer::interrupt_cb;
 238                         oc_->interrupt_callback.opaque   = this;
 239
 240                         CASPAR_VERIFY(format_desc.format != core::video_format::invalid);
 241
 242                         in_video_format_ = format_desc;
 243
 244                         CASPAR_VERIFY(oc_->oformat);
 245
 246                         const auto video_codec_name =
 247                                 try_remove_arg<std::string>(
 248                                         options_,
 249                                         boost::regex("^c:v|codec:v|vcodec$"));
 250
 251                         const auto video_codec =
 252                                 video_codec_name
 253                                         ? avcodec_find_encoder_by_name(video_codec_name->c_str())
 254                                         : avcodec_find_encoder(oc_->oformat->video_codec);
 255
 256                         const auto audio_codec_name =
 257                                 try_remove_arg<std::string>(
 258                                         options_,
 259                                          boost::regex("^c:a|codec:a|acodec$"));
 260
 261                         const auto audio_codec =
 262                                 audio_codec_name
 263                                         ? avcodec_find_encoder_by_name(audio_codec_name->c_str())
 264                                         : avcodec_find_encoder(oc_->oformat->audio_codec);
 265
 266                         if (!video_codec)
 267                                 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(
 268                                                 "Failed to find video codec " + (video_codec_name
 269                                                                 ? *video_codec_name
 270                                                                 : "with id " + boost::lexical_cast<std::string>(
 271                                                                                 oc_->oformat->video_codec))));
 272                         if (!audio_codec)
 273                                 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(
 274                                                 "Failed to find audio codec " + (audio_codec_name
 275                                                                 ? *audio_codec_name
 276                                                                 : "with id " + boost::lexical_cast<std::string>(
 277                                                                                 oc_->oformat->audio_codec))));
 278
 279                         // Filters
 280
 281                         {
 282                                 configure_video_filters(
 283                                         *video_codec,
 284                                         try_remove_arg<std::string>(options_,
 285                                         boost::regex("vf|f:v|filter:v")).get_value_or(""));
 286
 287                                 configure_audio_filters(
 288                                         *audio_codec,
 289                                         try_remove_arg<std::string>(options_,
 290                                         boost::regex("af|f:a|filter:a")).get_value_or(""));
 291                         }
 292
 293                         // Bistream Filters
 294                         {
 295                                 configue_audio_bistream_filters(options_);
 296                                 configue_video_bistream_filters(options_);
 297                         }
 298
 299                         // Encoders
 300
 301                         {
 302                                 auto video_options = options_;
 303                                 auto audio_options = options_;
 304
 305                                 video_st_ = open_encoder(
 306                                         *video_codec,
 307                                         video_options);
 308
 309                                 audio_st_ = open_encoder(
 310                                         *audio_codec,
 311                                         audio_options);
 312
 313                                 auto it = options_.begin();
 314                                 while(it != options_.end())
 315                                 {
 316                                         if(video_options.find(it->first) == video_options.end() || audio_options.find(it->first) == audio_options.end())
 317                                                 it = options_.erase(it);
 318                                         else
 319                                                 ++it;
 320                                 }
 321                         }
 322
 323                         // Output
 324                         {
 325                                 AVDictionary* av_opts = nullptr;
 326
 327                                 to_dict(
 328                                         &av_opts,
 329                                         std::move(options_));
 330
 331                                 CASPAR_SCOPE_EXIT
 332                                 {
 333                                         av_dict_free(&av_opts);
 334                                 };
 335
 336                                 if (!(oc_->oformat->flags & AVFMT_NOFILE))
 337                                 {
 338                                         FF(avio_open2(
 339                                                 &oc_->pb,
 340                                                 path_.string().c_str(),
 341                                                 AVIO_FLAG_WRITE,
 342                                                 &oc_->interrupt_callback,
 343                                                 &av_opts));
 344                                 }
 345
 346                                 FF(avformat_write_header(
 347                                         oc_.get(),
 348                                         &av_opts));
 349
 350                                 options_ = to_map(av_opts);
 351                         }
 352
 353                         // Dump Info
 354
 355                         av_dump_format(
 356                                 oc_.get(),
 357                                 0,
 358                                 oc_->filename,
 359                                 1);
 360
 361                         for (const auto& option : options_)
 362                         {
 363                                 CASPAR_LOG(warning)
 364                                         << L"Invalid option: -"
 365                                         << u16(option.first)
 366                                         << L" "
 367                                         << u16(option.second);
 368                         }
 369                 }
 370                 catch(...)
 371                 {
 372                         video_st_.reset();
 373                         audio_st_.reset();
 374                         oc_.reset();
 375                         throw;
 376                 }
 377         }
 378
 379         core::monitor::subject& monitor_output() override
 380         {
 381                 return subject_;
 382         }
 383
 384         std::wstring name() const override
 385         {
 386                 return L"streaming";
 387         }
 388
 389         std::future<bool> send(core::const_frame frame) override
 390         {
 391                 CASPAR_VERIFY(in_video_format_.format != core::video_format::invalid);
 392
 393                 --tokens_;
 394                 std::shared_ptr<void> token(
 395                         nullptr,
 396                         [this](void*)
 397                         {
 398                                 ++tokens_;
 399                                 tokens_cond_.notify_one();
 400                         });
 401
 402                 return executor_.begin_invoke([=]() -> bool
 403                 {
 404                         boost::unique_lock<boost::mutex> tokens_lock(tokens_mutex_);
 405
 406                         while(tokens_ < 0)
 407                                 tokens_cond_.wait(tokens_lock);
 408
 409                         video_encoder_executor_.begin_invoke([=]() mutable
 410                         {
 411                                 encode_video(
 412                                         frame,
 413                                         token);
 414                         });
 415
 416                         audio_encoder_executor_.begin_invoke([=]() mutable
 417                         {
 418                                 encode_audio(
 419                                         frame,
 420                                         token);
 421                         });
 422
 423                         return true;
 424                 });
 425         }
 426
 427         std::wstring print() const override
 428         {
 429                 return L"streaming_consumer[" + u16(path_.string()) + L"]";
 430         }
 431
 432         virtual boost::property_tree::wptree info() const override
 433         {
 434                 return boost::property_tree::wptree();
 435         }
 436
 437         bool has_synchronization_clock() const override
 438         {
 439                 return false;
 440         }
 441
 442         int buffer_depth() const override
 443         {
 444                 return -1;
 445         }
 446
 447         int index() const override
 448         {
 449                 return 100000 + consumer_index_offset_;
 450         }
 451
 452 private:
 453
 454         static int interrupt_cb(void* ctx)
 455         {
 456                 CASPAR_ASSERT(ctx);
 457                 return reinterpret_cast<streaming_consumer*>(ctx)->abort_request_;
 458         }
 459
 460         std::shared_ptr<AVStream> open_encoder(
 461                         const AVCodec& codec,
 462                         std::map<std::string,
 463                         std::string>& options)
 464         {
 465                 auto st =
 466                         avformat_new_stream(
 467                                 oc_.get(),
 468                                 &codec);
 469
 470                 if (!st)
 471                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info("Could not allocate video-stream.") << boost::errinfo_api_function("av_new_stream"));
 472
 473                 auto enc = st->codec;
 474
 475                 CASPAR_VERIFY(enc);
 476
 477                 switch(enc->codec_type)
 478                 {
 479                         case AVMEDIA_TYPE_VIDEO:
 480                         {
 481                                 enc->time_base                    = video_graph_out_->inputs[0]->time_base;
 482                                 enc->pix_fmt                      = static_cast<AVPixelFormat>(video_graph_out_->inputs[0]->format);
 483                                 enc->sample_aspect_ratio  = st->sample_aspect_ratio = video_graph_out_->inputs[0]->sample_aspect_ratio;
 484                                 enc->width                                = video_graph_out_->inputs[0]->w;
 485                                 enc->height                               = video_graph_out_->inputs[0]->h;
 486
 487                                 break;
 488                         }
 489                         case AVMEDIA_TYPE_AUDIO:
 490                         {
 491                                 enc->time_base                    = audio_graph_out_->inputs[0]->time_base;
 492                                 enc->sample_fmt                   = static_cast<AVSampleFormat>(audio_graph_out_->inputs[0]->format);
 493                                 enc->sample_rate                  = audio_graph_out_->inputs[0]->sample_rate;
 494                                 enc->channel_layout               = audio_graph_out_->inputs[0]->channel_layout;
 495                                 enc->channels                     = audio_graph_out_->inputs[0]->channels;
 496
 497                                 break;
 498                         }
 499                 }
 500
 501                 if(oc_->oformat->flags & AVFMT_GLOBALHEADER)
 502                         enc->flags |= CODEC_FLAG_GLOBAL_HEADER;
 503
 504                 static const std::array<std::string, 4> char_id_map = {{"v", "a", "d", "s"}};
 505
 506                 const auto char_id = char_id_map.at(enc->codec_type);
 507
 508                 const auto codec_opts =
 509                         remove_options(
 510                                 options,
 511                                 boost::regex("^(" + char_id + "?[^:]+):" + char_id + "$"));
 512
 513                 AVDictionary* av_codec_opts = nullptr;
 514
 515                 to_dict(
 516                         &av_codec_opts,
 517                         options);
 518
 519                 to_dict(
 520                         &av_codec_opts,
 521                         codec_opts);
 522
 523                 options.clear();
 524
 525                 FF(avcodec_open2(
 526                         enc,
 527                         &codec,
 528                         av_codec_opts ? &av_codec_opts : nullptr));
 529
 530                 if(av_codec_opts)
 531                 {
 532                         auto t =
 533                                 av_dict_get(
 534                                         av_codec_opts,
 535                                         "",
 536                                          nullptr,
 537                                         AV_DICT_IGNORE_SUFFIX);
 538
 539                         while(t)
 540                         {
 541                                 options[t->key + (codec_opts.find(t->key) != codec_opts.end() ? ":" + char_id : "")] = t->value;
 542
 543                                 t = av_dict_get(
 544                                                 av_codec_opts,
 545                                                 "",
 546                                                 t,
 547                                                 AV_DICT_IGNORE_SUFFIX);
 548                         }
 549
 550                         av_dict_free(&av_codec_opts);
 551                 }
 552
 553                 if(enc->codec_type == AVMEDIA_TYPE_AUDIO && !(codec.capabilities & CODEC_CAP_VARIABLE_FRAME_SIZE))
 554                 {
 555                         CASPAR_ASSERT(enc->frame_size > 0);
 556                         av_buffersink_set_frame_size(audio_graph_out_,
 557                                                                                  enc->frame_size);
 558                 }
 559
 560                 return std::shared_ptr<AVStream>(st, [this](AVStream* st)
 561                 {
 562                         avcodec_close(st->codec);
 563                 });
 564         }
 565
 566         void configue_audio_bistream_filters(
 567                         std::map<std::string, std::string>& options)
 568         {
 569                 const auto audio_bitstream_filter_str =
 570                         try_remove_arg<std::string>(
 571                                 options,
 572                                 boost::regex("^bsf:a|absf$"));
 573
 574                 const auto audio_bitstream_filter =
 575                         audio_bitstream_filter_str
 576                                 ? av_bitstream_filter_init(audio_bitstream_filter_str->c_str())
 577                                 : nullptr;
 578
 579                 CASPAR_VERIFY(!audio_bitstream_filter_str || audio_bitstream_filter);
 580
 581                 if(audio_bitstream_filter)
 582                 {
 583                         audio_bitstream_filter_.reset(
 584                                 audio_bitstream_filter,
 585                                 av_bitstream_filter_close);
 586                 }
 587
 588                 if(audio_bitstream_filter_str && !audio_bitstream_filter_)
 589                         options["bsf:a"] = *audio_bitstream_filter_str;
 590         }
 591
 592         void configue_video_bistream_filters(
 593                         std::map<std::string, std::string>& options)
 594         {
 595                 const auto video_bitstream_filter_str =
 596                                 try_remove_arg<std::string>(
 597                                         options,
 598                                         boost::regex("^bsf:v|vbsf$"));
 599
 600                 const auto video_bitstream_filter =
 601                         video_bitstream_filter_str
 602                                 ? av_bitstream_filter_init(video_bitstream_filter_str->c_str())
 603                                 : nullptr;
 604
 605                 CASPAR_VERIFY(!video_bitstream_filter_str || video_bitstream_filter);
 606
 607                 if(video_bitstream_filter)
 608                 {
 609                         video_bitstream_filter_.reset(
 610                                 video_bitstream_filter,
 611                                 av_bitstream_filter_close);
 612                 }
 613
 614                 if(video_bitstream_filter_str && !video_bitstream_filter_)
 615                         options["bsf:v"] = *video_bitstream_filter_str;
 616         }
 617
 618         void configure_video_filters(
 619                         const AVCodec& codec,
 620                         const std::string& filtergraph)
 621         {
 622                 video_graph_.reset(
 623                                 avfilter_graph_alloc(),
 624                                 [](AVFilterGraph* p)
 625                                 {
 626                                         avfilter_graph_free(&p);
 627                                 });
 628
 629                 video_graph_->nb_threads  = boost::thread::hardware_concurrency()/2;
 630                 video_graph_->thread_type = AVFILTER_THREAD_SLICE;
 631
 632                 const auto sample_aspect_ratio =
 633                         boost::rational<int>(
 634                                         in_video_format_.square_width,
 635                                         in_video_format_.square_height) /
 636                         boost::rational<int>(
 637                                         in_video_format_.width,
 638                                         in_video_format_.height);
 639
 640                 const auto vsrc_options = (boost::format("video_size=%1%x%2%:pix_fmt=%3%:time_base=%4%/%5%:pixel_aspect=%6%/%7%:frame_rate=%8%/%9%")
 641                         % in_video_format_.width % in_video_format_.height
 642                         % AV_PIX_FMT_BGRA
 643                         % in_video_format_.duration     % in_video_format_.time_scale
 644                         % sample_aspect_ratio.numerator() % sample_aspect_ratio.denominator()
 645                         % in_video_format_.time_scale % in_video_format_.duration).str();
 646
 647                 AVFilterContext* filt_vsrc = nullptr;
 648                 FF(avfilter_graph_create_filter(
 649                                 &filt_vsrc,
 650                                 avfilter_get_by_name("buffer"),
 651                                 "ffmpeg_consumer_buffer",
 652                                 vsrc_options.c_str(),
 653                                 nullptr,
 654                                 video_graph_.get()));
 655
 656                 AVFilterContext* filt_vsink = nullptr;
 657                 FF(avfilter_graph_create_filter(
 658                                 &filt_vsink,
 659                                 avfilter_get_by_name("buffersink"),
 660                                 "ffmpeg_consumer_buffersink",
 661                                 nullptr,
 662                                 nullptr,
 663                                 video_graph_.get()));
 664
 665 #pragma warning (push)
 666 #pragma warning (disable : 4245)
 667
 668                 FF(av_opt_set_int_list(
 669                                 filt_vsink,
 670                                 "pix_fmts",
 671                                 codec.pix_fmts,
 672                                 -1,
 673                                 AV_OPT_SEARCH_CHILDREN));
 674
 675 #pragma warning (pop)
 676
 677                 configure_filtergraph(
 678                                 *video_graph_,
 679                                 filtergraph,
 680                                 *filt_vsrc,
 681                                 *filt_vsink);
 682
 683                 video_graph_in_  = filt_vsrc;
 684                 video_graph_out_ = filt_vsink;
 685
 686                 CASPAR_LOG(info)
 687                         <<      u16(std::string("\n")
 688                                 + avfilter_graph_dump(
 689                                                 video_graph_.get(),
 690                                                 nullptr));
 691         }
 692
 693         void configure_audio_filters(
 694                         const AVCodec& codec,
 695                         const std::string& filtergraph)
 696         {
 697                 audio_graph_.reset(
 698                         avfilter_graph_alloc(),
 699                         [](AVFilterGraph* p)
 700                         {
 701                                 avfilter_graph_free(&p);
 702                         });
 703
 704                 audio_graph_->nb_threads  = boost::thread::hardware_concurrency()/2;
 705                 audio_graph_->thread_type = AVFILTER_THREAD_SLICE;
 706
 707                 const auto asrc_options = (boost::format("sample_rate=%1%:sample_fmt=%2%:channels=%3%:time_base=%4%/%5%:channel_layout=%6%")
 708                         % in_video_format_.audio_sample_rate
 709                         % av_get_sample_fmt_name(AV_SAMPLE_FMT_S32)
 710                         % in_video_format_.audio_channels
 711                         % 1     % in_video_format_.audio_sample_rate
 712                         % boost::io::group(
 713                                 std::hex,
 714                                 std::showbase,
 715                                 av_get_default_channel_layout(in_video_format_.audio_channels))).str();
 716
 717                 AVFilterContext* filt_asrc = nullptr;
 718                 FF(avfilter_graph_create_filter(
 719                         &filt_asrc,
 720                         avfilter_get_by_name("abuffer"),
 721                         "ffmpeg_consumer_abuffer",
 722                         asrc_options.c_str(),
 723                         nullptr,
 724                         audio_graph_.get()));
 725
 726                 AVFilterContext* filt_asink = nullptr;
 727                 FF(avfilter_graph_create_filter(
 728                         &filt_asink,
 729                         avfilter_get_by_name("abuffersink"),
 730                         "ffmpeg_consumer_abuffersink",
 731                         nullptr,
 732                         nullptr,
 733                         audio_graph_.get()));
 734
 735 #pragma warning (push)
 736 #pragma warning (disable : 4245)
 737
 738                 FF(av_opt_set_int(
 739                         filt_asink,
 740                         "all_channel_counts",
 741                         1,
 742                         AV_OPT_SEARCH_CHILDREN));
 743
 744                 FF(av_opt_set_int_list(
 745                         filt_asink,
 746                         "sample_fmts",
 747                         codec.sample_fmts,
 748                         -1,
 749                         AV_OPT_SEARCH_CHILDREN));
 750
 751                 FF(av_opt_set_int_list(
 752                         filt_asink,
 753                         "channel_layouts",
 754                         codec.channel_layouts,
 755                         -1,
 756                         AV_OPT_SEARCH_CHILDREN));
 757
 758                 FF(av_opt_set_int_list(
 759                         filt_asink,
 760                         "sample_rates" ,
 761                         codec.supported_samplerates,
 762                         -1,
 763                         AV_OPT_SEARCH_CHILDREN));
 764
 765 #pragma warning (pop)
 766
 767                 configure_filtergraph(
 768                         *audio_graph_,
 769                         filtergraph,
 770                         *filt_asrc,
 771                         *filt_asink);
 772
 773                 audio_graph_in_  = filt_asrc;
 774                 audio_graph_out_ = filt_asink;
 775
 776                 CASPAR_LOG(info)
 777                         <<      u16(std::string("\n")
 778                                 + avfilter_graph_dump(
 779                                         audio_graph_.get(),
 780                                         nullptr));
 781         }
 782
 783         void configure_filtergraph(
 784                         AVFilterGraph& graph,
 785                         const std::string& filtergraph,
 786                         AVFilterContext& source_ctx,
 787                         AVFilterContext& sink_ctx)
 788         {
 789                 AVFilterInOut* outputs = nullptr;
 790                 AVFilterInOut* inputs = nullptr;
 791
 792                 try
 793                 {
 794                         if(!filtergraph.empty())
 795                         {
 796                                 outputs = avfilter_inout_alloc();
 797                                 inputs  = avfilter_inout_alloc();
 798
 799                                 CASPAR_VERIFY(outputs && inputs);
 800
 801                                 outputs->name       = av_strdup("in");
 802                                 outputs->filter_ctx = &source_ctx;
 803                                 outputs->pad_idx    = 0;
 804                                 outputs->next       = nullptr;
 805
 806                                 inputs->name        = av_strdup("out");
 807                                 inputs->filter_ctx  = &sink_ctx;
 808                                 inputs->pad_idx     = 0;
 809                                 inputs->next        = nullptr;
 810
 811                                 FF(avfilter_graph_parse(
 812                                         &graph,
 813                                         filtergraph.c_str(),
 814                                         &inputs,
 815                                         &outputs,
 816                                         nullptr));
 817                         }
 818                         else
 819                         {
 820                                 FF(avfilter_link(
 821                                         &source_ctx,
 822                                         0,
 823                                         &sink_ctx,
 824                                         0));
 825                         }
 826
 827                         FF(avfilter_graph_config(
 828                                 &graph,
 829                                 nullptr));
 830                 }
 831                 catch(...)
 832                 {
 833                         avfilter_inout_free(&outputs);
 834                         avfilter_inout_free(&inputs);
 835                         throw;
 836                 }
 837         }
 838
 839         void encode_video(core::const_frame frame_ptr, std::shared_ptr<void> token)
 840         {
 841                 if(!video_st_)
 842                         return;
 843
 844                 auto enc = video_st_->codec;
 845
 846                 std::shared_ptr<AVFrame> src_av_frame;
 847
 848                 if(frame_ptr != core::const_frame::empty())
 849                 {
 850                         src_av_frame.reset(
 851                                 av_frame_alloc(),
 852                                 [frame_ptr](AVFrame* frame)
 853                                 {
 854                                         av_frame_free(&frame);
 855                                 });
 856
 857                         avcodec_get_frame_defaults(src_av_frame.get());
 858
 859                         const auto sample_aspect_ratio =
 860                                 boost::rational<int>(
 861                                         in_video_format_.square_width,
 862                                         in_video_format_.square_height) /
 863                                 boost::rational<int>(
 864                                         in_video_format_.width,
 865                                         in_video_format_.height);
 866
 867                         src_av_frame->format                              = AV_PIX_FMT_BGRA;
 868                         src_av_frame->width                                       = in_video_format_.width;
 869                         src_av_frame->height                              = in_video_format_.height;
 870                         src_av_frame->sample_aspect_ratio.num = sample_aspect_ratio.numerator();
 871                         src_av_frame->sample_aspect_ratio.den = sample_aspect_ratio.denominator();
 872                         src_av_frame->pts                                         = video_pts_;
 873
 874                         video_pts_ += 1;
 875
 876                         FF(av_image_fill_arrays(
 877                                 src_av_frame->data,
 878                                 src_av_frame->linesize,
 879                                 frame_ptr.image_data().begin(),
 880                                 static_cast<AVPixelFormat>(src_av_frame->format),
 881                                 in_video_format_.width,
 882                                 in_video_format_.height,
 883                                 1));
 884
 885                         FF(av_buffersrc_add_frame(
 886                                 video_graph_in_,
 887                                 src_av_frame.get()));
 888                 }
 889
 890                 int ret = 0;
 891
 892                 while(ret >= 0)
 893                 {
 894                         std::shared_ptr<AVFrame> filt_frame(
 895                                 av_frame_alloc(),
 896                                 [](AVFrame* p)
 897                                 {
 898                                         av_frame_free(&p);
 899                                 });
 900
 901                         ret = av_buffersink_get_frame(
 902                                 video_graph_out_,
 903                                 filt_frame.get());
 904
 905                         video_encoder_executor_.begin_invoke([=]
 906                         {
 907                                 if(ret == AVERROR_EOF)
 908                                 {
 909                                         if(enc->codec->capabilities & CODEC_CAP_DELAY)
 910                                         {
 911                                                 while(encode_av_frame(
 912                                                                 *video_st_,
 913                                                                 video_bitstream_filter_.get(),
 914                                                                 avcodec_encode_video2,
 915                                                                 nullptr, token))
 916                                                 {
 917                                                         boost::this_thread::yield(); // TODO:
 918                                                 }
 919                                         }
 920                                 }
 921                                 else if(ret != AVERROR(EAGAIN))
 922                                 {
 923                                         FF_RET(ret, "av_buffersink_get_frame");
 924
 925                                         if (filt_frame->interlaced_frame)
 926                                         {
 927                                                 if (enc->codec->id == AV_CODEC_ID_MJPEG)
 928                                                         enc->field_order = filt_frame->top_field_first ? AV_FIELD_TT : AV_FIELD_BB;
 929                                                 else
 930                                                         enc->field_order = filt_frame->top_field_first ? AV_FIELD_TB : AV_FIELD_BT;
 931                                         }
 932                                         else
 933                                                 enc->field_order = AV_FIELD_PROGRESSIVE;
 934
 935                                         filt_frame->quality = enc->global_quality;
 936
 937                                         if (!enc->me_threshold)
 938                                                 filt_frame->pict_type = AV_PICTURE_TYPE_NONE;
 939
 940                                         encode_av_frame(
 941                                                 *video_st_,
 942                                                 video_bitstream_filter_.get(),
 943                                                 avcodec_encode_video2,
 944                                                 filt_frame,
 945                                                 token);
 946
 947                                         boost::this_thread::yield(); // TODO:
 948                                 }
 949                         });
 950                 }
 951         }
 952
 953         void encode_audio(core::const_frame frame_ptr, std::shared_ptr<void> token)
 954         {
 955                 if(!audio_st_)
 956                         return;
 957
 958                 auto enc = audio_st_->codec;
 959
 960                 std::shared_ptr<AVFrame> src_av_frame;
 961
 962                 if(frame_ptr != core::const_frame::empty())
 963                 {
 964                         src_av_frame.reset(
 965                                 av_frame_alloc(),
 966                                 [](AVFrame* p)
 967                                 {
 968                                         av_frame_free(&p);
 969                                 });
 970
 971                         src_av_frame->channels           = in_video_format_.audio_channels;
 972                         src_av_frame->channel_layout = av_get_default_channel_layout(in_video_format_.audio_channels);
 973                         src_av_frame->sample_rate        = in_video_format_.audio_sample_rate;
 974                         src_av_frame->nb_samples         = static_cast<int>(frame_ptr.audio_data().size()) / src_av_frame->channels;
 975                         src_av_frame->format             = AV_SAMPLE_FMT_S32;
 976                         src_av_frame->pts                        = audio_pts_;
 977
 978                         audio_pts_ += src_av_frame->nb_samples;
 979
 980                         FF(av_samples_fill_arrays(
 981                                         src_av_frame->extended_data,
 982                                         src_av_frame->linesize,
 983                                         reinterpret_cast<const std::uint8_t*>(&*frame_ptr.audio_data().begin()),
 984                                         src_av_frame->channels,
 985                                         src_av_frame->nb_samples,
 986                                         static_cast<AVSampleFormat>(src_av_frame->format),
 987                                         16));
 988
 989                         FF(av_buffersrc_add_frame(
 990                                         audio_graph_in_,
 991                                         src_av_frame.get()));
 992                 }
 993
 994                 int ret = 0;
 995
 996                 while(ret >= 0)
 997                 {
 998                         std::shared_ptr<AVFrame> filt_frame(
 999                                 av_frame_alloc(),
1000                                 [](AVFrame* p)
1001                                 {
1002                                         av_frame_free(&p);
1003                                 });
1004
1005                         ret = av_buffersink_get_frame(
1006                                 audio_graph_out_,
1007                                 filt_frame.get());
1008
1009                         audio_encoder_executor_.begin_invoke([=]
1010                         {
1011                                 if(ret == AVERROR_EOF)
1012                                 {
1013                                         if(enc->codec->capabilities & CODEC_CAP_DELAY)
1014                                         {
1015                                                 while(encode_av_frame(
1016                                                                 *audio_st_,
1017                                                                 audio_bitstream_filter_.get(),
1018                                                                 avcodec_encode_audio2,
1019                                                                 nullptr,
1020                                                                 token))
1021                                                 {
1022                                                         boost::this_thread::yield(); // TODO:
1023                                                 }
1024                                         }
1025                                 }
1026                                 else if(ret != AVERROR(EAGAIN))
1027                                 {
1028                                         FF_RET(
1029                                                 ret,
1030                                                 "av_buffersink_get_frame");
1031
1032                                         encode_av_frame(
1033                                                 *audio_st_,
1034                                                 audio_bitstream_filter_.get(),
1035                                                 avcodec_encode_audio2,
1036                                                 filt_frame,
1037                                                 token);
1038
1039                                         boost::this_thread::yield(); // TODO:
1040                                 }
1041                         });
1042                 }
1043         }
1044
1045         template<typename F>
1046         bool encode_av_frame(
1047                         AVStream& st,
1048                         AVBitStreamFilterContext* bsfc,
1049                         const F& func,
1050                         const std::shared_ptr<AVFrame>& src_av_frame,
1051                         std::shared_ptr<void> token)
1052         {
1053                 AVPacket pkt = {};
1054                 av_init_packet(&pkt);
1055
1056                 int got_packet = 0;
1057
1058                 FF(func(
1059                         st.codec,
1060                         &pkt,
1061                         src_av_frame.get(),
1062                         &got_packet));
1063
1064                 if(!got_packet || pkt.size <= 0)
1065                         return false;
1066
1067                 pkt.stream_index = st.index;
1068
1069                 if(bsfc)
1070                 {
1071                         auto new_pkt = pkt;
1072
1073                         auto a = av_bitstream_filter_filter(
1074                                         bsfc,
1075                                         st.codec,
1076                                         nullptr,
1077                                         &new_pkt.data,
1078                                         &new_pkt.size,
1079                                         pkt.data,
1080                                         pkt.size,
1081                                         pkt.flags & AV_PKT_FLAG_KEY);
1082
1083                         if(a == 0 && new_pkt.data != pkt.data && new_pkt.destruct)
1084                         {
1085                                 auto t = reinterpret_cast<std::uint8_t*>(av_malloc(new_pkt.size + FF_INPUT_BUFFER_PADDING_SIZE));
1086
1087                                 if(t)
1088                                 {
1089                                         memcpy(
1090                                                 t,
1091                                                 new_pkt.data,
1092                                                 new_pkt.size);
1093
1094                                         memset(
1095                                                 t + new_pkt.size,
1096                                                 0,
1097                                                 FF_INPUT_BUFFER_PADDING_SIZE);
1098
1099                                         new_pkt.data = t;
1100                                         new_pkt.buf  = nullptr;
1101                                 }
1102                                 else
1103                                         a = AVERROR(ENOMEM);
1104                         }
1105
1106                         av_free_packet(&pkt);
1107
1108                         FF_RET(
1109                                 a,
1110                                 "av_bitstream_filter_filter");
1111
1112                         new_pkt.buf =
1113                                 av_buffer_create(
1114                                         new_pkt.data,
1115                                         new_pkt.size,
1116                                         av_buffer_default_free,
1117                                         nullptr,
1118                                         0);
1119
1120                         CASPAR_VERIFY(new_pkt.buf);
1121
1122                         pkt = new_pkt;
1123                 }
1124
1125                 if (pkt.pts != AV_NOPTS_VALUE)
1126                 {
1127                         pkt.pts =
1128                                 av_rescale_q(
1129                                         pkt.pts,
1130                                         st.codec->time_base,
1131                                         st.time_base);
1132                 }
1133
1134                 if (pkt.dts != AV_NOPTS_VALUE)
1135                 {
1136                         pkt.dts =
1137                                 av_rescale_q(
1138                                         pkt.dts,
1139                                         st.codec->time_base,
1140                                         st.time_base);
1141                 }
1142
1143                 pkt.duration =
1144                         static_cast<int>(
1145                                 av_rescale_q(
1146                                         pkt.duration,
1147                                         st.codec->time_base, st.time_base));
1148
1149                 write_packet(
1150                         std::shared_ptr<AVPacket>(
1151                                 new AVPacket(pkt),
1152                                 [](AVPacket* p)
1153                                 {
1154                                         av_free_packet(p);
1155                                         delete p;
1156                                 }), token);
1157
1158                 return true;
1159         }
1160
1161         void write_packet(
1162                         const std::shared_ptr<AVPacket>& pkt_ptr,
1163                         std::shared_ptr<void> token)
1164         {
1165                 write_executor_.begin_invoke([this, pkt_ptr, token]() mutable
1166                 {
1167                         FF(av_interleaved_write_frame(
1168                                 oc_.get(),
1169                                 pkt_ptr.get()));
1170                 });
1171         }
1172
1173         template<typename T>
1174         static boost::optional<T> try_remove_arg(
1175                         std::map<std::string, std::string>& options,
1176                         const boost::regex& expr)
1177         {
1178                 for(auto it = options.begin(); it != options.end(); ++it)
1179                 {
1180                         if(boost::regex_search(it->first, expr))
1181                         {
1182                                 auto arg = it->second;
1183                                 options.erase(it);
1184                                 return boost::lexical_cast<T>(arg);
1185                         }
1186                 }
1187
1188                 return boost::optional<T>();
1189         }
1190
1191         static std::map<std::string, std::string> remove_options(
1192                         std::map<std::string, std::string>& options,
1193                         const boost::regex& expr)
1194         {
1195                 std::map<std::string, std::string> result;
1196
1197                 auto it = options.begin();
1198                 while(it != options.end())
1199                 {
1200                         boost::smatch what;
1201                         if(boost::regex_search(it->first, what, expr))
1202                         {
1203                                 result[
1204                                         what.size() > 0 && what[1].matched
1205                                                 ? what[1].str()
1206                                                 : it->first] = it->second;
1207                                 it = options.erase(it);
1208                         }
1209                         else
1210                                 ++it;
1211                 }
1212
1213                 return result;
1214         }
1215
1216         static void to_dict(AVDictionary** dest, const std::map<std::string, std::string>& c)
1217         {
1218                 for (const auto& entry : c)
1219                 {
1220                         av_dict_set(
1221                                 dest,
1222                                 entry.first.c_str(),
1223                                 entry.second.c_str(), 0);
1224                 }
1225         }
1226
1227         static std::map<std::string, std::string> to_map(AVDictionary* dict)
1228         {
1229                 std::map<std::string, std::string> result;
1230
1231                 for(auto t = dict
1232                                 ? av_dict_get(
1233                                         dict,
1234                                         "",
1235                                         nullptr,
1236                                         AV_DICT_IGNORE_SUFFIX)
1237                                 : nullptr;
1238                         t;
1239                         t = av_dict_get(
1240                                 dict,
1241                                 "",
1242                                 t,
1243                                 AV_DICT_IGNORE_SUFFIX))
1244                 {
1245                         result[t->key] = t->value;
1246                 }
1247
1248                 return result;
1249         }
1250 };
1251
1252 spl::shared_ptr<core::frame_consumer> create_streaming_consumer(
1253                 const std::vector<std::wstring>& params, core::interaction_sink*)
1254 {
1255         if (params.size() < 1 || params.at(0) != L"STREAM")
1256                 return core::frame_consumer::empty();
1257
1258         auto path = u8(params.at(1));
1259         auto args = u8(boost::join(params, L" "));
1260
1261         return spl::make_shared<streaming_consumer>(path, args);
1262 }
1263
1264 spl::shared_ptr<core::frame_consumer> create_preconfigured_streaming_consumer(
1265                 const boost::property_tree::wptree& ptree, core::interaction_sink*)
1266 {
1267         return spl::make_shared<streaming_consumer>(
1268                         u8(ptree.get<std::wstring>(L"path")),
1269                         u8(ptree.get<std::wstring>(L"args", L"")));
1270 }
1271
1272 }}