git.sesse.net Git - casparcg/blob - modules/ffmpeg/ffmpeg_pipeline_backend_internal.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Helge Norberg, helge.norberg@svt.se
  20 * Author: Robert Nagy, ronag89@gmail.com
  21 */
  22
  23 #include "StdAfx.h"
  24
  25 #include "ffmpeg_pipeline_backend.h"
  26 #include "ffmpeg_pipeline_backend_internal.h"
  27 #include "producer/input/input.h"
  28 #include "producer/video/video_decoder.h"
  29 #include "producer/audio/audio_decoder.h"
  30 #include "producer/filter/audio_filter.h"
  31 #include "producer/filter/filter.h"
  32 #include "producer/util/util.h"
  33 #include "ffmpeg_error.h"
  34 #include "ffmpeg.h"
  35
  36 #include <common/diagnostics/graph.h>
  37 #include <common/os/general_protection_fault.h>
  38 #include <common/enum_class.h>
  39
  40 #include <core/frame/audio_channel_layout.h>
  41 #include <core/frame/frame.h>
  42 #include <core/frame/frame_factory.h>
  43 #include <core/video_format.h>
  44
  45 #include <functional>
  46 #include <limits>
  47 #include <queue>
  48 #include <map>
  49
  50 #include <tbb/atomic.h>
  51 #include <tbb/concurrent_queue.h>
  52 #include <tbb/spin_mutex.h>
  53
  54 #include <boost/thread.hpp>
  55 #include <boost/optional.hpp>
  56 #include <boost/exception_ptr.hpp>
  57
  58 namespace caspar { namespace ffmpeg {
  59
  60 std::string to_string(const boost::rational<int>& framerate)
  61 {
  62         return boost::lexical_cast<std::string>(framerate.numerator())
  63                 + "/" + boost::lexical_cast<std::string>(framerate.denominator()) + " (" + boost::lexical_cast<std::string>(static_cast<double>(framerate.numerator()) / static_cast<double>(framerate.denominator())) + ") fps";
  64 }
  65
  66 std::vector<int> find_audio_cadence(const boost::rational<int>& framerate)
  67 {
  68         static std::map<boost::rational<int>, std::vector<int>> CADENCES_BY_FRAMERATE = []
  69         {
  70                 std::map<boost::rational<int>, std::vector<int>> result;
  71
  72                 for (core::video_format format : enum_constants<core::video_format>())
  73                 {
  74                         core::video_format_desc desc(format);
  75                         boost::rational<int> format_rate(desc.time_scale, desc.duration);
  76
  77                         result.insert(std::make_pair(format_rate, desc.audio_cadence));
  78                 }
  79
  80                 return result;
  81         }();
  82
  83         auto exact_match = CADENCES_BY_FRAMERATE.find(framerate);
  84
  85         if (exact_match != CADENCES_BY_FRAMERATE.end())
  86                 return exact_match->second;
  87
  88         boost::rational<int> closest_framerate_diff     = std::numeric_limits<int>::max();
  89         boost::rational<int> closest_framerate          = 0;
  90
  91         for (auto format_framerate : CADENCES_BY_FRAMERATE | boost::adaptors::map_keys)
  92         {
  93                 auto diff = boost::abs(framerate - format_framerate);
  94
  95                 if (diff < closest_framerate_diff)
  96                 {
  97                         closest_framerate_diff  = diff;
  98                         closest_framerate               = format_framerate;
  99                 }
 100         }
 101
 102         if (is_logging_quiet_for_thread())
 103                 CASPAR_LOG(debug) << "No exact audio cadence match found for framerate " << to_string(framerate)
 104                         << "\nClosest match is " << to_string(closest_framerate)
 105                         << "\nwhich is a " << to_string(closest_framerate_diff) << " difference.";
 106         else
 107                 CASPAR_LOG(warning) << "No exact audio cadence match found for framerate " << to_string(framerate)
 108                         << "\nClosest match is " << to_string(closest_framerate)
 109                         << "\nwhich is a " << to_string(closest_framerate_diff) << " difference.";
 110
 111         return CADENCES_BY_FRAMERATE[closest_framerate];
 112 }
 113
 114 struct source
 115 {
 116         virtual ~source() { }
 117
 118         virtual std::wstring                                                    print() const                                                                                   = 0;
 119         virtual void                                                                    start()                                                                                                 { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 120         virtual void                                                                    graph(spl::shared_ptr<caspar::diagnostics::graph> g)    { }
 121         virtual void                                                                    stop()                                                                                                  { }
 122         virtual void                                                                    start_frame(std::uint32_t frame)                                                { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 123         virtual std::uint32_t                                                   start_frame() const                                                                             { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 124         virtual void                                                                    loop(bool value)                                                                                { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 125         virtual bool                                                                    loop() const                                                                                    { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 126         virtual void                                                                    length(std::uint32_t frames)                                                    { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 127         virtual std::uint32_t                                                   length() const                                                                                  { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 128         virtual std::string                                                             filename() const                                                                                { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print())); }
 129         virtual void                                                                    seek(std::uint32_t frame)                                                               { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 130         virtual bool                                                                    has_audio() const                                                                               { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 131         virtual int                                                                             samplerate() const                                                                              { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 132         virtual bool                                                                    has_video() const                                                                               { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 133         virtual bool                                                                    eof() const                                                                                             { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 134         virtual boost::rational<int>                                    framerate() const                                                                               { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 135         virtual std::uint32_t                                                   frame_number() const                                                                    { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 136         virtual std::vector<std::shared_ptr<AVFrame>>   get_input_frames_for_streams(AVMediaType type)                  { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 137 };
 138
 139 struct no_source_selected : public source
 140 {
 141         std::wstring print() const override
 142         {
 143                 return L"[no_source_selected]";
 144         }
 145 };
 146
 147 class file_source : public source
 148 {
 149         std::wstring                                                            filename_;
 150         spl::shared_ptr<diagnostics::graph>                     graph_;
 151         std::uint32_t                                                           start_frame_    = 0;
 152         std::uint32_t                                                           length_                 = std::numeric_limits<std::uint32_t>::max();
 153         bool                                                                            loop_                   = false;
 154         mutable boost::mutex                                            pointer_mutex_;
 155         std::shared_ptr<input>                                          input_;
 156         std::vector<spl::shared_ptr<audio_decoder>>     audio_decoders_;
 157         std::shared_ptr<video_decoder>                          video_decoder_;
 158         bool                                                                            started_                = false;
 159 public:
 160         file_source(std::string filename)
 161                 : filename_(u16(filename))
 162         {
 163         }
 164
 165         std::wstring print() const override
 166         {
 167                 return L"[file_source " + filename_ + L"]";
 168         }
 169
 170         void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
 171         {
 172                 graph_ = std::move(g);
 173         }
 174
 175         void start() override
 176         {
 177                 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
 178                 bool thumbnail_mode = is_logging_quiet_for_thread();
 179                 input_.reset(new input(graph_, filename_, loop_, start_frame_, length_, thumbnail_mode));
 180
 181                 for (int i = 0; i < input_->num_audio_streams(); ++i)
 182                 {
 183                         try
 184                         {
 185                                 audio_decoders_.push_back(spl::make_shared<audio_decoder>(*input_, core::video_format::invalid, i));
 186                         }
 187                         catch (...)
 188                         {
 189                                 if (is_logging_quiet_for_thread())
 190                                 {
 191                                         CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
 192                                         CASPAR_LOG(info) << print() << " Failed to open audio-stream. Turn on log level debug to see more information.";
 193                                 }
 194                                 else
 195                                 {
 196                                         CASPAR_LOG_CURRENT_EXCEPTION();
 197                                         CASPAR_LOG(warning) << print() << " Failed to open audio-stream.";
 198                                 }
 199                         }
 200                 }
 201
 202                 if (audio_decoders_.empty())
 203                         CASPAR_LOG(debug) << print() << " No audio-stream found. Running without audio.";
 204
 205                 try
 206                 {
 207                         video_decoder_.reset(new video_decoder(*input_, false));
 208                 }
 209                 catch (averror_stream_not_found&)
 210                 {
 211                         CASPAR_LOG(debug) << print() << " No video-stream found. Running without video.";
 212                 }
 213                 catch (...)
 214                 {
 215                         if (is_logging_quiet_for_thread())
 216                         {
 217                                 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
 218                                 CASPAR_LOG(info) << print() << " Failed to open video-stream. Running without audio. Turn on log level debug to see more information.";
 219                         }
 220                         else
 221                         {
 222                                 CASPAR_LOG_CURRENT_EXCEPTION();
 223                                 CASPAR_LOG(warning) << print() << " Failed to open video-stream. Running without audio.";
 224                         }
 225                 }
 226
 227                 started_ = true;
 228         }
 229
 230         void stop() override
 231         {
 232                 started_ = false;
 233         }
 234
 235         void start_frame(std::uint32_t frame) override
 236         {
 237                 start_frame_ = frame;
 238
 239                 auto i = get_input();
 240                 if (i)
 241                         i->start(frame);
 242         }
 243
 244         std::uint32_t start_frame() const override
 245         {
 246                 return start_frame_;
 247         }
 248
 249         void loop(bool value) override
 250         {
 251                 loop_ = value;
 252
 253                 auto i = get_input();
 254                 if (i)
 255                         i->loop(value);
 256         }
 257
 258         bool loop() const override
 259         {
 260                 return loop_;
 261         }
 262
 263         void length(std::uint32_t frames) override
 264         {
 265                 length_ = frames;
 266
 267                 auto i = get_input();
 268                 if (i)
 269                         i->length(frames);
 270         }
 271
 272         std::uint32_t length() const override
 273         {
 274                 auto v = get_video_decoder();
 275
 276                 if (v)
 277                         return v->nb_frames();
 278
 279                 auto a = get_audio_decoders();
 280
 281                 if (!a.empty())
 282                         return a.at(0)->nb_frames(); // Should be ok.
 283
 284                 return length_;
 285         }
 286
 287         std::string filename() const override
 288         {
 289                 return u8(filename_);
 290         }
 291
 292         void seek(std::uint32_t frame) override
 293         {
 294                 expect_started();
 295                 get_input()->seek(frame);
 296         }
 297
 298         bool eof() const override
 299         {
 300                 auto i = get_input();
 301                 return !i || i->eof();
 302         }
 303
 304         bool has_audio() const override
 305         {
 306                 return !get_audio_decoders().empty();
 307         }
 308
 309         int samplerate() const override
 310         {
 311                 if (get_audio_decoders().empty())
 312                         return -1;
 313
 314                 return 48000;
 315         }
 316
 317         bool has_video() const override
 318         {
 319                 return static_cast<bool>(get_video_decoder());
 320         }
 321
 322         boost::rational<int> framerate() const override
 323         {
 324                 auto decoder = get_video_decoder();
 325
 326                 if (!decoder)
 327                         return -1;
 328
 329                 return decoder->framerate();
 330         }
 331
 332         std::uint32_t frame_number() const override
 333         {
 334                 auto decoder = get_video_decoder();
 335
 336                 if (!decoder)
 337                         return 0;
 338
 339                 return decoder->file_frame_number();
 340         }
 341
 342         std::vector<std::shared_ptr<AVFrame>> get_input_frames_for_streams(AVMediaType type) override
 343         {
 344                 auto a_decoders = get_audio_decoders();
 345                 auto v_decoder  = get_video_decoder();
 346                 expect_started();
 347
 348                 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && !a_decoders.empty())
 349                 {
 350                         std::vector<std::shared_ptr<AVFrame>> frames;
 351
 352                         for (auto& a_decoder : a_decoders)
 353                         {
 354                                 std::shared_ptr<AVFrame> frame;
 355
 356                                 for (int i = 0; i < 64; ++i)
 357                                 {
 358                                         frame = (*a_decoder)();
 359
 360                                         if (frame == flush() || (frame && frame->data[0]))
 361                                                 break;
 362                                         else
 363                                                 frame.reset();
 364                                 }
 365
 366                                 frames.push_back(std::move(frame));
 367                         }
 368
 369                         return frames;
 370                 }
 371                 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO && v_decoder)
 372                 {
 373                         std::shared_ptr<AVFrame> frame;
 374
 375                         for (int i = 0; i < 128; ++i)
 376                         {
 377                                 frame = (*v_decoder)();
 378
 379                                 if (frame == flush() || (frame && frame->data[0]))
 380                                         return { frame };
 381                         }
 382                 }
 383                 else
 384                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(
 385                                 print() + L" Unhandled media type " + boost::lexical_cast<std::wstring>(type)));
 386
 387                 return { };
 388         }
 389 private:
 390         void expect_started() const
 391         {
 392                 if (!started_)
 393                         CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" Not started."));
 394         }
 395
 396         std::shared_ptr<input> get_input() const
 397         {
 398                 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
 399                 return input_;
 400         }
 401
 402         std::vector<spl::shared_ptr<audio_decoder>> get_audio_decoders() const
 403         {
 404                 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
 405                 return audio_decoders_;
 406         }
 407
 408         std::shared_ptr<video_decoder> get_video_decoder() const
 409         {
 410                 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
 411                 return video_decoder_;
 412         }
 413 };
 414
 415 class memory_source : public source
 416 {
 417         int                                                                                                                     samplerate_             = -1;
 418         int                                                                                                                     num_channels_   = -1;
 419         int                                                                                                                     width_                  = -1;
 420         int                                                                                                                     height_                 = -1;
 421         boost::rational<int>                                                                            framerate_              = -1;
 422
 423         tbb::atomic<bool>                                                                                       running_;
 424         tbb::concurrent_bounded_queue<caspar::array<const int32_t>>     audio_frames_;
 425         tbb::concurrent_bounded_queue<caspar::array<const uint8_t>>     video_frames_;
 426         int64_t                                                                                                         audio_pts_              = 0;
 427         int64_t                                                                                                         video_pts_              = 0;
 428 public:
 429         memory_source()
 430         {
 431                 running_ = false;
 432                 video_frames_.set_capacity(1);
 433                 audio_frames_.set_capacity(1);
 434         }
 435
 436         ~memory_source()
 437         {
 438                 stop();
 439         }
 440
 441         void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
 442         {
 443         }
 444
 445         std::wstring print() const override
 446         {
 447                 return L"[memory_source]";
 448         }
 449
 450         void enable_audio(int samplerate, int num_channels)
 451         {
 452                 samplerate_ = samplerate;
 453                 num_channels_ = num_channels;
 454         }
 455
 456         void enable_video(int width, int height, boost::rational<int> framerate)
 457         {
 458                 width_ = width;
 459                 height_ = height;
 460         }
 461
 462         void start() override
 463         {
 464                 running_ = true;
 465         }
 466
 467         void stop() override
 468         {
 469                 running_ = false;
 470                 video_frames_.try_push(caspar::array<const uint8_t>());
 471                 audio_frames_.try_push(caspar::array<const int32_t>());
 472         }
 473
 474         bool has_audio() const override
 475         {
 476                 return samplerate_ != -1;
 477         }
 478
 479         int samplerate() const override
 480         {
 481                 return samplerate_;
 482         }
 483
 484         bool has_video() const override
 485         {
 486                 return width_ != -1;
 487         }
 488
 489         bool eof() const override
 490         {
 491                 return !running_;
 492         }
 493
 494         boost::rational<int> framerate() const override
 495         {
 496                 return framerate_;
 497         }
 498
 499         bool try_push_audio(caspar::array<const std::int32_t> data)
 500         {
 501                 if (!has_audio())
 502                         CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" audio not enabled."));
 503
 504                 if (data.empty() || data.size() % num_channels_ != 0)
 505                         CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info(print() + L" audio with incorrect number of channels submitted."));
 506
 507                 return audio_frames_.try_push(std::move(data));
 508         }
 509
 510         bool try_push_video(caspar::array<const std::uint8_t> data)
 511         {
 512                 if (!has_video())
 513                         CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" video not enabled."));
 514
 515                 if (data.size() != width_ * height_ * 4)
 516                         CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info(print() + L" video with incorrect size submitted."));
 517
 518                 return video_frames_.try_push(std::move(data));
 519         }
 520
 521         std::vector<std::shared_ptr<AVFrame>> get_input_frames_for_streams(AVMediaType type) override
 522         {
 523                 if (!running_)
 524                         CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not running."));
 525
 526                 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && has_audio())
 527                 {
 528                         caspar::array<const std::int32_t> samples;
 529                         audio_frames_.pop(samples);
 530
 531                         if (samples.empty())
 532                                 return { };
 533
 534                         spl::shared_ptr<AVFrame> av_frame(av_frame_alloc(), [samples](AVFrame* p) { av_frame_free(&p); });
 535
 536                         av_frame->channels                      = num_channels_;
 537                         av_frame->channel_layout        = av_get_default_channel_layout(num_channels_);
 538                         av_frame->sample_rate           = samplerate_;
 539                         av_frame->nb_samples            = static_cast<int>(samples.size()) / num_channels_;
 540                         av_frame->format                        = AV_SAMPLE_FMT_S32;
 541                         av_frame->pts                           = audio_pts_;
 542
 543                         audio_pts_ += av_frame->nb_samples;
 544
 545                         FF(av_samples_fill_arrays(
 546                                         av_frame->extended_data,
 547                                         av_frame->linesize,
 548                                         reinterpret_cast<const std::uint8_t*>(&*samples.begin()),
 549                                         av_frame->channels,
 550                                         av_frame->nb_samples,
 551                                         static_cast<AVSampleFormat>(av_frame->format),
 552                                         16));
 553
 554                         return { av_frame };
 555                 }
 556                 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO && has_video())
 557                 {
 558                         caspar::array<const std::uint8_t> data;
 559                         video_frames_.pop(data);
 560
 561                         if (data.empty())
 562                                 return {};
 563
 564                         spl::shared_ptr<AVFrame> av_frame(av_frame_alloc(), [data](AVFrame* p) { av_frame_free(&p); });
 565                         avcodec_get_frame_defaults(av_frame.get());
 566
 567                         const auto sample_aspect_ratio = boost::rational<int>(width_, height_);
 568
 569                         av_frame->format                                  = AV_PIX_FMT_BGRA;
 570                         av_frame->width                                   = width_;
 571                         av_frame->height                                  = height_;
 572                         av_frame->sample_aspect_ratio.num = sample_aspect_ratio.numerator();
 573                         av_frame->sample_aspect_ratio.den = sample_aspect_ratio.denominator();
 574                         av_frame->pts                                     = video_pts_;
 575
 576                         video_pts_ += 1;
 577
 578                         FF(av_image_fill_arrays(
 579                                         av_frame->data,
 580                                         av_frame->linesize,
 581                                         data.begin(),
 582                                         static_cast<AVPixelFormat>(av_frame->format),
 583                                         width_,
 584                                         height_,
 585                                         1));
 586
 587                         return { av_frame };
 588                 }
 589                 else
 590                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(
 591                                 print() + L" Unhandled media type " + boost::lexical_cast<std::wstring>(type)));
 592         }
 593 };
 594
 595 struct sink
 596 {
 597         virtual ~sink() { }
 598
 599         virtual std::wstring                                    print() const                                                                                                                                   = 0;
 600         virtual void                                                    graph(spl::shared_ptr<caspar::diagnostics::graph> g)                                                    { }
 601         virtual void                                                    acodec(std::string codec)                                                                                                               { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
 602         virtual void                                                    vcodec(std::string codec)                                                                                                               { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
 603         virtual void                                                    format(std::string fmt)                                                                                                                 { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
 604         virtual void                                                    framerate(boost::rational<int> framerate)                                                                               { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
 605         virtual void                                                    start(bool has_audio, bool has_video)                                                                                   { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 606         virtual void                                                    stop()                                                                                                                                                  { }
 607         virtual void                                                    flush_all()                                                                                                                                             { }
 608         virtual std::vector<AVSampleFormat>             supported_sample_formats() const                                                                                                { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 609         virtual std::vector<int>                                supported_samplerates() const                                                                                                   { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 610         virtual std::vector<AVPixelFormat>              supported_pixel_formats() const                                                                                                 { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 611         virtual int                                                             wanted_num_audio_streams() const                                                                                                { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 612         virtual boost::optional<int>                    wanted_num_channels_per_stream() const                                                                                  { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 613         virtual boost::optional<AVMediaType>    try_push(AVMediaType type, int stream_index, spl::shared_ptr<AVFrame> frame)    { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 614         virtual void                                                    eof()                                                                                                                                                   { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 615 };
 616
 617 struct no_sink_selected : public sink
 618 {
 619         std::wstring print() const override
 620         {
 621                 return L"[no_sink_selected]";
 622         }
 623 };
 624
 625 class file_sink : public sink
 626 {
 627         std::wstring                                            filename_;
 628         spl::shared_ptr<diagnostics::graph>     graph_;
 629 public:
 630         file_sink(std::string filename)
 631                 : filename_(u16(std::move(filename)))
 632         {
 633         }
 634
 635         std::wstring print() const override
 636         {
 637                 return L"[file_sink " + filename_ + L"]";
 638         }
 639
 640         void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
 641         {
 642                 graph_ = std::move(g);
 643         }
 644 };
 645
 646 class memory_sink : public sink
 647 {
 648         spl::shared_ptr<core::frame_factory>                    factory_;
 649
 650         bool                                                                                    has_audio_                      = false;
 651         bool                                                                                    has_video_                      = false;
 652         std::vector<int>                                                                audio_cadence_;
 653         core::audio_channel_layout                                              channel_layout_         = core::audio_channel_layout::invalid();
 654         core::mutable_audio_buffer                                              audio_samples_;
 655
 656         std::queue<std::shared_ptr<AVFrame>>                    video_frames_;
 657         std::shared_ptr<AVFrame>                                                last_video_frame_;
 658
 659         tbb::concurrent_bounded_queue<core::draw_frame> output_frames_;
 660         tbb::atomic<bool>                                                               running_;
 661 public:
 662         memory_sink(spl::shared_ptr<core::frame_factory> factory, core::video_format_desc format)
 663                 : factory_(std::move(factory))
 664                 , audio_cadence_(format.audio_cadence)
 665         {
 666                 output_frames_.set_capacity(2);
 667                 running_ = false;
 668                 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
 669                 // This cadence fills the audio mixer most optimally.
 670                 boost::range::rotate(audio_cadence_, std::end(audio_cadence_) - 1);
 671         }
 672
 673         ~memory_sink()
 674         {
 675                 stop();
 676         }
 677
 678         std::wstring print() const override
 679         {
 680                 return L"[memory_sink]";
 681         }
 682
 683         void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
 684         {
 685         }
 686
 687         void framerate(boost::rational<int> framerate) override
 688         {
 689                 audio_cadence_ = find_audio_cadence(framerate);
 690                 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
 691                 // This cadence fills the audio mixer most optimally.
 692                 boost::range::rotate(audio_cadence_, std::end(audio_cadence_) - 1);
 693         }
 694
 695         void start(bool has_audio, bool has_video) override
 696         {
 697                 has_audio_      = has_audio;
 698                 has_video_      = has_video;
 699                 running_        = true;
 700         }
 701
 702         void stop() override
 703         {
 704                 running_ = false;
 705                 output_frames_.set_capacity(4);
 706         }
 707
 708         std::vector<AVSampleFormat> supported_sample_formats() const override
 709         {
 710                 return { AVSampleFormat::AV_SAMPLE_FMT_S32 };
 711         }
 712
 713         std::vector<int> supported_samplerates() const override {
 714                 return { 48000 };
 715         }
 716
 717         std::vector<AVPixelFormat> supported_pixel_formats() const override
 718         {
 719                 return {
 720                         AVPixelFormat::AV_PIX_FMT_YUVA420P,
 721                         AVPixelFormat::AV_PIX_FMT_YUV444P,
 722                         AVPixelFormat::AV_PIX_FMT_YUV422P,
 723                         AVPixelFormat::AV_PIX_FMT_YUV420P,
 724                         AVPixelFormat::AV_PIX_FMT_YUV411P,
 725                         AVPixelFormat::AV_PIX_FMT_BGRA,
 726                         AVPixelFormat::AV_PIX_FMT_ARGB,
 727                         AVPixelFormat::AV_PIX_FMT_RGBA,
 728                         AVPixelFormat::AV_PIX_FMT_ABGR,
 729                         AVPixelFormat::AV_PIX_FMT_GRAY8
 730                 };
 731         }
 732
 733         int wanted_num_audio_streams() const override
 734         {
 735                 return 1;
 736         }
 737
 738         boost::optional<int> wanted_num_channels_per_stream() const
 739         {
 740                 return boost::none;
 741         }
 742
 743         void flush_all() override
 744         {
 745                 audio_samples_.clear();
 746
 747                 while (!video_frames_.empty())
 748                         video_frames_.pop();
 749         }
 750
 751         boost::optional<AVMediaType> try_push(AVMediaType type, int stream_index, spl::shared_ptr<AVFrame> av_frame) override
 752         {
 753                 if (!has_audio_ && !has_video_)
 754                         CASPAR_THROW_EXCEPTION(invalid_operation());
 755
 756                 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && av_frame->data[0])
 757                 {
 758                         if (channel_layout_ == core::audio_channel_layout::invalid()) // First audio
 759                         {
 760                                 channel_layout_ = get_audio_channel_layout(av_frame->channels, av_frame->channel_layout, L"");
 761
 762                                 // Insert silence samples so that the audio mixer is guaranteed to be filled.
 763                                 auto min_num_samples_per_frame  = *boost::min_element(audio_cadence_);
 764                                 auto max_num_samples_per_frame  = *boost::max_element(audio_cadence_);
 765                                 auto cadence_safety_samples             = max_num_samples_per_frame - min_num_samples_per_frame;
 766                                 audio_samples_.resize(channel_layout_.num_channels * cadence_safety_samples, 0);
 767                         }
 768
 769                         auto ptr = reinterpret_cast<int32_t*>(av_frame->data[0]);
 770
 771                         audio_samples_.insert(audio_samples_.end(), ptr, ptr + av_frame->linesize[0] / sizeof(int32_t));
 772                 }
 773                 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO)
 774                 {
 775                         video_frames_.push(std::move(av_frame));
 776                 }
 777
 778                 while (true)
 779                 {
 780                         bool enough_audio =
 781                                 !has_audio_ ||
 782                                 (channel_layout_ != core::audio_channel_layout::invalid() && audio_samples_.size() >= audio_cadence_.front() * channel_layout_.num_channels);
 783                         bool enough_video =
 784                                 !has_video_ ||
 785                                 !video_frames_.empty();
 786
 787                         if (!enough_audio)
 788                                 return AVMediaType::AVMEDIA_TYPE_AUDIO;
 789
 790                         if (!enough_video)
 791                                 return AVMediaType::AVMEDIA_TYPE_VIDEO;
 792
 793                         core::mutable_audio_buffer audio_data;
 794
 795                         if (has_audio_)
 796                         {
 797                                 auto begin = audio_samples_.begin();
 798                                 auto end = begin + audio_cadence_.front() * channel_layout_.num_channels;
 799
 800                                 audio_data.insert(audio_data.begin(), begin, end);
 801                                 audio_samples_.erase(begin, end);
 802                                 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_) + 1);
 803                         }
 804
 805                         if (!has_video_) // Audio only
 806                         {
 807                                 core::mutable_frame audio_only_frame(
 808                                                 { },
 809                                                 std::move(audio_data),
 810                                                 this,
 811                                                 core::pixel_format_desc(core::pixel_format::invalid),
 812                                                 channel_layout_);
 813
 814                                 output_frames_.push(core::draw_frame(std::move(audio_only_frame)));
 815
 816                                 return AVMediaType::AVMEDIA_TYPE_AUDIO;
 817                         }
 818
 819                         auto output_frame = make_frame(this, spl::make_shared_ptr(video_frames_.front()), *factory_, channel_layout_);
 820                         last_video_frame_ = video_frames_.front();
 821                         video_frames_.pop();
 822                         output_frame.audio_data() = std::move(audio_data);
 823
 824                         output_frames_.push(core::draw_frame(std::move(output_frame)));
 825                 }
 826         }
 827
 828         void eof() override
 829         {
 830                 // Drain rest, regardless of it being enough or not.
 831                 while (!video_frames_.empty() || !audio_samples_.empty())
 832                 {
 833                         core::mutable_audio_buffer audio_data;
 834
 835                         audio_data.swap(audio_samples_);
 836
 837                         if (video_frames_.empty() && !audio_data.empty() && last_video_frame_) // More audio samples than video
 838                         {
 839                                 video_frames_.push(last_video_frame_);
 840                         }
 841
 842                         if (!video_frames_.empty())
 843                         {
 844                                 auto output_frame = make_frame(this, spl::make_shared_ptr(video_frames_.front()), *factory_, channel_layout_);
 845                                 last_video_frame_ = video_frames_.front();
 846                                 video_frames_.pop();
 847                                 output_frame.audio_data() = std::move(audio_data);
 848
 849                                 output_frames_.push(core::draw_frame(std::move(output_frame)));
 850                         }
 851                         else
 852                         {
 853                                 core::mutable_frame audio_only_frame(
 854                                                 {},
 855                                                 std::move(audio_data),
 856                                                 this,
 857                                                 core::pixel_format_desc(core::pixel_format::invalid),
 858                                                 channel_layout_);
 859
 860                                 output_frames_.push(core::draw_frame(std::move(audio_only_frame)));
 861                                 output_frames_.push(core::draw_frame::empty());
 862                         }
 863                 }
 864         }
 865
 866         core::draw_frame try_pop_frame()
 867         {
 868                 core::draw_frame frame = core::draw_frame::late();
 869
 870                 if (!output_frames_.try_pop(frame) && !running_)
 871                         return core::draw_frame::empty();
 872
 873                 return frame;
 874         }
 875 };
 876
 877 struct audio_stream_info
 878 {
 879         int                             num_channels    = 0;
 880         AVSampleFormat  sampleformat    = AVSampleFormat::AV_SAMPLE_FMT_NONE;
 881         uint64_t                channel_layout  = 0;
 882 };
 883
 884 struct video_stream_info
 885 {
 886         int                                     width           = 0;
 887         int                                     height          = 0;
 888         AVPixelFormat           pixelformat     = AVPixelFormat::AV_PIX_FMT_NONE;
 889         core::field_mode        fieldmode       = core::field_mode::progressive;
 890 };
 891
 892 class ffmpeg_pipeline_backend_internal : public ffmpeg_pipeline_backend
 893 {
 894         spl::shared_ptr<diagnostics::graph>                                                             graph_;
 895
 896         spl::unique_ptr<source>                                                                                 source_                                 = spl::make_unique<no_source_selected>();
 897         std::function<bool (caspar::array<const std::int32_t> data)>    try_push_audio_;
 898         std::function<bool (caspar::array<const std::uint8_t> data)>    try_push_video_;
 899
 900         std::vector<audio_stream_info>                                                                  source_audio_streams_;
 901         video_stream_info                                                                                               source_video_stream_;
 902
 903         std::string                                                                                                             afilter_;
 904         std::unique_ptr<audio_filter>                                                                   audio_filter_;
 905         std::string                                                                                                             vfilter_;
 906         std::unique_ptr<filter>                                                                                 video_filter_;
 907
 908         spl::unique_ptr<sink>                                                                                   sink_                                   = spl::make_unique<no_sink_selected>();
 909         std::function<core::draw_frame ()>                                                              try_pop_frame_;
 910
 911         tbb::atomic<bool>                                                                                               started_;
 912         tbb::spin_mutex                                                                                                 exception_mutex_;
 913         boost::exception_ptr                                                                                    exception_;
 914         boost::thread                                                                                                   thread_;
 915 public:
 916         ffmpeg_pipeline_backend_internal()
 917         {
 918                 started_ = false;
 919                 diagnostics::register_graph(graph_);
 920         }
 921
 922         ~ffmpeg_pipeline_backend_internal()
 923         {
 924                 stop();
 925         }
 926
 927         void throw_if_error()
 928         {
 929                 boost::lock_guard<tbb::spin_mutex> lock(exception_mutex_);
 930
 931                 if (exception_ != nullptr)
 932                         boost::rethrow_exception(exception_);
 933         }
 934
 935         void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
 936         {
 937                 graph_ = std::move(g);
 938                 source_->graph(graph_);
 939                 sink_->graph(graph_);
 940         }
 941
 942         // Source setup
 943
 944         void from_file(std::string filename) override
 945         {
 946                 source_                 = spl::make_unique<file_source>(std::move(filename));
 947                 try_push_audio_ = std::function<bool (caspar::array<const std::int32_t>)>();
 948                 try_push_video_ = std::function<bool (caspar::array<const std::uint8_t>)>();
 949                 source_->graph(graph_);
 950         }
 951
 952         void from_memory_only_audio(int num_channels, int samplerate) override
 953         {
 954                 auto source             = spl::make_unique<memory_source>();
 955                 auto source_ptr = source.get();
 956                 try_push_audio_ = [this, source_ptr](caspar::array<const std::int32_t> data) { return source_ptr->try_push_audio(std::move(data)); };
 957                 source->enable_audio(samplerate, num_channels);
 958
 959                 source_ = std::move(source);
 960                 source_->graph(graph_);
 961         }
 962
 963         void from_memory_only_video(int width, int height, boost::rational<int> framerate) override
 964         {
 965                 auto source             = spl::make_unique<memory_source>();
 966                 auto source_ptr = source.get();
 967                 try_push_video_ = [this, source_ptr](caspar::array<const std::uint8_t> data) { return source_ptr->try_push_video(std::move(data)); };
 968                 source->enable_video(width, height, std::move(framerate));
 969
 970                 source_ = std::move(source);
 971                 source_->graph(graph_);
 972         }
 973
 974         void from_memory(int num_channels, int samplerate, int width, int height, boost::rational<int> framerate) override
 975         {
 976                 auto source             = spl::make_unique<memory_source>();
 977                 auto source_ptr = source.get();
 978                 try_push_audio_ = [this, source_ptr](caspar::array<const std::int32_t> data) { return source_ptr->try_push_audio(std::move(data)); };
 979                 try_push_video_ = [this, source_ptr](caspar::array<const std::uint8_t> data) { return source_ptr->try_push_video(std::move(data)); };
 980                 source->enable_audio(samplerate, num_channels);
 981                 source->enable_video(width, height, std::move(framerate));
 982
 983                 source_ = std::move(source);
 984                 source_->graph(graph_);
 985         }
 986
 987         void                    start_frame(std::uint32_t frame) override       { source_->start_frame(frame);          }
 988         std::uint32_t   start_frame() const override                            { return source_->start_frame();        }
 989         void                    length(std::uint32_t frames) override           { source_->length(frames);                      }
 990         std::uint32_t   length() const override                                         { return source_->length();                     }
 991         void                    seek(std::uint32_t frame) override                      { source_->seek(frame);                         }
 992         void                    loop(bool value) override                                       { source_->loop(value);                         }
 993         bool                    loop() const override                                           { return source_->loop();                       }
 994         std::string             source_filename() const override                        { return source_->filename();           }
 995
 996         // Filter setup
 997
 998         void vfilter(std::string filter) override
 999         {
1000                 vfilter_ = std::move(filter);
1001         }
1002
1003         void afilter(std::string filter) override
1004         {
1005                 afilter_ = std::move(filter);
1006         }
1007
1008         int width() const override
1009         {
1010                 return source_video_stream_.width;
1011         }
1012
1013         int height() const override
1014         {
1015                 return source_video_stream_.height;
1016         }
1017
1018         boost::rational<int> framerate() const override
1019         {
1020                 bool double_rate = filter::is_double_rate(u16(vfilter_));
1021
1022                 return double_rate ? source_->framerate() * 2 : source_->framerate();
1023         }
1024
1025         bool progressive() const override
1026         {
1027                 return true;//TODO
1028         }
1029
1030         // Sink setup
1031
1032         void to_memory(spl::shared_ptr<core::frame_factory> factory, core::video_format_desc format) override
1033         {
1034                 auto sink               = spl::make_unique<memory_sink>(std::move(factory), std::move(format));
1035                 auto sink_ptr   = sink.get();
1036                 try_pop_frame_  = [sink_ptr] { return sink_ptr->try_pop_frame(); };
1037
1038                 sink_ = std::move(sink);
1039                 sink_->graph(graph_);
1040         }
1041
1042         void to_file(std::string filename) override
1043         {
1044                 sink_                   = spl::make_unique<file_sink>(std::move(filename));
1045                 try_pop_frame_  = std::function<core::draw_frame ()>();
1046                 sink_->graph(graph_);
1047         }
1048
1049         void acodec(std::string codec) override { sink_->acodec(std::move(codec)); }
1050         void vcodec(std::string codec) override { sink_->vcodec(std::move(codec)); }
1051         void format(std::string fmt) override   { sink_->format(std::move(fmt)); }
1052
1053         // Runtime control
1054
1055         void start() override
1056         {
1057                 source_->start();
1058                 sink_->start(source_->has_audio(), source_->has_video());
1059                 started_ = true;
1060                 bool quiet = is_logging_quiet_for_thread();
1061
1062                 thread_ = boost::thread([=] { run(quiet); });
1063         }
1064
1065         bool try_push_audio(caspar::array<const std::int32_t> data) override
1066         {
1067                 throw_if_error();
1068
1069                 if (try_push_audio_)
1070                         return try_push_audio_(std::move(data));
1071                 else
1072                         return false;
1073         }
1074
1075         bool try_push_video(caspar::array<const std::uint8_t> data) override
1076         {
1077                 throw_if_error();
1078
1079                 if (try_push_video_)
1080                         return try_push_video_(std::move(data));
1081                 else
1082                         return false;
1083         }
1084
1085         core::draw_frame try_pop_frame() override
1086         {
1087                 throw_if_error();
1088
1089                 if (!try_pop_frame_)
1090                         CASPAR_THROW_EXCEPTION(invalid_operation());
1091
1092                 return try_pop_frame_();
1093         }
1094
1095         std::uint32_t last_frame() const override
1096         {
1097                 return source_->frame_number();
1098         }
1099
1100         bool started() const override
1101         {
1102                 return started_;
1103         }
1104
1105         void stop() override
1106         {
1107                 started_ = false;
1108
1109                 sink_->stop();
1110                 source_->stop();
1111
1112                 if (thread_.joinable())
1113                         thread_.join();
1114         }
1115
1116 private:
1117         void run(bool quiet)
1118         {
1119                 ensure_gpf_handler_installed_for_thread(u8(L"ffmpeg-pipeline: " + source_->print() + L" -> " + sink_->print()).c_str());
1120                 auto quiet_logging = temporary_enable_quiet_logging_for_thread(quiet);
1121
1122                 try
1123                 {
1124                         boost::optional<AVMediaType> result = source_->has_audio() ? AVMediaType::AVMEDIA_TYPE_AUDIO : AVMediaType::AVMEDIA_TYPE_VIDEO;
1125
1126                         while (started_ && (source_->has_audio() || source_->has_video()))
1127                         {
1128                                 auto needed                                             = *result;
1129                                 auto input_frames_for_streams   = source_->get_input_frames_for_streams(needed);
1130                                 bool flush_all                                  = !input_frames_for_streams.empty() && input_frames_for_streams.at(0) == flush();
1131
1132                                 if (flush_all)
1133                                 {
1134                                         sink_->flush_all();
1135
1136                                         if (source_->has_audio() && source_->has_video())
1137                                                 result = needed == AVMediaType::AVMEDIA_TYPE_AUDIO ? AVMediaType::AVMEDIA_TYPE_VIDEO : AVMediaType::AVMEDIA_TYPE_AUDIO;
1138
1139                                         continue;
1140                                 }
1141
1142                                 bool got_requested_media_type   = !input_frames_for_streams.empty() && input_frames_for_streams.at(0);
1143
1144                                 if (got_requested_media_type)
1145                                 {
1146                                         for (int input_stream_index = 0; input_stream_index < input_frames_for_streams.size(); ++input_stream_index)
1147                                         {
1148                                                 if (needed == AVMediaType::AVMEDIA_TYPE_AUDIO)
1149                                                 {
1150                                                         initialize_audio_filter_if_needed(input_frames_for_streams);
1151                                                         audio_filter_->push(input_stream_index, std::move(input_frames_for_streams.at(input_stream_index)));
1152
1153                                                         for (int output_stream_index = 0; output_stream_index < sink_->wanted_num_audio_streams(); ++output_stream_index)
1154                                                                 for (auto filtered_frame : audio_filter_->poll_all(output_stream_index))
1155                                                                         result = sink_->try_push(AVMediaType::AVMEDIA_TYPE_AUDIO, output_stream_index, std::move(filtered_frame));
1156                                                 }
1157                                                 else if (needed == AVMediaType::AVMEDIA_TYPE_VIDEO)
1158                                                 {
1159                                                         initialize_video_filter_if_needed(*input_frames_for_streams.at(input_stream_index));
1160                                                         video_filter_->push(std::move(input_frames_for_streams.at(input_stream_index)));
1161
1162                                                         for (auto filtered_frame : video_filter_->poll_all())
1163                                                                 result = sink_->try_push(AVMediaType::AVMEDIA_TYPE_VIDEO, 0, std::move(filtered_frame));
1164                                                 }
1165                                                 else
1166                                                         CASPAR_THROW_EXCEPTION(not_supported());
1167                                         }
1168                                 }
1169                                 else if (source_->eof())
1170                                 {
1171                                         started_ = false;
1172                                         sink_->eof();
1173                                         break;
1174                                 }
1175                                 else
1176                                         result = boost::none;
1177
1178                                 if (!result)
1179                                 {
1180                                         graph_->set_tag(caspar::diagnostics::tag_severity::WARNING, "dropped-frame");
1181                                         result = needed; // Repeat same media type
1182                                 }
1183                         }
1184                 }
1185                 catch (...)
1186                 {
1187                         if (is_logging_quiet_for_thread())
1188                         {
1189                                 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
1190                         }
1191                         else
1192                         {
1193                                 CASPAR_LOG_CURRENT_EXCEPTION();
1194                         }
1195
1196                         boost::lock_guard<tbb::spin_mutex> lock(exception_mutex_);
1197                         exception_ = boost::current_exception();
1198                 }
1199
1200                 video_filter_.reset();
1201                 audio_filter_.reset();
1202                 source_->stop();
1203                 sink_->stop();
1204                 started_ = false;
1205         }
1206
1207         template<typename T>
1208         void set_if_changed(bool& changed, T& old_value, T new_value)
1209         {
1210                 if (old_value != new_value)
1211                 {
1212                         changed = true;
1213                         old_value = new_value;
1214                 }
1215         }
1216
1217         void initialize_audio_filter_if_needed(const std::vector<std::shared_ptr<AVFrame>>& av_frames_per_stream)
1218         {
1219                 bool changed = av_frames_per_stream.size() != source_audio_streams_.size();
1220                 source_audio_streams_.resize(av_frames_per_stream.size());
1221
1222                 for (int i = 0; i < av_frames_per_stream.size(); ++i)
1223                 {
1224                         auto& av_frame  = *av_frames_per_stream.at(i);
1225                         auto& stream    = source_audio_streams_.at(i);
1226
1227                         auto channel_layout = av_frame.channel_layout == 0
1228                                         ? av_get_default_channel_layout(av_frame.channels)
1229                                         : av_frame.channel_layout;
1230
1231                         set_if_changed(changed, stream.sampleformat, static_cast<AVSampleFormat>(av_frame.format));
1232                         set_if_changed(changed, stream.num_channels, av_frame.channels);
1233                         set_if_changed(changed, stream.channel_layout, channel_layout);
1234                 }
1235
1236                 if (changed)
1237                         initialize_audio_filter();
1238         }
1239
1240         void initialize_audio_filter()
1241         {
1242                 std::vector<audio_input_pad> input_pads;
1243                 std::vector<audio_output_pad> output_pads;
1244
1245                 for (auto& source_audio_stream : source_audio_streams_)
1246                 {
1247                         input_pads.emplace_back(
1248                                         boost::rational<int>(1, source_->samplerate()),
1249                                         source_->samplerate(),
1250                                         source_audio_stream.sampleformat,
1251                                         source_audio_stream.channel_layout);
1252                 }
1253
1254                 auto total_num_channels = cpplinq::from(source_audio_streams_)
1255                                 .select([](const audio_stream_info& info) { return info.num_channels; })
1256                                 .aggregate(0, std::plus<int>());
1257
1258                 if (total_num_channels > 1 && sink_->wanted_num_audio_streams() > 1)
1259                         CASPAR_THROW_EXCEPTION(invalid_operation()
1260                                         << msg_info("only one-to-many or many-to-one audio stream conversion supported."));
1261
1262                 std::wstring amerge;
1263
1264                 if (sink_->wanted_num_audio_streams() == 1 && !sink_->wanted_num_channels_per_stream())
1265                 {
1266                         output_pads.emplace_back(
1267                                         sink_->supported_samplerates(),
1268                                         sink_->supported_sample_formats(),
1269                                         std::vector<int64_t>({ av_get_default_channel_layout(total_num_channels) }));
1270
1271                         if (source_audio_streams_.size() > 1)
1272                         {
1273                                 for (int i = 0; i < source_audio_streams_.size(); ++i)
1274                                         amerge += L"[a:" + boost::lexical_cast<std::wstring>(i) + L"]";
1275
1276                                 amerge += L"amerge=inputs=" + boost::lexical_cast<std::wstring>(source_audio_streams_.size());
1277                         }
1278                 }
1279
1280                 std::wstring afilter = u16(afilter_);
1281
1282                 if (!amerge.empty())
1283                 {
1284                         afilter = prepend_filter(u16(afilter), amerge);
1285                         afilter += L"[aout:0]";
1286                 }
1287
1288                 audio_filter_.reset(new audio_filter(input_pads, output_pads, u8(afilter)));
1289         }
1290
1291         void initialize_video_filter_if_needed(const AVFrame& av_frame)
1292         {
1293                 bool changed = false;
1294
1295                 set_if_changed(changed, source_video_stream_.width, av_frame.width);
1296                 set_if_changed(changed, source_video_stream_.height, av_frame.height);
1297                 set_if_changed(changed, source_video_stream_.pixelformat, static_cast<AVPixelFormat>(av_frame.format));
1298
1299                 core::field_mode field_mode = core::field_mode::progressive;
1300
1301                 if (av_frame.interlaced_frame)
1302                         field_mode = av_frame.top_field_first ? core::field_mode::upper : core::field_mode::lower;
1303
1304                 set_if_changed(changed, source_video_stream_.fieldmode, field_mode);
1305
1306                 if (changed)
1307                         initialize_video_filter();
1308         }
1309
1310         void initialize_video_filter()
1311         {
1312                 if (source_video_stream_.fieldmode != core::field_mode::progressive && !filter::is_deinterlacing(u16(vfilter_)))
1313                         vfilter_ = u8(append_filter(u16(vfilter_), L"YADIF=1:-1"));
1314
1315                 if (source_video_stream_.height == 480) // NTSC DV
1316                 {
1317                         auto pad_str = L"PAD=" + boost::lexical_cast<std::wstring>(source_video_stream_.width) + L":486:0:2:black";
1318                         vfilter_ = u8(append_filter(u16(vfilter_), pad_str));
1319                 }
1320
1321                 video_filter_.reset(new filter(
1322                                 source_video_stream_.width,
1323                                 source_video_stream_.height,
1324                                 1 / source_->framerate(),
1325                                 source_->framerate(),
1326                                 boost::rational<int>(1, 1), // TODO
1327                                 source_video_stream_.pixelformat,
1328                                 sink_->supported_pixel_formats(),
1329                                 vfilter_));
1330                 sink_->framerate(framerate());
1331         }
1332 };
1333
1334 spl::shared_ptr<struct ffmpeg_pipeline_backend> create_internal_pipeline()
1335 {
1336         return spl::make_shared<ffmpeg_pipeline_backend_internal>();
1337 }
1338
1339 }}