git.sesse.net Git - casparcg/blob - modules/ffmpeg/ffmpeg_pipeline_backend_internal.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Helge Norberg, helge.norberg@svt.se
  20 * Author: Robert Nagy, ronag89@gmail.com
  21 */
  22
  23 #include "StdAfx.h"
  24
  25 #include "ffmpeg_pipeline_backend.h"
  26 #include "ffmpeg_pipeline_backend_internal.h"
  27 #include "producer/input/input.h"
  28 #include "producer/video/video_decoder.h"
  29 #include "producer/audio/audio_decoder.h"
  30 #include "producer/filter/audio_filter.h"
  31 #include "producer/filter/filter.h"
  32 #include "producer/util/util.h"
  33 #include "ffmpeg_error.h"
  34 #include "ffmpeg.h"
  35
  36 #include <common/diagnostics/graph.h>
  37 #include <common/os/general_protection_fault.h>
  38 #include <common/enum_class.h>
  39
  40 #include <core/frame/audio_channel_layout.h>
  41 #include <core/frame/frame.h>
  42 #include <core/frame/frame_factory.h>
  43 #include <core/video_format.h>
  44
  45 #include <functional>
  46 #include <limits>
  47 #include <queue>
  48 #include <map>
  49
  50 #include <tbb/atomic.h>
  51 #include <tbb/concurrent_queue.h>
  52 #include <tbb/spin_mutex.h>
  53
  54 #include <boost/thread.hpp>
  55 #include <boost/optional.hpp>
  56 #include <boost/exception_ptr.hpp>
  57
  58 namespace caspar { namespace ffmpeg {
  59
  60 std::string to_string(const boost::rational<int>& framerate)
  61 {
  62         return boost::lexical_cast<std::string>(framerate.numerator())
  63                 + "/" + boost::lexical_cast<std::string>(framerate.denominator()) + " (" + boost::lexical_cast<std::string>(static_cast<double>(framerate.numerator()) / static_cast<double>(framerate.denominator())) + ") fps";
  64 }
  65
  66 std::vector<int> find_audio_cadence(const boost::rational<int>& framerate)
  67 {
  68         static std::map<boost::rational<int>, std::vector<int>> CADENCES_BY_FRAMERATE = []
  69         {
  70                 std::map<boost::rational<int>, std::vector<int>> result;
  71
  72                 for (core::video_format format : enum_constants<core::video_format>())
  73                 {
  74                         core::video_format_desc desc(format);
  75                         boost::rational<int> format_rate(desc.time_scale, desc.duration);
  76
  77                         result.insert(std::make_pair(format_rate, desc.audio_cadence));
  78                 }
  79
  80                 return result;
  81         }();
  82
  83         auto exact_match = CADENCES_BY_FRAMERATE.find(framerate);
  84
  85         if (exact_match != CADENCES_BY_FRAMERATE.end())
  86                 return exact_match->second;
  87
  88         boost::rational<int> closest_framerate_diff     = std::numeric_limits<int>::max();
  89         boost::rational<int> closest_framerate          = 0;
  90
  91         for (auto format_framerate : CADENCES_BY_FRAMERATE | boost::adaptors::map_keys)
  92         {
  93                 auto diff = boost::abs(framerate - format_framerate);
  94
  95                 if (diff < closest_framerate_diff)
  96                 {
  97                         closest_framerate_diff  = diff;
  98                         closest_framerate               = format_framerate;
  99                 }
 100         }
 101
 102         if (is_logging_quiet_for_thread())
 103                 CASPAR_LOG(debug) << "No exact audio cadence match found for framerate " << to_string(framerate)
 104                         << "\nClosest match is " << to_string(closest_framerate)
 105                         << "\nwhich is a " << to_string(closest_framerate_diff) << " difference.";
 106         else
 107                 CASPAR_LOG(warning) << "No exact audio cadence match found for framerate " << to_string(framerate)
 108                         << "\nClosest match is " << to_string(closest_framerate)
 109                         << "\nwhich is a " << to_string(closest_framerate_diff) << " difference.";
 110
 111         return CADENCES_BY_FRAMERATE[closest_framerate];
 112 }
 113
 114 struct source
 115 {
 116         virtual ~source() { }
 117
 118         virtual std::wstring                                                    print() const                                                                                   = 0;
 119         virtual void                                                                    start()                                                                                                 { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 120         virtual void                                                                    graph(spl::shared_ptr<caspar::diagnostics::graph> g)    { }
 121         virtual void                                                                    stop()                                                                                                  { }
 122         virtual void                                                                    start_frame(std::uint32_t frame)                                                { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 123         virtual std::uint32_t                                                   start_frame() const                                                                             { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 124         virtual void                                                                    loop(bool value)                                                                                { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 125         virtual bool                                                                    loop() const                                                                                    { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 126         virtual void                                                                    length(std::uint32_t frames)                                                    { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 127         virtual std::uint32_t                                                   length() const                                                                                  { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 128         virtual std::string                                                             filename() const                                                                                { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print())); }
 129         virtual void                                                                    seek(std::uint32_t frame)                                                               { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 130         virtual bool                                                                    has_audio() const                                                                               { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 131         virtual int                                                                             samplerate() const                                                                              { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 132         virtual bool                                                                    has_video() const                                                                               { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 133         virtual bool                                                                    eof() const                                                                                             { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 134         virtual boost::rational<int>                                    framerate() const                                                                               { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 135         virtual std::uint32_t                                                   frame_number() const                                                                    { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 136         virtual std::vector<std::shared_ptr<AVFrame>>   get_input_frames_for_streams(AVMediaType type)                  { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 137 };
 138
 139 struct no_source_selected : public source
 140 {
 141         std::wstring print() const override
 142         {
 143                 return L"[no_source_selected]";
 144         }
 145 };
 146
 147 class file_source : public source
 148 {
 149         std::wstring                                                            filename_;
 150         spl::shared_ptr<diagnostics::graph>                     graph_;
 151         std::uint32_t                                                           start_frame_    = 0;
 152         std::uint32_t                                                           length_                 = std::numeric_limits<std::uint32_t>::max();
 153         bool                                                                            loop_                   = false;
 154         mutable boost::mutex                                            pointer_mutex_;
 155         std::shared_ptr<input>                                          input_;
 156         std::vector<spl::shared_ptr<audio_decoder>>     audio_decoders_;
 157         std::shared_ptr<video_decoder>                          video_decoder_;
 158         bool                                                                            started_                = false;
 159 public:
 160         file_source(std::string filename)
 161                 : filename_(u16(filename))
 162         {
 163         }
 164
 165         std::wstring print() const override
 166         {
 167                 return L"[file_source " + filename_ + L"]";
 168         }
 169
 170         void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
 171         {
 172                 graph_ = std::move(g);
 173         }
 174
 175         void start() override
 176         {
 177                 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
 178                 bool thumbnail_mode = is_logging_quiet_for_thread();
 179                 input_.reset(new input(graph_, filename_, loop_, start_frame_, length_, thumbnail_mode));
 180
 181                 for (int i = 0; i < input_->num_audio_streams(); ++i)
 182                 {
 183                         try
 184                         {
 185                                 audio_decoders_.push_back(spl::make_shared<audio_decoder>(*input_, core::video_format::invalid, i));
 186                         }
 187                         catch (...)
 188                         {
 189                                 if (is_logging_quiet_for_thread())
 190                                 {
 191                                         CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
 192                                         CASPAR_LOG(info) << print() << " Failed to open audio-stream. Turn on log level debug to see more information.";
 193                                 }
 194                                 else
 195                                 {
 196                                         CASPAR_LOG_CURRENT_EXCEPTION();
 197                                         CASPAR_LOG(warning) << print() << " Failed to open audio-stream.";
 198                                 }
 199                         }
 200                 }
 201
 202                 if (audio_decoders_.empty())
 203                         CASPAR_LOG(debug) << print() << " No audio-stream found. Running without audio.";
 204
 205                 try
 206                 {
 207                         video_decoder_.reset(new video_decoder(*input_, false));
 208                 }
 209                 catch (averror_stream_not_found&)
 210                 {
 211                         CASPAR_LOG(debug) << print() << " No video-stream found. Running without video.";
 212                 }
 213                 catch (...)
 214                 {
 215                         if (is_logging_quiet_for_thread())
 216                         {
 217                                 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
 218                                 CASPAR_LOG(info) << print() << " Failed to open video-stream. Running without audio. Turn on log level debug to see more information.";
 219                         }
 220                         else
 221                         {
 222                                 CASPAR_LOG_CURRENT_EXCEPTION();
 223                                 CASPAR_LOG(warning) << print() << " Failed to open video-stream. Running without audio.";
 224                         }
 225                 }
 226
 227                 started_ = true;
 228         }
 229
 230         void stop() override
 231         {
 232                 started_ = false;
 233         }
 234
 235         void start_frame(std::uint32_t frame) override
 236         {
 237                 start_frame_ = frame;
 238
 239                 auto i = get_input();
 240                 if (i)
 241                         i->start(frame);
 242         }
 243
 244         std::uint32_t start_frame() const override
 245         {
 246                 return start_frame_;
 247         }
 248
 249         void loop(bool value) override
 250         {
 251                 loop_ = value;
 252
 253                 auto i = get_input();
 254                 if (i)
 255                         i->loop(value);
 256         }
 257
 258         bool loop() const override
 259         {
 260                 return loop_;
 261         }
 262
 263         void length(std::uint32_t frames) override
 264         {
 265                 length_ = frames;
 266
 267                 auto i = get_input();
 268                 if (i)
 269                         i->length(frames);
 270         }
 271
 272         std::uint32_t length() const override
 273         {
 274                 auto v = get_video_decoder();
 275
 276                 if (v)
 277                         return v->nb_frames();
 278
 279                 auto a = get_audio_decoders();
 280
 281                 if (!a.empty())
 282                         return a.at(0)->nb_frames(); // Should be ok.
 283
 284                 return length_;
 285         }
 286
 287         std::string filename() const override
 288         {
 289                 return u8(filename_);
 290         }
 291
 292         void seek(std::uint32_t frame) override
 293         {
 294                 expect_started();
 295                 get_input()->seek(frame);
 296         }
 297
 298         bool eof() const override
 299         {
 300                 auto i = get_input();
 301                 return !i || i->eof();
 302         }
 303
 304         bool has_audio() const override
 305         {
 306                 return !get_audio_decoders().empty();
 307         }
 308
 309         int samplerate() const override
 310         {
 311                 if (get_audio_decoders().empty())
 312                         return -1;
 313
 314                 return 48000;
 315         }
 316
 317         bool has_video() const override
 318         {
 319                 return static_cast<bool>(get_video_decoder());
 320         }
 321
 322         boost::rational<int> framerate() const override
 323         {
 324                 auto decoder = get_video_decoder();
 325
 326                 if (!decoder)
 327                         return -1;
 328
 329                 return decoder->framerate();
 330         }
 331
 332         std::uint32_t frame_number() const override
 333         {
 334                 auto decoder = get_video_decoder();
 335
 336                 if (!decoder)
 337                         return 0;
 338
 339                 return decoder->file_frame_number();
 340         }
 341
 342         std::vector<std::shared_ptr<AVFrame>> get_input_frames_for_streams(AVMediaType type) override
 343         {
 344                 auto a_decoders = get_audio_decoders();
 345                 auto v_decoder  = get_video_decoder();
 346                 expect_started();
 347
 348                 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && !a_decoders.empty())
 349                 {
 350                         std::vector<std::shared_ptr<AVFrame>> frames;
 351
 352                         for (auto& a_decoder : a_decoders)
 353                         {
 354                                 std::shared_ptr<AVFrame> frame;
 355
 356                                 for (int i = 0; i < 64; ++i)
 357                                 {
 358                                         frame = (*a_decoder)();
 359
 360                                         if (frame == flush() || (frame && frame->data[0]))
 361                                                 break;
 362                                         else
 363                                                 frame.reset();
 364                                 }
 365
 366                                 frames.push_back(std::move(frame));
 367                         }
 368
 369                         return frames;
 370                 }
 371                 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO && v_decoder)
 372                 {
 373                         std::shared_ptr<AVFrame> frame;
 374
 375                         for (int i = 0; i < 128; ++i)
 376                         {
 377                                 frame = (*v_decoder)();
 378
 379                                 if (frame == flush() || (frame && frame->data[0]))
 380                                         return { frame };
 381                         }
 382                 }
 383                 else
 384                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(
 385                                 print() + L" Unhandled media type " + boost::lexical_cast<std::wstring>(type)));
 386
 387                 return { };
 388         }
 389 private:
 390         void expect_started() const
 391         {
 392                 if (!started_)
 393                         CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" Not started."));
 394         }
 395
 396         std::shared_ptr<input> get_input() const
 397         {
 398                 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
 399                 return input_;
 400         }
 401
 402         std::vector<spl::shared_ptr<audio_decoder>> get_audio_decoders() const
 403         {
 404                 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
 405                 return audio_decoders_;
 406         }
 407
 408         std::shared_ptr<video_decoder> get_video_decoder() const
 409         {
 410                 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
 411                 return video_decoder_;
 412         }
 413 };
 414
 415 class memory_source : public source
 416 {
 417         int                                                                                                                     samplerate_             = -1;
 418         int                                                                                                                     num_channels_   = -1;
 419         int                                                                                                                     width_                  = -1;
 420         int                                                                                                                     height_                 = -1;
 421         boost::rational<int>                                                                            framerate_              = -1;
 422
 423         tbb::atomic<bool>                                                                                       running_;
 424         tbb::concurrent_bounded_queue<caspar::array<const int32_t>>     audio_frames_;
 425         tbb::concurrent_bounded_queue<caspar::array<const uint8_t>>     video_frames_;
 426         int64_t                                                                                                         audio_pts_              = 0;
 427         int64_t                                                                                                         video_pts_              = 0;
 428 public:
 429         memory_source()
 430         {
 431                 running_ = false;
 432                 video_frames_.set_capacity(1);
 433                 audio_frames_.set_capacity(1);
 434         }
 435
 436         ~memory_source()
 437         {
 438                 stop();
 439         }
 440
 441         void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
 442         {
 443         }
 444
 445         std::wstring print() const override
 446         {
 447                 return L"[memory_source]";
 448         }
 449
 450         void enable_audio(int samplerate, int num_channels)
 451         {
 452                 samplerate_ = samplerate;
 453                 num_channels_ = num_channels;
 454         }
 455
 456         void enable_video(int width, int height, boost::rational<int> framerate)
 457         {
 458                 width_ = width;
 459                 height_ = height;
 460         }
 461
 462         void start() override
 463         {
 464                 running_ = true;
 465         }
 466
 467         void stop() override
 468         {
 469                 running_ = false;
 470                 video_frames_.try_push(caspar::array<const uint8_t>());
 471                 audio_frames_.try_push(caspar::array<const int32_t>());
 472         }
 473
 474         bool has_audio() const override
 475         {
 476                 return samplerate_ != -1;
 477         }
 478
 479         int samplerate() const override
 480         {
 481                 return samplerate_;
 482         }
 483
 484         bool has_video() const override
 485         {
 486                 return width_ != -1;
 487         }
 488
 489         bool eof() const override
 490         {
 491                 return !running_;
 492         }
 493
 494         boost::rational<int> framerate() const override
 495         {
 496                 return framerate_;
 497         }
 498
 499         bool try_push_audio(caspar::array<const std::int32_t> data)
 500         {
 501                 if (!has_audio())
 502                         CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" audio not enabled."));
 503
 504                 if (data.empty() || data.size() % num_channels_ != 0)
 505                         CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info(print() + L" audio with incorrect number of channels submitted."));
 506
 507                 return audio_frames_.try_push(std::move(data));
 508         }
 509
 510         bool try_push_video(caspar::array<const std::uint8_t> data)
 511         {
 512                 if (!has_video())
 513                         CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" video not enabled."));
 514
 515                 if (data.size() != width_ * height_ * 4)
 516                         CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info(print() + L" video with incorrect size submitted."));
 517
 518                 return video_frames_.try_push(std::move(data));
 519         }
 520
 521         std::vector<std::shared_ptr<AVFrame>> get_input_frames_for_streams(AVMediaType type) override
 522         {
 523                 if (!running_)
 524                         CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not running."));
 525
 526                 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && has_audio())
 527                 {
 528                         caspar::array<const std::int32_t> samples;
 529                         audio_frames_.pop(samples);
 530
 531                         if (samples.empty())
 532                                 return { };
 533
 534                         spl::shared_ptr<AVFrame> av_frame(av_frame_alloc(), [samples](AVFrame* p) { av_frame_free(&p); });
 535
 536                         av_frame->channels                      = num_channels_;
 537                         av_frame->channel_layout        = av_get_default_channel_layout(num_channels_);
 538                         av_frame->sample_rate           = samplerate_;
 539                         av_frame->nb_samples            = static_cast<int>(samples.size()) / num_channels_;
 540                         av_frame->format                        = AV_SAMPLE_FMT_S32;
 541                         av_frame->pts                           = audio_pts_;
 542
 543                         audio_pts_ += av_frame->nb_samples;
 544
 545                         FF(av_samples_fill_arrays(
 546                                         av_frame->extended_data,
 547                                         av_frame->linesize,
 548                                         reinterpret_cast<const std::uint8_t*>(&*samples.begin()),
 549                                         av_frame->channels,
 550                                         av_frame->nb_samples,
 551                                         static_cast<AVSampleFormat>(av_frame->format),
 552                                         16));
 553
 554                         return { av_frame };
 555                 }
 556                 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO && has_video())
 557                 {
 558                         caspar::array<const std::uint8_t> data;
 559                         video_frames_.pop(data);
 560
 561                         if (data.empty())
 562                                 return {};
 563
 564                         spl::shared_ptr<AVFrame> av_frame(av_frame_alloc(), [data](AVFrame* p) { av_frame_free(&p); });
 565                         avcodec_get_frame_defaults(av_frame.get());
 566
 567                         const auto sample_aspect_ratio = boost::rational<int>(width_, height_);
 568
 569                         av_frame->format                                  = AV_PIX_FMT_BGRA;
 570                         av_frame->width                                   = width_;
 571                         av_frame->height                                  = height_;
 572                         av_frame->sample_aspect_ratio.num = sample_aspect_ratio.numerator();
 573                         av_frame->sample_aspect_ratio.den = sample_aspect_ratio.denominator();
 574                         av_frame->pts                                     = video_pts_;
 575
 576                         video_pts_ += 1;
 577
 578                         FF(av_image_fill_arrays(
 579                                         av_frame->data,
 580                                         av_frame->linesize,
 581                                         data.begin(),
 582                                         static_cast<AVPixelFormat>(av_frame->format),
 583                                         width_,
 584                                         height_,
 585                                         1));
 586
 587                         return { av_frame };
 588                 }
 589                 else
 590                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(
 591                                 print() + L" Unhandled media type " + boost::lexical_cast<std::wstring>(type)));
 592         }
 593 };
 594
 595 struct sink
 596 {
 597         virtual ~sink() { }
 598
 599         virtual std::wstring                                    print() const                                                                                                                                   = 0;
 600         virtual void                                                    graph(spl::shared_ptr<caspar::diagnostics::graph> g)                                                    { }
 601         virtual void                                                    acodec(std::string codec)                                                                                                               { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
 602         virtual void                                                    vcodec(std::string codec)                                                                                                               { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
 603         virtual void                                                    format(std::string fmt)                                                                                                                 { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
 604         virtual void                                                    framerate(boost::rational<int> framerate)                                                                               { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
 605         virtual void                                                    start(bool has_audio, bool has_video)                                                                                   { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 606         virtual void                                                    stop()                                                                                                                                                  { }
 607         virtual void                                                    flush_all()                                                                                                                                             { }
 608         virtual std::vector<AVSampleFormat>             supported_sample_formats() const                                                                                                { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 609         virtual std::vector<int>                                supported_samplerates() const                                                                                                   { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 610         virtual std::vector<AVPixelFormat>              supported_pixel_formats() const                                                                                                 { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 611         virtual int                                                             wanted_num_audio_streams() const                                                                                                { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 612         virtual boost::optional<int>                    wanted_num_channels_per_stream() const                                                                                  { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 613         virtual boost::optional<AVMediaType>    try_push(AVMediaType type, int stream_index, spl::shared_ptr<AVFrame> frame)    { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 614         virtual void                                                    eof()                                                                                                                                                   { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 615 };
 616
 617 struct no_sink_selected : public sink
 618 {
 619         std::wstring print() const override
 620         {
 621                 return L"[no_sink_selected]";
 622         }
 623 };
 624
 625 class file_sink : public sink
 626 {
 627         std::wstring                                            filename_;
 628         spl::shared_ptr<diagnostics::graph>     graph_;
 629 public:
 630         file_sink(std::string filename)
 631                 : filename_(u16(std::move(filename)))
 632         {
 633         }
 634
 635         std::wstring print() const override
 636         {
 637                 return L"[file_sink " + filename_ + L"]";
 638         }
 639
 640         void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
 641         {
 642                 graph_ = std::move(g);
 643         }
 644 };
 645
 646 class memory_sink : public sink
 647 {
 648         spl::shared_ptr<core::frame_factory>                    factory_;
 649
 650         bool                                                                                    has_audio_                      = false;
 651         bool                                                                                    has_video_                      = false;
 652         std::vector<int>                                                                audio_cadence_;
 653         core::audio_channel_layout                                              channel_layout_         = core::audio_channel_layout::invalid();
 654         core::mutable_audio_buffer                                              audio_samples_;
 655
 656         std::queue<std::shared_ptr<AVFrame>>                    video_frames_;
 657
 658         tbb::concurrent_bounded_queue<core::draw_frame> output_frames_;
 659         tbb::atomic<bool>                                                               running_;
 660 public:
 661         memory_sink(spl::shared_ptr<core::frame_factory> factory, core::video_format_desc format)
 662                 : factory_(std::move(factory))
 663                 , audio_cadence_(format.audio_cadence)
 664         {
 665                 output_frames_.set_capacity(2);
 666                 running_ = false;
 667                 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
 668                 // This cadence fills the audio mixer most optimally.
 669                 boost::range::rotate(audio_cadence_, std::end(audio_cadence_) - 1);
 670         }
 671
 672         ~memory_sink()
 673         {
 674                 stop();
 675         }
 676
 677         std::wstring print() const override
 678         {
 679                 return L"[memory_sink]";
 680         }
 681
 682         void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
 683         {
 684         }
 685
 686         void framerate(boost::rational<int> framerate) override
 687         {
 688                 audio_cadence_ = find_audio_cadence(framerate);
 689                 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
 690                 // This cadence fills the audio mixer most optimally.
 691                 boost::range::rotate(audio_cadence_, std::end(audio_cadence_) - 1);
 692         }
 693
 694         void start(bool has_audio, bool has_video) override
 695         {
 696                 has_audio_      = has_audio;
 697                 has_video_      = has_video;
 698                 running_        = true;
 699         }
 700
 701         void stop() override
 702         {
 703                 running_ = false;
 704                 output_frames_.set_capacity(4);
 705         }
 706
 707         std::vector<AVSampleFormat> supported_sample_formats() const override
 708         {
 709                 return { AVSampleFormat::AV_SAMPLE_FMT_S32 };
 710         }
 711
 712         std::vector<int> supported_samplerates() const override {
 713                 return { 48000 };
 714         }
 715
 716         std::vector<AVPixelFormat> supported_pixel_formats() const override
 717         {
 718                 return {
 719                         AVPixelFormat::AV_PIX_FMT_YUVA420P,
 720                         AVPixelFormat::AV_PIX_FMT_YUV444P,
 721                         AVPixelFormat::AV_PIX_FMT_YUV422P,
 722                         AVPixelFormat::AV_PIX_FMT_YUV420P,
 723                         AVPixelFormat::AV_PIX_FMT_YUV411P,
 724                         AVPixelFormat::AV_PIX_FMT_BGRA,
 725                         AVPixelFormat::AV_PIX_FMT_ARGB,
 726                         AVPixelFormat::AV_PIX_FMT_RGBA,
 727                         AVPixelFormat::AV_PIX_FMT_ABGR,
 728                         AVPixelFormat::AV_PIX_FMT_GRAY8
 729                 };
 730         }
 731
 732         int wanted_num_audio_streams() const override
 733         {
 734                 return 1;
 735         }
 736
 737         boost::optional<int> wanted_num_channels_per_stream() const
 738         {
 739                 return boost::none;
 740         }
 741
 742         void flush_all() override
 743         {
 744                 audio_samples_.clear();
 745
 746                 while (!video_frames_.empty())
 747                         video_frames_.pop();
 748         }
 749
 750         boost::optional<AVMediaType> try_push(AVMediaType type, int stream_index, spl::shared_ptr<AVFrame> av_frame) override
 751         {
 752                 if (!has_audio_ && !has_video_)
 753                         CASPAR_THROW_EXCEPTION(invalid_operation());
 754
 755                 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && av_frame->data[0])
 756                 {
 757                         if (channel_layout_ == core::audio_channel_layout::invalid()) // First audio
 758                         {
 759                                 channel_layout_ = get_audio_channel_layout(av_frame->channels, av_frame->channel_layout, L"");
 760
 761                                 // Insert silence samples so that the audio mixer is guaranteed to be filled.
 762                                 auto min_num_samples_per_frame  = *boost::min_element(audio_cadence_);
 763                                 auto max_num_samples_per_frame  = *boost::max_element(audio_cadence_);
 764                                 auto cadence_safety_samples             = max_num_samples_per_frame - min_num_samples_per_frame;
 765                                 audio_samples_.resize(channel_layout_.num_channels * cadence_safety_samples, 0);
 766                         }
 767
 768                         auto ptr = reinterpret_cast<int32_t*>(av_frame->data[0]);
 769
 770                         audio_samples_.insert(audio_samples_.end(), ptr, ptr + av_frame->linesize[0] / sizeof(int32_t));
 771                 }
 772                 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO)
 773                 {
 774                         video_frames_.push(std::move(av_frame));
 775                 }
 776
 777                 while (true)
 778                 {
 779                         bool enough_audio =
 780                                 !has_audio_ ||
 781                                 (channel_layout_ != core::audio_channel_layout::invalid() && audio_samples_.size() >= audio_cadence_.front() * channel_layout_.num_channels);
 782                         bool enough_video =
 783                                 !has_video_ ||
 784                                 !video_frames_.empty();
 785
 786                         if (!enough_audio)
 787                                 return AVMediaType::AVMEDIA_TYPE_AUDIO;
 788
 789                         if (!enough_video)
 790                                 return AVMediaType::AVMEDIA_TYPE_VIDEO;
 791
 792                         core::mutable_audio_buffer audio_data;
 793
 794                         if (has_audio_)
 795                         {
 796                                 auto begin = audio_samples_.begin();
 797                                 auto end = begin + audio_cadence_.front() * channel_layout_.num_channels;
 798
 799                                 audio_data.insert(audio_data.begin(), begin, end);
 800                                 audio_samples_.erase(begin, end);
 801                                 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_) + 1);
 802                         }
 803
 804                         if (!has_video_) // Audio only
 805                         {
 806                                 core::mutable_frame audio_only_frame(
 807                                                 { },
 808                                                 std::move(audio_data),
 809                                                 this,
 810                                                 core::pixel_format_desc(core::pixel_format::invalid),
 811                                                 channel_layout_);
 812
 813                                 output_frames_.push(core::draw_frame(std::move(audio_only_frame)));
 814
 815                                 return AVMediaType::AVMEDIA_TYPE_AUDIO;
 816                         }
 817
 818                         auto output_frame = make_frame(this, spl::make_shared_ptr(video_frames_.front()), *factory_, channel_layout_);
 819                         video_frames_.pop();
 820                         output_frame.audio_data() = std::move(audio_data);
 821
 822                         output_frames_.push(core::draw_frame(std::move(output_frame)));
 823                 }
 824         }
 825
 826         void eof() override
 827         {
 828                 // Drain rest, regardless of it being enough or not.
 829                 while (!video_frames_.empty() || !audio_samples_.empty())
 830                 {
 831                         core::mutable_audio_buffer audio_data;
 832
 833                         audio_data.swap(audio_samples_);
 834
 835                         if (!video_frames_.empty())
 836                         {
 837                                 auto output_frame = make_frame(this, spl::make_shared_ptr(video_frames_.front()), *factory_, channel_layout_);
 838                                 video_frames_.pop();
 839                                 output_frame.audio_data() = std::move(audio_data);
 840
 841                                 output_frames_.push(core::draw_frame(std::move(output_frame)));
 842                         }
 843                         else
 844                         {
 845                                 core::mutable_frame audio_only_frame(
 846                                                 {},
 847                                                 std::move(audio_data),
 848                                                 this,
 849                                                 core::pixel_format_desc(core::pixel_format::invalid),
 850                                                 channel_layout_);
 851
 852                                 output_frames_.push(core::draw_frame(std::move(audio_only_frame)));
 853                                 output_frames_.push(core::draw_frame::empty());
 854                         }
 855                 }
 856         }
 857
 858         core::draw_frame try_pop_frame()
 859         {
 860                 core::draw_frame frame = core::draw_frame::late();
 861
 862                 if (!output_frames_.try_pop(frame) && !running_)
 863                         return core::draw_frame::empty();
 864
 865                 return frame;
 866         }
 867 };
 868
 869 struct audio_stream_info
 870 {
 871         int                             num_channels    = 0;
 872         AVSampleFormat  sampleformat    = AVSampleFormat::AV_SAMPLE_FMT_NONE;
 873         uint64_t                channel_layout  = 0;
 874 };
 875
 876 struct video_stream_info
 877 {
 878         int                                     width           = 0;
 879         int                                     height          = 0;
 880         AVPixelFormat           pixelformat     = AVPixelFormat::AV_PIX_FMT_NONE;
 881         core::field_mode        fieldmode       = core::field_mode::progressive;
 882 };
 883
 884 class ffmpeg_pipeline_backend_internal : public ffmpeg_pipeline_backend
 885 {
 886         spl::shared_ptr<diagnostics::graph>                                                             graph_;
 887
 888         spl::unique_ptr<source>                                                                                 source_                                 = spl::make_unique<no_source_selected>();
 889         std::function<bool (caspar::array<const std::int32_t> data)>    try_push_audio_;
 890         std::function<bool (caspar::array<const std::uint8_t> data)>    try_push_video_;
 891
 892         std::vector<audio_stream_info>                                                                  source_audio_streams_;
 893         video_stream_info                                                                                               source_video_stream_;
 894
 895         std::string                                                                                                             afilter_;
 896         std::unique_ptr<audio_filter>                                                                   audio_filter_;
 897         std::string                                                                                                             vfilter_;
 898         std::unique_ptr<filter>                                                                                 video_filter_;
 899
 900         spl::unique_ptr<sink>                                                                                   sink_                                   = spl::make_unique<no_sink_selected>();
 901         std::function<core::draw_frame ()>                                                              try_pop_frame_;
 902
 903         tbb::atomic<bool>                                                                                               started_;
 904         tbb::spin_mutex                                                                                                 exception_mutex_;
 905         boost::exception_ptr                                                                                    exception_;
 906         boost::thread                                                                                                   thread_;
 907 public:
 908         ffmpeg_pipeline_backend_internal()
 909         {
 910                 started_ = false;
 911                 diagnostics::register_graph(graph_);
 912         }
 913
 914         ~ffmpeg_pipeline_backend_internal()
 915         {
 916                 stop();
 917         }
 918
 919         void throw_if_error()
 920         {
 921                 boost::lock_guard<tbb::spin_mutex> lock(exception_mutex_);
 922
 923                 if (exception_ != nullptr)
 924                         boost::rethrow_exception(exception_);
 925         }
 926
 927         void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
 928         {
 929                 graph_ = std::move(g);
 930                 source_->graph(graph_);
 931                 sink_->graph(graph_);
 932         }
 933
 934         // Source setup
 935
 936         void from_file(std::string filename) override
 937         {
 938                 source_                 = spl::make_unique<file_source>(std::move(filename));
 939                 try_push_audio_ = std::function<bool (caspar::array<const std::int32_t>)>();
 940                 try_push_video_ = std::function<bool (caspar::array<const std::uint8_t>)>();
 941                 source_->graph(graph_);
 942         }
 943
 944         void from_memory_only_audio(int num_channels, int samplerate) override
 945         {
 946                 auto source             = spl::make_unique<memory_source>();
 947                 auto source_ptr = source.get();
 948                 try_push_audio_ = [this, source_ptr](caspar::array<const std::int32_t> data) { return source_ptr->try_push_audio(std::move(data)); };
 949                 source->enable_audio(samplerate, num_channels);
 950
 951                 source_ = std::move(source);
 952                 source_->graph(graph_);
 953         }
 954
 955         void from_memory_only_video(int width, int height, boost::rational<int> framerate) override
 956         {
 957                 auto source             = spl::make_unique<memory_source>();
 958                 auto source_ptr = source.get();
 959                 try_push_video_ = [this, source_ptr](caspar::array<const std::uint8_t> data) { return source_ptr->try_push_video(std::move(data)); };
 960                 source->enable_video(width, height, std::move(framerate));
 961
 962                 source_ = std::move(source);
 963                 source_->graph(graph_);
 964         }
 965
 966         void from_memory(int num_channels, int samplerate, int width, int height, boost::rational<int> framerate) override
 967         {
 968                 auto source             = spl::make_unique<memory_source>();
 969                 auto source_ptr = source.get();
 970                 try_push_audio_ = [this, source_ptr](caspar::array<const std::int32_t> data) { return source_ptr->try_push_audio(std::move(data)); };
 971                 try_push_video_ = [this, source_ptr](caspar::array<const std::uint8_t> data) { return source_ptr->try_push_video(std::move(data)); };
 972                 source->enable_audio(samplerate, num_channels);
 973                 source->enable_video(width, height, std::move(framerate));
 974
 975                 source_ = std::move(source);
 976                 source_->graph(graph_);
 977         }
 978
 979         void                    start_frame(std::uint32_t frame) override       { source_->start_frame(frame);          }
 980         std::uint32_t   start_frame() const override                            { return source_->start_frame();        }
 981         void                    length(std::uint32_t frames) override           { source_->length(frames);                      }
 982         std::uint32_t   length() const override                                         { return source_->length();                     }
 983         void                    seek(std::uint32_t frame) override                      { source_->seek(frame);                         }
 984         void                    loop(bool value) override                                       { source_->loop(value);                         }
 985         bool                    loop() const override                                           { return source_->loop();                       }
 986         std::string             source_filename() const override                        { return source_->filename();           }
 987
 988         // Filter setup
 989
 990         void vfilter(std::string filter) override
 991         {
 992                 vfilter_ = std::move(filter);
 993         }
 994
 995         void afilter(std::string filter) override
 996         {
 997                 afilter_ = std::move(filter);
 998         }
 999
1000         int width() const override
1001         {
1002                 return source_video_stream_.width;
1003         }
1004
1005         int height() const override
1006         {
1007                 return source_video_stream_.height;
1008         }
1009
1010         boost::rational<int> framerate() const override
1011         {
1012                 bool double_rate = filter::is_double_rate(u16(vfilter_));
1013
1014                 return double_rate ? source_->framerate() * 2 : source_->framerate();
1015         }
1016
1017         bool progressive() const override
1018         {
1019                 return true;//TODO
1020         }
1021
1022         // Sink setup
1023
1024         void to_memory(spl::shared_ptr<core::frame_factory> factory, core::video_format_desc format) override
1025         {
1026                 auto sink               = spl::make_unique<memory_sink>(std::move(factory), std::move(format));
1027                 auto sink_ptr   = sink.get();
1028                 try_pop_frame_  = [sink_ptr] { return sink_ptr->try_pop_frame(); };
1029
1030                 sink_ = std::move(sink);
1031                 sink_->graph(graph_);
1032         }
1033
1034         void to_file(std::string filename) override
1035         {
1036                 sink_                   = spl::make_unique<file_sink>(std::move(filename));
1037                 try_pop_frame_  = std::function<core::draw_frame ()>();
1038                 sink_->graph(graph_);
1039         }
1040
1041         void acodec(std::string codec) override { sink_->acodec(std::move(codec)); }
1042         void vcodec(std::string codec) override { sink_->vcodec(std::move(codec)); }
1043         void format(std::string fmt) override   { sink_->format(std::move(fmt)); }
1044
1045         // Runtime control
1046
1047         void start() override
1048         {
1049                 source_->start();
1050                 sink_->start(source_->has_audio(), source_->has_video());
1051                 started_ = true;
1052                 bool quiet = is_logging_quiet_for_thread();
1053
1054                 thread_ = boost::thread([=] { run(quiet); });
1055         }
1056
1057         bool try_push_audio(caspar::array<const std::int32_t> data) override
1058         {
1059                 throw_if_error();
1060
1061                 if (try_push_audio_)
1062                         return try_push_audio_(std::move(data));
1063                 else
1064                         return false;
1065         }
1066
1067         bool try_push_video(caspar::array<const std::uint8_t> data) override
1068         {
1069                 throw_if_error();
1070
1071                 if (try_push_video_)
1072                         return try_push_video_(std::move(data));
1073                 else
1074                         return false;
1075         }
1076
1077         core::draw_frame try_pop_frame() override
1078         {
1079                 throw_if_error();
1080
1081                 if (!try_pop_frame_)
1082                         CASPAR_THROW_EXCEPTION(invalid_operation());
1083
1084                 return try_pop_frame_();
1085         }
1086
1087         std::uint32_t last_frame() const override
1088         {
1089                 return source_->frame_number();
1090         }
1091
1092         bool started() const override
1093         {
1094                 return started_;
1095         }
1096
1097         void stop() override
1098         {
1099                 started_ = false;
1100
1101                 sink_->stop();
1102                 source_->stop();
1103
1104                 if (thread_.joinable())
1105                         thread_.join();
1106         }
1107
1108 private:
1109         void run(bool quiet)
1110         {
1111                 ensure_gpf_handler_installed_for_thread(u8(L"ffmpeg-pipeline: " + source_->print() + L" -> " + sink_->print()).c_str());
1112                 auto quiet_logging = temporary_enable_quiet_logging_for_thread(quiet);
1113
1114                 try
1115                 {
1116                         boost::optional<AVMediaType> result = source_->has_audio() ? AVMediaType::AVMEDIA_TYPE_AUDIO : AVMediaType::AVMEDIA_TYPE_VIDEO;
1117
1118                         while (started_ && (source_->has_audio() || source_->has_video()))
1119                         {
1120                                 auto needed                                             = *result;
1121                                 auto input_frames_for_streams   = source_->get_input_frames_for_streams(needed);
1122                                 bool flush_all                                  = !input_frames_for_streams.empty() && input_frames_for_streams.at(0) == flush();
1123
1124                                 if (flush_all)
1125                                 {
1126                                         sink_->flush_all();
1127
1128                                         if (source_->has_audio() && source_->has_video())
1129                                                 result = needed == AVMediaType::AVMEDIA_TYPE_AUDIO ? AVMediaType::AVMEDIA_TYPE_VIDEO : AVMediaType::AVMEDIA_TYPE_AUDIO;
1130
1131                                         continue;
1132                                 }
1133
1134                                 bool got_requested_media_type   = !input_frames_for_streams.empty() && input_frames_for_streams.at(0);
1135
1136                                 if (got_requested_media_type)
1137                                 {
1138                                         for (int input_stream_index = 0; input_stream_index < input_frames_for_streams.size(); ++input_stream_index)
1139                                         {
1140                                                 if (needed == AVMediaType::AVMEDIA_TYPE_AUDIO)
1141                                                 {
1142                                                         initialize_audio_filter_if_needed(input_frames_for_streams);
1143                                                         audio_filter_->push(input_stream_index, std::move(input_frames_for_streams.at(input_stream_index)));
1144
1145                                                         for (int output_stream_index = 0; output_stream_index < sink_->wanted_num_audio_streams(); ++output_stream_index)
1146                                                                 for (auto filtered_frame : audio_filter_->poll_all(output_stream_index))
1147                                                                         result = sink_->try_push(AVMediaType::AVMEDIA_TYPE_AUDIO, output_stream_index, std::move(filtered_frame));
1148                                                 }
1149                                                 else if (needed == AVMediaType::AVMEDIA_TYPE_VIDEO)
1150                                                 {
1151                                                         initialize_video_filter_if_needed(*input_frames_for_streams.at(input_stream_index));
1152                                                         video_filter_->push(std::move(input_frames_for_streams.at(input_stream_index)));
1153
1154                                                         for (auto filtered_frame : video_filter_->poll_all())
1155                                                                 result = sink_->try_push(AVMediaType::AVMEDIA_TYPE_VIDEO, 0, std::move(filtered_frame));
1156                                                 }
1157                                                 else
1158                                                         CASPAR_THROW_EXCEPTION(not_supported());
1159                                         }
1160                                 }
1161                                 else if (source_->eof())
1162                                 {
1163                                         started_ = false;
1164                                         sink_->eof();
1165                                         break;
1166                                 }
1167                                 else
1168                                         result = boost::none;
1169
1170                                 if (!result)
1171                                 {
1172                                         graph_->set_tag(caspar::diagnostics::tag_severity::WARNING, "dropped-frame");
1173                                         result = needed; // Repeat same media type
1174                                 }
1175                         }
1176                 }
1177                 catch (...)
1178                 {
1179                         if (is_logging_quiet_for_thread())
1180                         {
1181                                 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
1182                         }
1183                         else
1184                         {
1185                                 CASPAR_LOG_CURRENT_EXCEPTION();
1186                         }
1187
1188                         boost::lock_guard<tbb::spin_mutex> lock(exception_mutex_);
1189                         exception_ = boost::current_exception();
1190                 }
1191
1192                 video_filter_.reset();
1193                 audio_filter_.reset();
1194                 source_->stop();
1195                 sink_->stop();
1196                 started_ = false;
1197         }
1198
1199         template<typename T>
1200         void set_if_changed(bool& changed, T& old_value, T new_value)
1201         {
1202                 if (old_value != new_value)
1203                 {
1204                         changed = true;
1205                         old_value = new_value;
1206                 }
1207         }
1208
1209         void initialize_audio_filter_if_needed(const std::vector<std::shared_ptr<AVFrame>>& av_frames_per_stream)
1210         {
1211                 bool changed = av_frames_per_stream.size() != source_audio_streams_.size();
1212                 source_audio_streams_.resize(av_frames_per_stream.size());
1213
1214                 for (int i = 0; i < av_frames_per_stream.size(); ++i)
1215                 {
1216                         auto& av_frame  = *av_frames_per_stream.at(i);
1217                         auto& stream    = source_audio_streams_.at(i);
1218
1219                         auto channel_layout = av_frame.channel_layout == 0
1220                                         ? av_get_default_channel_layout(av_frame.channels)
1221                                         : av_frame.channel_layout;
1222
1223                         set_if_changed(changed, stream.sampleformat, static_cast<AVSampleFormat>(av_frame.format));
1224                         set_if_changed(changed, stream.num_channels, av_frame.channels);
1225                         set_if_changed(changed, stream.channel_layout, channel_layout);
1226                 }
1227
1228                 if (changed)
1229                         initialize_audio_filter();
1230         }
1231
1232         void initialize_audio_filter()
1233         {
1234                 std::vector<audio_input_pad> input_pads;
1235                 std::vector<audio_output_pad> output_pads;
1236
1237                 for (auto& source_audio_stream : source_audio_streams_)
1238                 {
1239                         input_pads.emplace_back(
1240                                         boost::rational<int>(1, source_->samplerate()),
1241                                         source_->samplerate(),
1242                                         source_audio_stream.sampleformat,
1243                                         source_audio_stream.channel_layout);
1244                 }
1245
1246                 auto total_num_channels = cpplinq::from(source_audio_streams_)
1247                                 .select([](const audio_stream_info& info) { return info.num_channels; })
1248                                 .aggregate(0, std::plus<int>());
1249
1250                 if (total_num_channels > 1 && sink_->wanted_num_audio_streams() > 1)
1251                         CASPAR_THROW_EXCEPTION(invalid_operation()
1252                                         << msg_info("only one-to-many or many-to-one audio stream conversion supported."));
1253
1254                 std::wstring amerge;
1255
1256                 if (sink_->wanted_num_audio_streams() == 1 && !sink_->wanted_num_channels_per_stream())
1257                 {
1258                         output_pads.emplace_back(
1259                                         sink_->supported_samplerates(),
1260                                         sink_->supported_sample_formats(),
1261                                         std::vector<int64_t>({ av_get_default_channel_layout(total_num_channels) }));
1262
1263                         if (source_audio_streams_.size() > 1)
1264                         {
1265                                 for (int i = 0; i < source_audio_streams_.size(); ++i)
1266                                         amerge += L"[a:" + boost::lexical_cast<std::wstring>(i) + L"]";
1267
1268                                 amerge += L"amerge=inputs=" + boost::lexical_cast<std::wstring>(source_audio_streams_.size());
1269                         }
1270                 }
1271
1272                 std::wstring afilter = u16(afilter_);
1273
1274                 if (!amerge.empty())
1275                 {
1276                         afilter = prepend_filter(u16(afilter), amerge);
1277                         afilter += L"[aout:0]";
1278                 }
1279
1280                 audio_filter_.reset(new audio_filter(input_pads, output_pads, u8(afilter)));
1281         }
1282
1283         void initialize_video_filter_if_needed(const AVFrame& av_frame)
1284         {
1285                 bool changed = false;
1286
1287                 set_if_changed(changed, source_video_stream_.width, av_frame.width);
1288                 set_if_changed(changed, source_video_stream_.height, av_frame.height);
1289                 set_if_changed(changed, source_video_stream_.pixelformat, static_cast<AVPixelFormat>(av_frame.format));
1290
1291                 core::field_mode field_mode = core::field_mode::progressive;
1292
1293                 if (av_frame.interlaced_frame)
1294                         field_mode = av_frame.top_field_first ? core::field_mode::upper : core::field_mode::lower;
1295
1296                 set_if_changed(changed, source_video_stream_.fieldmode, field_mode);
1297
1298                 if (changed)
1299                         initialize_video_filter();
1300         }
1301
1302         void initialize_video_filter()
1303         {
1304                 if (source_video_stream_.fieldmode != core::field_mode::progressive && !filter::is_deinterlacing(u16(vfilter_)))
1305                         vfilter_ = u8(append_filter(u16(vfilter_), L"YADIF=1:-1"));
1306
1307                 if (source_video_stream_.height == 480) // NTSC DV
1308                 {
1309                         auto pad_str = L"PAD=" + boost::lexical_cast<std::wstring>(source_video_stream_.width) + L":486:0:2:black";
1310                         vfilter_ = u8(append_filter(u16(vfilter_), pad_str));
1311                 }
1312
1313                 video_filter_.reset(new filter(
1314                                 source_video_stream_.width,
1315                                 source_video_stream_.height,
1316                                 1 / source_->framerate(),
1317                                 source_->framerate(),
1318                                 boost::rational<int>(1, 1), // TODO
1319                                 source_video_stream_.pixelformat,
1320                                 sink_->supported_pixel_formats(),
1321                                 vfilter_));
1322                 sink_->framerate(framerate());
1323         }
1324 };
1325
1326 spl::shared_ptr<struct ffmpeg_pipeline_backend> create_internal_pipeline()
1327 {
1328         return spl::make_shared<ffmpeg_pipeline_backend_internal>();
1329 }
1330
1331 }}