git.sesse.net Git - casparcg/blob - modules/ffmpeg/ffmpeg_pipeline_backend_internal.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Helge Norberg, helge.norberg@svt.se
  20 */
  21
  22 #include "StdAfx.h"
  23
  24 #include "ffmpeg_pipeline_backend.h"
  25 #include "ffmpeg_pipeline_backend_internal.h"
  26 #include "producer/input/input.h"
  27 #include "producer/video/video_decoder.h"
  28 #include "producer/audio/audio_decoder.h"
  29 #include "producer/filter/audio_filter.h"
  30 #include "producer/filter/filter.h"
  31 #include "producer/util/util.h"
  32 #include "ffmpeg_error.h"
  33 #include "ffmpeg.h"
  34
  35 #include <common/diagnostics/graph.h>
  36 #include <common/os/general_protection_fault.h>
  37 #include <common/enum_class.h>
  38
  39 #include <core/frame/audio_channel_layout.h>
  40 #include <core/frame/frame.h>
  41 #include <core/frame/frame_factory.h>
  42 #include <core/video_format.h>
  43
  44 #include <functional>
  45 #include <limits>
  46 #include <queue>
  47 #include <map>
  48
  49 #include <tbb/atomic.h>
  50 #include <tbb/concurrent_queue.h>
  51
  52 #include <boost/thread.hpp>
  53 #include <boost/optional.hpp>
  54
  55 namespace caspar { namespace ffmpeg {
  56
  57 std::string to_string(const boost::rational<int>& framerate)
  58 {
  59         return boost::lexical_cast<std::string>(framerate.numerator())
  60                 + "/" + boost::lexical_cast<std::string>(framerate.denominator()) + " (" + boost::lexical_cast<std::string>(static_cast<double>(framerate.numerator()) / static_cast<double>(framerate.denominator())) + ") fps";
  61 }
  62
  63 std::vector<int> find_audio_cadence(const boost::rational<int>& framerate)
  64 {
  65         static std::map<boost::rational<int>, std::vector<int>> CADENCES_BY_FRAMERATE = []
  66         {
  67                 std::map<boost::rational<int>, std::vector<int>> result;
  68
  69                 for (core::video_format format : enum_constants<core::video_format>())
  70                 {
  71                         core::video_format_desc desc(format);
  72                         boost::rational<int> format_rate(desc.time_scale, desc.duration);
  73
  74                         result.insert(std::make_pair(format_rate, desc.audio_cadence));
  75                 }
  76
  77                 return result;
  78         }();
  79
  80         auto exact_match = CADENCES_BY_FRAMERATE.find(framerate);
  81
  82         if (exact_match != CADENCES_BY_FRAMERATE.end())
  83                 return exact_match->second;
  84
  85         boost::rational<int> closest_framerate_diff     = std::numeric_limits<int>::max();
  86         boost::rational<int> closest_framerate          = 0;
  87
  88         for (auto format_framerate : CADENCES_BY_FRAMERATE | boost::adaptors::map_keys)
  89         {
  90                 auto diff = boost::abs(framerate - format_framerate);
  91
  92                 if (diff < closest_framerate_diff)
  93                 {
  94                         closest_framerate_diff  = diff;
  95                         closest_framerate               = format_framerate;
  96                 }
  97         }
  98
  99         if (is_logging_quiet_for_thread())
 100                 CASPAR_LOG(debug) << "No exact audio cadence match found for framerate " << to_string(framerate)
 101                         << "\nClosest match is " << to_string(closest_framerate)
 102                         << "\nwhich is a " << to_string(closest_framerate_diff) << " difference.";
 103         else
 104                 CASPAR_LOG(warning) << "No exact audio cadence match found for framerate " << to_string(framerate)
 105                         << "\nClosest match is " << to_string(closest_framerate)
 106                         << "\nwhich is a " << to_string(closest_framerate_diff) << " difference.";
 107
 108         return CADENCES_BY_FRAMERATE[closest_framerate];
 109 }
 110
 111 struct source
 112 {
 113         virtual ~source() { }
 114
 115         virtual std::wstring                            print() const                                                                                   = 0;
 116         virtual void                                            start()                                                                                                 { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 117         virtual void                                            graph(spl::shared_ptr<caspar::diagnostics::graph> g)    { }
 118         virtual void                                            stop()                                                                                                  { }
 119         virtual void                                            start_frame(std::uint32_t frame)                                                { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 120         virtual std::uint32_t                           start_frame() const                                                                             { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 121         virtual void                                            loop(bool value)                                                                                { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 122         virtual bool                                            loop() const                                                                                    { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 123         virtual void                                            length(std::uint32_t frames)                                                    { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 124         virtual std::uint32_t                           length() const                                                                                  { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 125         virtual std::string                                     filename() const                                                                                { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print())); }
 126         virtual void                                            seek(std::uint32_t frame)                                                               { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
 127         virtual bool                                            has_audio() const                                                                               { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 128         virtual int                                                     samplerate() const                                                                              { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 129         virtual bool                                            has_video() const                                                                               { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 130         virtual bool                                            eof() const                                                                                             { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 131         virtual boost::rational<int>            framerate() const                                                                               { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 132         virtual std::uint32_t                           frame_number() const                                                                    { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 133         virtual std::shared_ptr<AVFrame>        get_input_frame(AVMediaType type)                                               { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 134 };
 135
 136 struct no_source_selected : public source
 137 {
 138         std::wstring print() const override
 139         {
 140                 return L"[no_source_selected]";
 141         }
 142 };
 143
 144 class file_source : public source
 145 {
 146         std::wstring                                            filename_;
 147         spl::shared_ptr<diagnostics::graph>     graph_;
 148         std::uint32_t                                           start_frame_    = 0;
 149         std::uint32_t                                           length_                 = std::numeric_limits<std::uint32_t>::max();
 150         bool                                                            loop_                   = false;
 151         mutable boost::mutex                            pointer_mutex_;
 152         std::shared_ptr<input>                          input_;
 153         std::shared_ptr<audio_decoder>          audio_decoder_;
 154         std::shared_ptr<video_decoder>          video_decoder_;
 155         bool                                                            started_                = false;
 156 public:
 157         file_source(std::string filename)
 158                 : filename_(u16(filename))
 159         {
 160         }
 161
 162         std::wstring print() const override
 163         {
 164                 return L"[file_source " + filename_ + L"]";
 165         }
 166
 167         void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
 168         {
 169                 graph_ = std::move(g);
 170         }
 171
 172         void start() override
 173         {
 174                 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
 175                 bool thumbnail_mode = is_logging_quiet_for_thread();
 176                 input_.reset(new input(graph_, filename_, loop_, start_frame_, length_, thumbnail_mode));
 177
 178                 try
 179                 {
 180                         audio_decoder_.reset(new audio_decoder(*input_, core::video_format_desc()));
 181                 }
 182                 catch (averror_stream_not_found&)
 183                 {
 184                         CASPAR_LOG(debug) << print() << " No audio-stream found. Running without audio.";
 185                 }
 186                 catch (...)
 187                 {
 188                         if (is_logging_quiet_for_thread())
 189                         {
 190                                 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
 191                                 CASPAR_LOG(info) << print() << " Failed to open audio-stream. Running without audio. Turn on log level debug to see more information.";
 192                         }
 193                         else
 194                         {
 195                                 CASPAR_LOG_CURRENT_EXCEPTION();
 196                                 CASPAR_LOG(warning) << print() << " Failed to open audio-stream. Running without audio.";
 197                         }
 198                 }
 199
 200                 try
 201                 {
 202                         video_decoder_.reset(new video_decoder(*input_, false));
 203                 }
 204                 catch (averror_stream_not_found&)
 205                 {
 206                         CASPAR_LOG(debug) << print() << " No video-stream found. Running without video.";
 207                 }
 208                 catch (...)
 209                 {
 210                         if (is_logging_quiet_for_thread())
 211                         {
 212                                 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
 213                                 CASPAR_LOG(info) << print() << " Failed to open video-stream. Running without audio. Turn on log level debug to see more information.";
 214                         }
 215                         else
 216                         {
 217                                 CASPAR_LOG_CURRENT_EXCEPTION();
 218                                 CASPAR_LOG(warning) << print() << " Failed to open video-stream. Running without audio.";
 219                         }
 220                 }
 221
 222                 started_ = true;
 223         }
 224
 225         void stop() override
 226         {
 227                 started_ = false;
 228         }
 229
 230         void start_frame(std::uint32_t frame) override
 231         {
 232                 start_frame_ = frame;
 233
 234                 auto i = get_input();
 235                 if (i)
 236                         i->start(frame);
 237         }
 238
 239         std::uint32_t start_frame() const override
 240         {
 241                 return start_frame_;
 242         }
 243
 244         void loop(bool value) override
 245         {
 246                 loop_ = value;
 247
 248                 auto i = get_input();
 249                 if (i)
 250                         i->loop(value);
 251         }
 252
 253         bool loop() const override
 254         {
 255                 return loop_;
 256         }
 257
 258         void length(std::uint32_t frames) override
 259         {
 260                 length_ = frames;
 261
 262                 auto i = get_input();
 263                 if (i)
 264                         i->length(frames);
 265         }
 266
 267         std::uint32_t length() const override
 268         {
 269                 auto v = get_video_decoder();
 270
 271                 if (v)
 272                         return v->nb_frames();
 273
 274                 auto a = get_audio_decoder();
 275
 276                 if (a)
 277                         return a->nb_frames();
 278
 279                 return length_;
 280         }
 281
 282         std::string filename() const override
 283         {
 284                 return u8(filename_);
 285         }
 286
 287         void seek(std::uint32_t frame) override
 288         {
 289                 expect_started();
 290                 get_input()->seek(frame);
 291         }
 292
 293         bool eof() const override
 294         {
 295                 auto i = get_input();
 296                 return !i || i->eof();
 297         }
 298
 299         bool has_audio() const override
 300         {
 301                 return static_cast<bool>(get_audio_decoder());
 302         }
 303
 304         int samplerate() const override
 305         {
 306                 auto decoder = get_audio_decoder();
 307
 308                 if (!decoder)
 309                         return -1;
 310
 311                 return 48000;
 312         }
 313
 314         bool has_video() const override
 315         {
 316                 return static_cast<bool>(get_video_decoder());
 317         }
 318
 319         boost::rational<int> framerate() const override
 320         {
 321                 auto decoder = get_video_decoder();
 322
 323                 if (!decoder)
 324                         return -1;
 325
 326                 return decoder->framerate();
 327         }
 328
 329         std::uint32_t frame_number() const override
 330         {
 331                 auto decoder = get_video_decoder();
 332
 333                 if (!decoder)
 334                         return 0;
 335
 336                 return decoder->file_frame_number();
 337         }
 338
 339         std::shared_ptr<AVFrame> get_input_frame(AVMediaType type) override
 340         {
 341                 auto a_decoder  = get_audio_decoder();
 342                 auto v_decoder  = get_video_decoder();
 343                 expect_started();
 344
 345                 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && a_decoder)
 346                 {
 347                         std::shared_ptr<AVFrame> frame;
 348
 349                         for (int i = 0; i < 64; ++i)
 350                         {
 351                                 frame = (*a_decoder)();
 352
 353                                 if (frame && frame->data[0])
 354                                         return spl::make_shared_ptr(frame);
 355                         }
 356                 }
 357                 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO && v_decoder)
 358                 {
 359                         std::shared_ptr<AVFrame> frame;
 360
 361                         for (int i = 0; i < 128; ++i)
 362                         {
 363                                 frame = (*v_decoder)();
 364
 365                                 if (frame && frame->data[0])
 366                                         return spl::make_shared_ptr(frame);
 367                         }
 368                 }
 369                 else
 370                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(
 371                                 print() + L" Unhandled media type " + boost::lexical_cast<std::wstring>(type)));
 372
 373                 return nullptr;
 374         }
 375 private:
 376         void expect_started() const
 377         {
 378                 if (!started_)
 379                         CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" Not started."));
 380         }
 381
 382         std::shared_ptr<input> get_input() const
 383         {
 384                 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
 385                 return input_;
 386         }
 387
 388         std::shared_ptr<audio_decoder> get_audio_decoder() const
 389         {
 390                 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
 391                 return audio_decoder_;
 392         }
 393
 394         std::shared_ptr<video_decoder> get_video_decoder() const
 395         {
 396                 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
 397                 return video_decoder_;
 398         }
 399 };
 400
 401 class memory_source : public source
 402 {
 403         int                                                                                                                     samplerate_             = -1;
 404         int                                                                                                                     num_channels_   = -1;
 405         int                                                                                                                     width_                  = -1;
 406         int                                                                                                                     height_                 = -1;
 407         boost::rational<int>                                                                            framerate_              = -1;
 408
 409         tbb::atomic<bool>                                                                                       running_;
 410         tbb::concurrent_bounded_queue<caspar::array<const int32_t>>     audio_frames_;
 411         tbb::concurrent_bounded_queue<caspar::array<const uint8_t>>     video_frames_;
 412         int64_t                                                                                                         audio_pts_              = 0;
 413         int64_t                                                                                                         video_pts_              = 0;
 414 public:
 415         memory_source()
 416         {
 417                 running_ = false;
 418                 video_frames_.set_capacity(1);
 419                 audio_frames_.set_capacity(1);
 420         }
 421
 422         ~memory_source()
 423         {
 424                 stop();
 425         }
 426
 427         void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
 428         {
 429         }
 430
 431         std::wstring print() const override
 432         {
 433                 return L"[memory_source]";
 434         }
 435
 436         void enable_audio(int samplerate, int num_channels)
 437         {
 438                 samplerate_ = samplerate;
 439                 num_channels_ = num_channels;
 440         }
 441
 442         void enable_video(int width, int height, boost::rational<int> framerate)
 443         {
 444                 width_ = width;
 445                 height_ = height;
 446         }
 447
 448         void start() override
 449         {
 450                 running_ = true;
 451         }
 452
 453         void stop() override
 454         {
 455                 running_ = false;
 456                 video_frames_.try_push(caspar::array<const uint8_t>());
 457                 audio_frames_.try_push(caspar::array<const int32_t>());
 458         }
 459
 460         bool has_audio() const override
 461         {
 462                 return samplerate_ != -1;
 463         }
 464
 465         int samplerate() const override
 466         {
 467                 return samplerate_;
 468         }
 469
 470         bool has_video() const override
 471         {
 472                 return width_ != -1;
 473         }
 474
 475         bool eof() const override
 476         {
 477                 return !running_;
 478         }
 479
 480         boost::rational<int> framerate() const override
 481         {
 482                 return framerate_;
 483         }
 484
 485         bool try_push_audio(caspar::array<const std::int32_t> data)
 486         {
 487                 if (!has_audio())
 488                         CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" audio not enabled."));
 489
 490                 if (data.empty() || data.size() % num_channels_ != 0)
 491                         CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info(print() + L" audio with incorrect number of channels submitted."));
 492
 493                 return audio_frames_.try_push(std::move(data));
 494         }
 495
 496         bool try_push_video(caspar::array<const std::uint8_t> data)
 497         {
 498                 if (!has_video())
 499                         CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" video not enabled."));
 500
 501                 if (data.size() != width_ * height_ * 4)
 502                         CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info(print() + L" video with incorrect size submitted."));
 503
 504                 return video_frames_.try_push(std::move(data));
 505         }
 506
 507         std::shared_ptr<AVFrame> get_input_frame(AVMediaType type) override
 508         {
 509                 if (!running_)
 510                         CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not running."));
 511
 512                 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && has_audio())
 513                 {
 514                         caspar::array<const std::int32_t> samples;
 515                         audio_frames_.pop(samples);
 516
 517                         if (samples.empty())
 518                                 return nullptr;
 519
 520                         spl::shared_ptr<AVFrame> av_frame(av_frame_alloc(), [samples](AVFrame* p) { av_frame_free(&p); });
 521
 522                         av_frame->channels                      = num_channels_;
 523                         av_frame->channel_layout        = av_get_default_channel_layout(num_channels_);
 524                         av_frame->sample_rate           = samplerate_;
 525                         av_frame->nb_samples            = static_cast<int>(samples.size()) / num_channels_;
 526                         av_frame->format                        = AV_SAMPLE_FMT_S32;
 527                         av_frame->pts                           = audio_pts_;
 528
 529                         audio_pts_ += av_frame->nb_samples;
 530
 531                         FF(av_samples_fill_arrays(
 532                                         av_frame->extended_data,
 533                                         av_frame->linesize,
 534                                         reinterpret_cast<const std::uint8_t*>(&*samples.begin()),
 535                                         av_frame->channels,
 536                                         av_frame->nb_samples,
 537                                         static_cast<AVSampleFormat>(av_frame->format),
 538                                         16));
 539
 540                         return av_frame;
 541                 }
 542                 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO && has_video())
 543                 {
 544                         caspar::array<const std::uint8_t> data;
 545                         video_frames_.pop(data);
 546
 547                         if (data.empty())
 548                                 return nullptr;
 549
 550                         spl::shared_ptr<AVFrame> av_frame(av_frame_alloc(), [data](AVFrame* p) { av_frame_free(&p); });
 551                         avcodec_get_frame_defaults(av_frame.get());
 552
 553                         const auto sample_aspect_ratio = boost::rational<int>(width_, height_);
 554
 555                         av_frame->format                                  = AV_PIX_FMT_BGRA;
 556                         av_frame->width                                   = width_;
 557                         av_frame->height                                  = height_;
 558                         av_frame->sample_aspect_ratio.num = sample_aspect_ratio.numerator();
 559                         av_frame->sample_aspect_ratio.den = sample_aspect_ratio.denominator();
 560                         av_frame->pts                                     = video_pts_;
 561
 562                         video_pts_ += 1;
 563
 564                         FF(av_image_fill_arrays(
 565                                         av_frame->data,
 566                                         av_frame->linesize,
 567                                         data.begin(),
 568                                         static_cast<AVPixelFormat>(av_frame->format),
 569                                         width_,
 570                                         height_,
 571                                         1));
 572
 573                         return av_frame;
 574                 }
 575                 else
 576                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(
 577                                 print() + L" Unhandled media type " + boost::lexical_cast<std::wstring>(type)));
 578         }
 579 };
 580
 581 struct sink
 582 {
 583         virtual ~sink() { }
 584
 585         virtual std::wstring                                    print() const                                                                                           = 0;
 586         virtual void                                                    graph(spl::shared_ptr<caspar::diagnostics::graph> g)            { }
 587         virtual void                                                    acodec(std::string codec)                                                                       { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
 588         virtual void                                                    vcodec(std::string codec)                                                                       { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
 589         virtual void                                                    format(std::string fmt)                                                                         { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
 590         virtual void                                                    framerate(boost::rational<int> framerate)                                       { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
 591         virtual void                                                    start(bool has_audio, bool has_video)                                           { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 592         virtual void                                                    stop()                                                                                                          { }
 593         virtual std::vector<AVSampleFormat>             supported_sample_formats() const                                                        { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 594         virtual std::vector<int>                                supported_samplerates() const                                                           { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 595         virtual std::vector<AVPixelFormat>              supported_pixel_formats() const                                                         { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 596         virtual boost::optional<AVMediaType>    try_push(AVMediaType type, spl::shared_ptr<AVFrame> frame)      { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 597         virtual void                                                    eof()                                                                                                           { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
 598 };
 599
 600 struct no_sink_selected : public sink
 601 {
 602         std::wstring print() const override
 603         {
 604                 return L"[no_sink_selected]";
 605         }
 606 };
 607
 608 class file_sink : public sink
 609 {
 610         std::wstring                                            filename_;
 611         spl::shared_ptr<diagnostics::graph>     graph_;
 612 public:
 613         file_sink(std::string filename)
 614                 : filename_(u16(std::move(filename)))
 615         {
 616         }
 617
 618         std::wstring print() const override
 619         {
 620                 return L"[file_sink " + filename_ + L"]";
 621         }
 622
 623         void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
 624         {
 625                 graph_ = std::move(g);
 626         }
 627 };
 628
 629 class memory_sink : public sink
 630 {
 631         spl::shared_ptr<core::frame_factory>                    factory_;
 632
 633         bool                                                                                    has_audio_                      = false;
 634         bool                                                                                    has_video_                      = false;
 635         std::vector<int>                                                                audio_cadence_;
 636         core::audio_channel_layout                                              channel_layout_         = core::audio_channel_layout::invalid();
 637         core::mutable_audio_buffer                                              audio_samples_;
 638
 639         std::queue<std::shared_ptr<AVFrame>>                    video_frames_;
 640
 641         tbb::concurrent_bounded_queue<core::draw_frame> output_frames_;
 642         tbb::atomic<bool>                                                               running_;
 643 public:
 644         memory_sink(spl::shared_ptr<core::frame_factory> factory, core::video_format_desc format)
 645                 : factory_(std::move(factory))
 646                 , audio_cadence_(format.audio_cadence)
 647         {
 648                 output_frames_.set_capacity(2);
 649                 running_ = false;
 650                 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
 651                 // This cadence fills the audio mixer most optimally.
 652                 boost::range::rotate(audio_cadence_, std::end(audio_cadence_) - 1);
 653         }
 654
 655         ~memory_sink()
 656         {
 657                 stop();
 658         }
 659
 660         std::wstring print() const override
 661         {
 662                 return L"[memory_sink]";
 663         }
 664
 665         void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
 666         {
 667         }
 668
 669         void framerate(boost::rational<int> framerate) override
 670         {
 671                 audio_cadence_ = find_audio_cadence(framerate);
 672                 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
 673                 // This cadence fills the audio mixer most optimally.
 674                 boost::range::rotate(audio_cadence_, std::end(audio_cadence_) - 1);
 675         }
 676
 677         void start(bool has_audio, bool has_video) override
 678         {
 679                 has_audio_      = has_audio;
 680                 has_video_      = has_video;
 681                 running_        = true;
 682         }
 683
 684         void stop() override
 685         {
 686                 running_ = false;
 687                 try_pop_frame();
 688                 try_pop_frame();
 689         }
 690
 691         std::vector<AVSampleFormat> supported_sample_formats() const override
 692         {
 693                 return { AVSampleFormat::AV_SAMPLE_FMT_S32 };
 694         }
 695
 696         std::vector<int> supported_samplerates() const override {
 697                 return { 48000 };
 698         }
 699
 700         std::vector<AVPixelFormat> supported_pixel_formats() const override
 701         {
 702                 return {
 703                         AVPixelFormat::AV_PIX_FMT_YUVA420P,
 704                         AVPixelFormat::AV_PIX_FMT_YUV444P,
 705                         AVPixelFormat::AV_PIX_FMT_YUV422P,
 706                         AVPixelFormat::AV_PIX_FMT_YUV420P,
 707                         AVPixelFormat::AV_PIX_FMT_YUV411P,
 708                         AVPixelFormat::AV_PIX_FMT_BGRA,
 709                         AVPixelFormat::AV_PIX_FMT_ARGB,
 710                         AVPixelFormat::AV_PIX_FMT_RGBA,
 711                         AVPixelFormat::AV_PIX_FMT_ABGR,
 712                         AVPixelFormat::AV_PIX_FMT_GRAY8
 713                 };
 714         }
 715
 716         boost::optional<AVMediaType> try_push(AVMediaType type, spl::shared_ptr<AVFrame> av_frame) override
 717         {
 718                 if (!has_audio_ && !has_video_)
 719                         CASPAR_THROW_EXCEPTION(invalid_operation());
 720
 721                 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && av_frame->data[0])
 722                 {
 723                         if (channel_layout_ == core::audio_channel_layout::invalid()) // First audio
 724                         {
 725                                 channel_layout_ = get_audio_channel_layout(av_frame->channels, av_frame->channel_layout, L"");
 726
 727                                 // Insert silence samples so that the audio mixer is guaranteed to be filled.
 728                                 auto min_num_samples_per_frame  = *boost::min_element(audio_cadence_);
 729                                 auto max_num_samples_per_frame  = *boost::max_element(audio_cadence_);
 730                                 auto cadence_safety_samples             = max_num_samples_per_frame - min_num_samples_per_frame;
 731                                 audio_samples_.resize(channel_layout_.num_channels * cadence_safety_samples, 0);
 732                         }
 733
 734                         auto ptr = reinterpret_cast<int32_t*>(av_frame->data[0]);
 735
 736                         audio_samples_.insert(audio_samples_.end(), ptr, ptr + av_frame->linesize[0] / sizeof(int32_t));
 737                 }
 738                 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO)
 739                 {
 740                         video_frames_.push(std::move(av_frame));
 741                 }
 742
 743                 while (true)
 744                 {
 745                         bool enough_audio =
 746                                 !has_audio_ ||
 747                                 (channel_layout_ != core::audio_channel_layout::invalid() && audio_samples_.size() >= audio_cadence_.front() * channel_layout_.num_channels);
 748                         bool enough_video =
 749                                 !has_video_ ||
 750                                 !video_frames_.empty();
 751
 752                         if (!enough_audio)
 753                                 return AVMediaType::AVMEDIA_TYPE_AUDIO;
 754
 755                         if (!enough_video)
 756                                 return AVMediaType::AVMEDIA_TYPE_VIDEO;
 757
 758                         core::mutable_audio_buffer audio_data;
 759
 760                         if (has_audio_)
 761                         {
 762                                 auto begin = audio_samples_.begin();
 763                                 auto end = begin + audio_cadence_.front() * channel_layout_.num_channels;
 764
 765                                 audio_data.insert(audio_data.begin(), begin, end);
 766                                 audio_samples_.erase(begin, end);
 767                                 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_) + 1);
 768                         }
 769
 770                         if (!has_video_) // Audio only
 771                         {
 772                                 core::mutable_frame audio_only_frame(
 773                                                 { },
 774                                                 std::move(audio_data),
 775                                                 this,
 776                                                 core::pixel_format_desc(core::pixel_format::invalid),
 777                                                 channel_layout_);
 778
 779                                 output_frames_.push(core::draw_frame(std::move(audio_only_frame)));
 780
 781                                 return AVMediaType::AVMEDIA_TYPE_AUDIO;
 782                         }
 783
 784                         auto output_frame = make_frame(this, spl::make_shared_ptr(video_frames_.front()), *factory_, channel_layout_);
 785                         video_frames_.pop();
 786                         output_frame.audio_data() = std::move(audio_data);
 787
 788                         output_frames_.push(core::draw_frame(std::move(output_frame)));
 789                 }
 790         }
 791
 792         void eof() override
 793         {
 794                 // Drain rest, regardless of it being enough or not.
 795                 while (!video_frames_.empty() || !audio_samples_.empty())
 796                 {
 797                         core::mutable_audio_buffer audio_data;
 798
 799                         audio_data.swap(audio_samples_);
 800
 801                         if (!video_frames_.empty())
 802                         {
 803                                 auto output_frame = make_frame(this, spl::make_shared_ptr(video_frames_.front()), *factory_, channel_layout_);
 804                                 video_frames_.pop();
 805                                 output_frame.audio_data() = std::move(audio_data);
 806
 807                                 output_frames_.push(core::draw_frame(std::move(output_frame)));
 808                         }
 809                         else
 810                         {
 811                                 core::mutable_frame audio_only_frame(
 812                                                 {},
 813                                                 std::move(audio_data),
 814                                                 this,
 815                                                 core::pixel_format_desc(core::pixel_format::invalid),
 816                                                 channel_layout_);
 817
 818                                 output_frames_.push(core::draw_frame(std::move(audio_only_frame)));
 819                                 output_frames_.push(core::draw_frame::empty());
 820                         }
 821                 }
 822         }
 823
 824         core::draw_frame try_pop_frame()
 825         {
 826                 core::draw_frame frame = core::draw_frame::late();
 827
 828                 if (!output_frames_.try_pop(frame) && !running_)
 829                         return core::draw_frame::empty();
 830
 831                 return frame;
 832         }
 833 };
 834
 835 class ffmpeg_pipeline_backend_internal : public ffmpeg_pipeline_backend
 836 {
 837         spl::shared_ptr<diagnostics::graph>                                                             graph_;
 838
 839         spl::unique_ptr<source>                                                                                 source_                                 = spl::make_unique<no_source_selected>();
 840         std::function<bool (caspar::array<const std::int32_t> data)>    try_push_audio_;
 841         std::function<bool (caspar::array<const std::uint8_t> data)>    try_push_video_;
 842
 843         int                                                                                                                             source_num_channels_    = 0;
 844         AVSampleFormat                                                                                                  source_sampleformat_    = AVSampleFormat::AV_SAMPLE_FMT_NONE;
 845         int                                                                                                                             source_width_                   = 0;
 846         int                                                                                                                             source_height_                  = 0;
 847         AVPixelFormat                                                                                                   source_pixelformat_             = AVPixelFormat::AV_PIX_FMT_NONE;
 848         core::field_mode                                                                                                source_fieldmode_               = core::field_mode::progressive;
 849
 850         std::string                                                                                                             afilter_;
 851         std::unique_ptr<audio_filter>                                                                   audio_filter_;
 852         std::string                                                                                                             vfilter_;
 853         std::unique_ptr<filter>                                                                                 video_filter_;
 854
 855         spl::unique_ptr<sink>                                                                                   sink_                                   = spl::make_unique<no_sink_selected>();
 856         std::function<core::draw_frame ()>                                                              try_pop_frame_;
 857
 858         tbb::atomic<bool>                                                                                               started_;
 859         boost::thread                                                                                                   thread_;
 860 public:
 861         ffmpeg_pipeline_backend_internal()
 862         {
 863                 started_ = false;
 864                 diagnostics::register_graph(graph_);
 865         }
 866
 867         ~ffmpeg_pipeline_backend_internal()
 868         {
 869                 stop();
 870         }
 871
 872         void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
 873         {
 874                 graph_ = std::move(g);
 875                 source_->graph(graph_);
 876                 sink_->graph(graph_);
 877         }
 878
 879         // Source setup
 880
 881         void from_file(std::string filename) override
 882         {
 883                 source_                 = spl::make_unique<file_source>(std::move(filename));
 884                 try_push_audio_ = std::function<bool (caspar::array<const std::int32_t>)>();
 885                 try_push_video_ = std::function<bool (caspar::array<const std::uint8_t>)>();
 886                 source_->graph(graph_);
 887         }
 888
 889         void from_memory_only_audio(int num_channels, int samplerate) override
 890         {
 891                 auto source             = spl::make_unique<memory_source>();
 892                 auto source_ptr = source.get();
 893                 try_push_audio_ = [this, source_ptr](caspar::array<const std::int32_t> data) { return source_ptr->try_push_audio(std::move(data)); };
 894                 source->enable_audio(samplerate, num_channels);
 895
 896                 source_ = std::move(source);
 897                 source_->graph(graph_);
 898         }
 899
 900         void from_memory_only_video(int width, int height, boost::rational<int> framerate) override
 901         {
 902                 auto source             = spl::make_unique<memory_source>();
 903                 auto source_ptr = source.get();
 904                 try_push_video_ = [this, source_ptr](caspar::array<const std::uint8_t> data) { return source_ptr->try_push_video(std::move(data)); };
 905                 source->enable_video(width, height, std::move(framerate));
 906
 907                 source_ = std::move(source);
 908                 source_->graph(graph_);
 909         }
 910
 911         void from_memory(int num_channels, int samplerate, int width, int height, boost::rational<int> framerate) override
 912         {
 913                 auto source             = spl::make_unique<memory_source>();
 914                 auto source_ptr = source.get();
 915                 try_push_audio_ = [this, source_ptr](caspar::array<const std::int32_t> data) { return source_ptr->try_push_audio(std::move(data)); };
 916                 try_push_video_ = [this, source_ptr](caspar::array<const std::uint8_t> data) { return source_ptr->try_push_video(std::move(data)); };
 917                 source->enable_audio(samplerate, num_channels);
 918                 source->enable_video(width, height, std::move(framerate));
 919
 920                 source_ = std::move(source);
 921                 source_->graph(graph_);
 922         }
 923
 924         void                    start_frame(std::uint32_t frame) override       { source_->start_frame(frame);          }
 925         std::uint32_t   start_frame() const override                            { return source_->start_frame();        }
 926         void                    length(std::uint32_t frames) override           { source_->length(frames);                      }
 927         std::uint32_t   length() const override                                         { return source_->length();                     }
 928         void                    seek(std::uint32_t frame) override                      { source_->seek(frame);                         }
 929         void                    loop(bool value) override                                       { source_->loop(value);                         }
 930         bool                    loop() const override                                           { return source_->loop();                       }
 931         std::string             source_filename() const override                        { return source_->filename();           }
 932
 933         // Filter setup
 934
 935         void vfilter(std::string filter) override
 936         {
 937                 vfilter_ = std::move(filter);
 938         }
 939
 940         void afilter(std::string filter) override
 941         {
 942                 afilter_ = std::move(filter);
 943         }
 944
 945         int width() const override
 946         {
 947                 return source_width_;
 948         }
 949
 950         int height() const override
 951         {
 952                 return source_height_;
 953         }
 954
 955         boost::rational<int> framerate() const override
 956         {
 957                 bool double_rate = filter::is_double_rate(u16(vfilter_));
 958
 959                 return double_rate ? source_->framerate() * 2 : source_->framerate();
 960         }
 961
 962         bool progressive() const override
 963         {
 964                 return true;//TODO
 965         }
 966
 967         // Sink setup
 968
 969         void to_memory(spl::shared_ptr<core::frame_factory> factory, core::video_format_desc format) override
 970         {
 971                 auto sink               = spl::make_unique<memory_sink>(std::move(factory), std::move(format));
 972                 auto sink_ptr   = sink.get();
 973                 try_pop_frame_  = [sink_ptr] { return sink_ptr->try_pop_frame(); };
 974
 975                 sink_ = std::move(sink);
 976                 sink_->graph(graph_);
 977         }
 978
 979         void to_file(std::string filename) override
 980         {
 981                 sink_                   = spl::make_unique<file_sink>(std::move(filename));
 982                 try_pop_frame_  = std::function<core::draw_frame ()>();
 983                 sink_->graph(graph_);
 984         }
 985
 986         void acodec(std::string codec) override { sink_->acodec(std::move(codec)); }
 987         void vcodec(std::string codec) override { sink_->vcodec(std::move(codec)); }
 988         void format(std::string fmt) override   { sink_->format(std::move(fmt)); }
 989
 990         // Runtime control
 991
 992         void start() override
 993         {
 994                 source_->start();
 995                 sink_->start(source_->has_audio(), source_->has_video());
 996                 started_ = true;
 997                 bool quiet = is_logging_quiet_for_thread();
 998
 999                 thread_ = boost::thread([=] { run(quiet); });
1000         }
1001
1002         bool try_push_audio(caspar::array<const std::int32_t> data) override
1003         {
1004                 if (try_push_audio_)
1005                         return try_push_audio_(std::move(data));
1006                 else
1007                         return false;
1008         }
1009
1010         bool try_push_video(caspar::array<const std::uint8_t> data) override
1011         {
1012                 if (try_push_video_)
1013                         return try_push_video_(std::move(data));
1014                 else
1015                         return false;
1016         }
1017
1018         core::draw_frame try_pop_frame() override
1019         {
1020                 if (!try_pop_frame_)
1021                         CASPAR_THROW_EXCEPTION(invalid_operation());
1022
1023                 return try_pop_frame_();
1024         }
1025
1026         std::uint32_t last_frame() const override
1027         {
1028                 return source_->frame_number();
1029         }
1030
1031         bool started() const override
1032         {
1033                 return started_;
1034         }
1035
1036         void stop() override
1037         {
1038                 started_ = false;
1039
1040                 sink_->stop();
1041                 source_->stop();
1042
1043                 if (thread_.joinable())
1044                         thread_.join();
1045         }
1046
1047 private:
1048         void run(bool quiet)
1049         {
1050                 ensure_gpf_handler_installed_for_thread(u8(L"ffmpeg-pipeline: " + source_->print() + L" -> " + sink_->print()).c_str());
1051                 auto quiet_logging = temporary_enable_quiet_logging_for_thread(quiet);
1052
1053                 try
1054                 {
1055                         boost::optional<AVMediaType> result = source_->has_audio() ? AVMediaType::AVMEDIA_TYPE_AUDIO : AVMediaType::AVMEDIA_TYPE_VIDEO;
1056
1057                         while (started_ && (source_->has_audio() || source_->has_video()))
1058                         {
1059                                 auto needed = *result;
1060                                 auto input_frame = source_->get_input_frame(needed);
1061
1062                                 if (input_frame)
1063                                 {
1064                                         if (needed == AVMediaType::AVMEDIA_TYPE_AUDIO)
1065                                         {
1066                                                 result = sink_->try_push(AVMediaType::AVMEDIA_TYPE_AUDIO, spl::make_shared_ptr(std::move(input_frame)));
1067                                         }
1068                                         else if (needed == AVMediaType::AVMEDIA_TYPE_VIDEO)
1069                                         {
1070                                                 initialize_video_filter_if_needed(*input_frame);
1071                                                 video_filter_->push(std::move(input_frame));
1072
1073                                                 for (auto filtered_frame : video_filter_->poll_all())
1074                                                 {
1075                                                         result = sink_->try_push(AVMediaType::AVMEDIA_TYPE_VIDEO, std::move(filtered_frame));
1076                                                 }
1077                                         }
1078                                         else
1079                                                 CASPAR_THROW_EXCEPTION(not_supported());
1080                                 }
1081                                 else if (source_->eof())
1082                                 {
1083                                         started_ = false;
1084                                         sink_->eof();
1085                                         break;
1086                                 }
1087                                 else
1088                                         result = boost::none;
1089
1090                                 if (!result)
1091                                 {
1092                                         graph_->set_tag(caspar::diagnostics::tag_severity::WARNING, "dropped-frame");
1093                                         result = needed; // Repeat same media type
1094                                 }
1095                         }
1096                 }
1097                 catch (...)
1098                 {
1099                         if (is_logging_quiet_for_thread())
1100                         {
1101                                 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
1102                         }
1103                         else
1104                         {
1105                                 CASPAR_LOG_CURRENT_EXCEPTION();
1106                         }
1107                 }
1108
1109                 video_filter_.reset();
1110                 audio_filter_.reset();
1111                 source_->stop();
1112                 sink_->stop();
1113                 started_ = false;
1114         }
1115
1116         template<typename T>
1117         void set_if_changed(bool& changed, T& old_value, T new_value)
1118         {
1119                 if (old_value != new_value)
1120                 {
1121                         changed = true;
1122                         old_value = new_value;
1123                 }
1124         }
1125
1126         void initialize_audio_filter_if_needed(const AVFrame& av_frame)
1127         {
1128                 bool changed = false;
1129
1130                 set_if_changed(changed, source_sampleformat_, static_cast<AVSampleFormat>(av_frame.format));
1131                 set_if_changed(changed, source_num_channels_, av_frame.channels);
1132
1133                 if (changed)
1134                         initialize_audio_filter();
1135         }
1136
1137         void initialize_audio_filter()
1138         {
1139                 audio_filter_.reset(new audio_filter(
1140                                 boost::rational<int>(1, source_->samplerate()),
1141                                 source_->samplerate(),
1142                                 source_sampleformat_,
1143                                 av_get_default_channel_layout(source_num_channels_),
1144                                 sink_->supported_samplerates(),
1145                                 sink_->supported_sample_formats(),
1146                                 {},
1147                                 afilter_));
1148         }
1149
1150         void initialize_video_filter_if_needed(const AVFrame& av_frame)
1151         {
1152                 bool changed = false;
1153
1154                 set_if_changed(changed, source_width_, av_frame.width);
1155                 set_if_changed(changed, source_height_, av_frame.height);
1156                 set_if_changed(changed, source_pixelformat_, static_cast<AVPixelFormat>(av_frame.format));
1157
1158                 core::field_mode field_mode = core::field_mode::progressive;
1159
1160                 if (av_frame.interlaced_frame)
1161                         field_mode = av_frame.top_field_first ? core::field_mode::upper : core::field_mode::lower;
1162
1163                 set_if_changed(changed, source_fieldmode_, field_mode);
1164
1165                 if (changed)
1166                         initialize_video_filter();
1167         }
1168
1169         void initialize_video_filter()
1170         {
1171                 if (source_fieldmode_ != core::field_mode::progressive && !filter::is_deinterlacing(u16(vfilter_)))
1172                         vfilter_ = u8(append_filter(u16(vfilter_), L"YADIF=1:-1"));
1173
1174                 if (source_height_ == 480) // NTSC DV
1175                 {
1176                         auto pad_str = L"PAD=" + boost::lexical_cast<std::wstring>(source_width_) + L":486:0:2:black";
1177                         vfilter_ = u8(append_filter(u16(vfilter_), pad_str));
1178                 }
1179
1180                 video_filter_.reset(new filter(
1181                                 source_width_,
1182                                 source_height_,
1183                                 1 / source_->framerate(),
1184                                 source_->framerate(),
1185                                 boost::rational<int>(1, 1), // TODO
1186                                 source_pixelformat_,
1187                                 sink_->supported_pixel_formats(),
1188                                 vfilter_));
1189                 sink_->framerate(framerate());
1190         }
1191 };
1192
1193 spl::shared_ptr<struct ffmpeg_pipeline_backend> create_internal_pipeline()
1194 {
1195         return spl::make_shared<ffmpeg_pipeline_backend_internal>();
1196 }
1197
1198 }}