2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Helge Norberg, helge.norberg@svt.se
24 #include "ffmpeg_pipeline_backend.h"
25 #include "ffmpeg_pipeline_backend_internal.h"
26 #include "producer/input/input.h"
27 #include "producer/video/video_decoder.h"
28 #include "producer/audio/audio_decoder.h"
29 #include "producer/filter/audio_filter.h"
30 #include "producer/filter/filter.h"
31 #include "producer/util/util.h"
32 #include "ffmpeg_error.h"
35 #include <common/diagnostics/graph.h>
36 #include <common/os/general_protection_fault.h>
37 #include <common/enum_class.h>
39 #include <core/frame/audio_channel_layout.h>
40 #include <core/frame/frame.h>
41 #include <core/frame/frame_factory.h>
42 #include <core/video_format.h>
49 #include <tbb/atomic.h>
50 #include <tbb/concurrent_queue.h>
51 #include <tbb/spin_mutex.h>
53 #include <boost/thread.hpp>
54 #include <boost/optional.hpp>
55 #include <boost/exception_ptr.hpp>
57 namespace caspar { namespace ffmpeg {
59 std::string to_string(const boost::rational<int>& framerate)
61 return boost::lexical_cast<std::string>(framerate.numerator())
62 + "/" + boost::lexical_cast<std::string>(framerate.denominator()) + " (" + boost::lexical_cast<std::string>(static_cast<double>(framerate.numerator()) / static_cast<double>(framerate.denominator())) + ") fps";
65 std::vector<int> find_audio_cadence(const boost::rational<int>& framerate)
67 static std::map<boost::rational<int>, std::vector<int>> CADENCES_BY_FRAMERATE = []
69 std::map<boost::rational<int>, std::vector<int>> result;
71 for (core::video_format format : enum_constants<core::video_format>())
73 core::video_format_desc desc(format);
74 boost::rational<int> format_rate(desc.time_scale, desc.duration);
76 result.insert(std::make_pair(format_rate, desc.audio_cadence));
82 auto exact_match = CADENCES_BY_FRAMERATE.find(framerate);
84 if (exact_match != CADENCES_BY_FRAMERATE.end())
85 return exact_match->second;
87 boost::rational<int> closest_framerate_diff = std::numeric_limits<int>::max();
88 boost::rational<int> closest_framerate = 0;
90 for (auto format_framerate : CADENCES_BY_FRAMERATE | boost::adaptors::map_keys)
92 auto diff = boost::abs(framerate - format_framerate);
94 if (diff < closest_framerate_diff)
96 closest_framerate_diff = diff;
97 closest_framerate = format_framerate;
101 if (is_logging_quiet_for_thread())
102 CASPAR_LOG(debug) << "No exact audio cadence match found for framerate " << to_string(framerate)
103 << "\nClosest match is " << to_string(closest_framerate)
104 << "\nwhich is a " << to_string(closest_framerate_diff) << " difference.";
106 CASPAR_LOG(warning) << "No exact audio cadence match found for framerate " << to_string(framerate)
107 << "\nClosest match is " << to_string(closest_framerate)
108 << "\nwhich is a " << to_string(closest_framerate_diff) << " difference.";
110 return CADENCES_BY_FRAMERATE[closest_framerate];
115 virtual ~source() { }
117 virtual std::wstring print() const = 0;
118 virtual void start() { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
119 virtual void graph(spl::shared_ptr<caspar::diagnostics::graph> g) { }
120 virtual void stop() { }
121 virtual void start_frame(std::uint32_t frame) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
122 virtual std::uint32_t start_frame() const { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
123 virtual void loop(bool value) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
124 virtual bool loop() const { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
125 virtual void length(std::uint32_t frames) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
126 virtual std::uint32_t length() const { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
127 virtual std::string filename() const { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print())); }
128 virtual void seek(std::uint32_t frame) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
129 virtual bool has_audio() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
130 virtual int samplerate() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
131 virtual bool has_video() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
132 virtual bool eof() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
133 virtual boost::rational<int> framerate() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
134 virtual std::uint32_t frame_number() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
135 virtual std::vector<std::shared_ptr<AVFrame>> get_input_frames_for_streams(AVMediaType type) { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
138 struct no_source_selected : public source
140 std::wstring print() const override
142 return L"[no_source_selected]";
146 class file_source : public source
148 std::wstring filename_;
149 spl::shared_ptr<diagnostics::graph> graph_;
150 std::uint32_t start_frame_ = 0;
151 std::uint32_t length_ = std::numeric_limits<std::uint32_t>::max();
153 mutable boost::mutex pointer_mutex_;
154 std::shared_ptr<input> input_;
155 std::vector<spl::shared_ptr<audio_decoder>> audio_decoders_;
156 std::shared_ptr<video_decoder> video_decoder_;
157 bool started_ = false;
159 file_source(std::string filename)
160 : filename_(u16(filename))
164 std::wstring print() const override
166 return L"[file_source " + filename_ + L"]";
169 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
171 graph_ = std::move(g);
174 void start() override
176 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
177 bool thumbnail_mode = is_logging_quiet_for_thread();
178 input_.reset(new input(graph_, filename_, loop_, start_frame_, length_, thumbnail_mode));
180 for (int i = 0; i < input_->num_audio_streams(); ++i)
184 audio_decoders_.push_back(spl::make_shared<audio_decoder>(*input_, core::video_format_desc(), i));
188 if (is_logging_quiet_for_thread())
190 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
191 CASPAR_LOG(info) << print() << " Failed to open audio-stream. Turn on log level debug to see more information.";
195 CASPAR_LOG_CURRENT_EXCEPTION();
196 CASPAR_LOG(warning) << print() << " Failed to open audio-stream.";
201 if (audio_decoders_.empty())
202 CASPAR_LOG(debug) << print() << " No audio-stream found. Running without audio.";
206 video_decoder_.reset(new video_decoder(*input_, false));
208 catch (averror_stream_not_found&)
210 CASPAR_LOG(debug) << print() << " No video-stream found. Running without video.";
214 if (is_logging_quiet_for_thread())
216 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
217 CASPAR_LOG(info) << print() << " Failed to open video-stream. Running without audio. Turn on log level debug to see more information.";
221 CASPAR_LOG_CURRENT_EXCEPTION();
222 CASPAR_LOG(warning) << print() << " Failed to open video-stream. Running without audio.";
234 void start_frame(std::uint32_t frame) override
236 start_frame_ = frame;
238 auto i = get_input();
243 std::uint32_t start_frame() const override
248 void loop(bool value) override
252 auto i = get_input();
257 bool loop() const override
262 void length(std::uint32_t frames) override
266 auto i = get_input();
271 std::uint32_t length() const override
273 auto v = get_video_decoder();
276 return v->nb_frames();
278 auto a = get_audio_decoders();
281 return a.at(0)->nb_frames(); // Should be ok.
286 std::string filename() const override
288 return u8(filename_);
291 void seek(std::uint32_t frame) override
294 get_input()->seek(frame);
297 bool eof() const override
299 auto i = get_input();
300 return !i || i->eof();
303 bool has_audio() const override
305 return !get_audio_decoders().empty();
308 int samplerate() const override
310 if (get_audio_decoders().empty())
316 bool has_video() const override
318 return static_cast<bool>(get_video_decoder());
321 boost::rational<int> framerate() const override
323 auto decoder = get_video_decoder();
328 return decoder->framerate();
331 std::uint32_t frame_number() const override
333 auto decoder = get_video_decoder();
338 return decoder->file_frame_number();
341 std::vector<std::shared_ptr<AVFrame>> get_input_frames_for_streams(AVMediaType type) override
343 auto a_decoders = get_audio_decoders();
344 auto v_decoder = get_video_decoder();
347 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && !a_decoders.empty())
349 std::vector<std::shared_ptr<AVFrame>> frames;
351 for (auto& a_decoder : a_decoders)
353 std::shared_ptr<AVFrame> frame;
355 for (int i = 0; i < 64; ++i)
357 frame = (*a_decoder)();
359 if (frame && frame->data[0])
365 frames.push_back(std::move(frame));
370 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO && v_decoder)
372 std::shared_ptr<AVFrame> frame;
374 for (int i = 0; i < 128; ++i)
376 frame = (*v_decoder)();
378 if (frame && frame->data[0])
383 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(
384 print() + L" Unhandled media type " + boost::lexical_cast<std::wstring>(type)));
389 void expect_started() const
392 CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" Not started."));
395 std::shared_ptr<input> get_input() const
397 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
401 std::vector<spl::shared_ptr<audio_decoder>> get_audio_decoders() const
403 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
404 return audio_decoders_;
407 std::shared_ptr<video_decoder> get_video_decoder() const
409 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
410 return video_decoder_;
414 class memory_source : public source
416 int samplerate_ = -1;
417 int num_channels_ = -1;
420 boost::rational<int> framerate_ = -1;
422 tbb::atomic<bool> running_;
423 tbb::concurrent_bounded_queue<caspar::array<const int32_t>> audio_frames_;
424 tbb::concurrent_bounded_queue<caspar::array<const uint8_t>> video_frames_;
425 int64_t audio_pts_ = 0;
426 int64_t video_pts_ = 0;
431 video_frames_.set_capacity(1);
432 audio_frames_.set_capacity(1);
440 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
444 std::wstring print() const override
446 return L"[memory_source]";
449 void enable_audio(int samplerate, int num_channels)
451 samplerate_ = samplerate;
452 num_channels_ = num_channels;
455 void enable_video(int width, int height, boost::rational<int> framerate)
461 void start() override
469 video_frames_.try_push(caspar::array<const uint8_t>());
470 audio_frames_.try_push(caspar::array<const int32_t>());
473 bool has_audio() const override
475 return samplerate_ != -1;
478 int samplerate() const override
483 bool has_video() const override
488 bool eof() const override
493 boost::rational<int> framerate() const override
498 bool try_push_audio(caspar::array<const std::int32_t> data)
501 CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" audio not enabled."));
503 if (data.empty() || data.size() % num_channels_ != 0)
504 CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info(print() + L" audio with incorrect number of channels submitted."));
506 return audio_frames_.try_push(std::move(data));
509 bool try_push_video(caspar::array<const std::uint8_t> data)
512 CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" video not enabled."));
514 if (data.size() != width_ * height_ * 4)
515 CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info(print() + L" video with incorrect size submitted."));
517 return video_frames_.try_push(std::move(data));
520 std::vector<std::shared_ptr<AVFrame>> get_input_frames_for_streams(AVMediaType type) override
523 CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not running."));
525 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && has_audio())
527 caspar::array<const std::int32_t> samples;
528 audio_frames_.pop(samples);
533 spl::shared_ptr<AVFrame> av_frame(av_frame_alloc(), [samples](AVFrame* p) { av_frame_free(&p); });
535 av_frame->channels = num_channels_;
536 av_frame->channel_layout = av_get_default_channel_layout(num_channels_);
537 av_frame->sample_rate = samplerate_;
538 av_frame->nb_samples = static_cast<int>(samples.size()) / num_channels_;
539 av_frame->format = AV_SAMPLE_FMT_S32;
540 av_frame->pts = audio_pts_;
542 audio_pts_ += av_frame->nb_samples;
544 FF(av_samples_fill_arrays(
545 av_frame->extended_data,
547 reinterpret_cast<const std::uint8_t*>(&*samples.begin()),
549 av_frame->nb_samples,
550 static_cast<AVSampleFormat>(av_frame->format),
555 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO && has_video())
557 caspar::array<const std::uint8_t> data;
558 video_frames_.pop(data);
563 spl::shared_ptr<AVFrame> av_frame(av_frame_alloc(), [data](AVFrame* p) { av_frame_free(&p); });
564 avcodec_get_frame_defaults(av_frame.get());
566 const auto sample_aspect_ratio = boost::rational<int>(width_, height_);
568 av_frame->format = AV_PIX_FMT_BGRA;
569 av_frame->width = width_;
570 av_frame->height = height_;
571 av_frame->sample_aspect_ratio.num = sample_aspect_ratio.numerator();
572 av_frame->sample_aspect_ratio.den = sample_aspect_ratio.denominator();
573 av_frame->pts = video_pts_;
577 FF(av_image_fill_arrays(
581 static_cast<AVPixelFormat>(av_frame->format),
589 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(
590 print() + L" Unhandled media type " + boost::lexical_cast<std::wstring>(type)));
598 virtual std::wstring print() const = 0;
599 virtual void graph(spl::shared_ptr<caspar::diagnostics::graph> g) { }
600 virtual void acodec(std::string codec) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
601 virtual void vcodec(std::string codec) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
602 virtual void format(std::string fmt) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
603 virtual void framerate(boost::rational<int> framerate) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
604 virtual void start(bool has_audio, bool has_video) { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
605 virtual void stop() { }
606 virtual std::vector<AVSampleFormat> supported_sample_formats() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
607 virtual std::vector<int> supported_samplerates() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
608 virtual std::vector<AVPixelFormat> supported_pixel_formats() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
609 virtual int wanted_num_audio_streams() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
610 virtual boost::optional<int> wanted_num_channels_per_stream() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
611 virtual boost::optional<AVMediaType> try_push(AVMediaType type, int stream_index, spl::shared_ptr<AVFrame> frame) { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
612 virtual void eof() { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
615 struct no_sink_selected : public sink
617 std::wstring print() const override
619 return L"[no_sink_selected]";
623 class file_sink : public sink
625 std::wstring filename_;
626 spl::shared_ptr<diagnostics::graph> graph_;
628 file_sink(std::string filename)
629 : filename_(u16(std::move(filename)))
633 std::wstring print() const override
635 return L"[file_sink " + filename_ + L"]";
638 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
640 graph_ = std::move(g);
644 class memory_sink : public sink
646 spl::shared_ptr<core::frame_factory> factory_;
648 bool has_audio_ = false;
649 bool has_video_ = false;
650 std::vector<int> audio_cadence_;
651 core::audio_channel_layout channel_layout_ = core::audio_channel_layout::invalid();
652 core::mutable_audio_buffer audio_samples_;
654 std::queue<std::shared_ptr<AVFrame>> video_frames_;
656 tbb::concurrent_bounded_queue<core::draw_frame> output_frames_;
657 tbb::atomic<bool> running_;
659 memory_sink(spl::shared_ptr<core::frame_factory> factory, core::video_format_desc format)
660 : factory_(std::move(factory))
661 , audio_cadence_(format.audio_cadence)
663 output_frames_.set_capacity(2);
665 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
666 // This cadence fills the audio mixer most optimally.
667 boost::range::rotate(audio_cadence_, std::end(audio_cadence_) - 1);
675 std::wstring print() const override
677 return L"[memory_sink]";
680 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
684 void framerate(boost::rational<int> framerate) override
686 audio_cadence_ = find_audio_cadence(framerate);
687 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
688 // This cadence fills the audio mixer most optimally.
689 boost::range::rotate(audio_cadence_, std::end(audio_cadence_) - 1);
692 void start(bool has_audio, bool has_video) override
694 has_audio_ = has_audio;
695 has_video_ = has_video;
706 std::vector<AVSampleFormat> supported_sample_formats() const override
708 return { AVSampleFormat::AV_SAMPLE_FMT_S32 };
711 std::vector<int> supported_samplerates() const override {
715 std::vector<AVPixelFormat> supported_pixel_formats() const override
718 AVPixelFormat::AV_PIX_FMT_YUVA420P,
719 AVPixelFormat::AV_PIX_FMT_YUV444P,
720 AVPixelFormat::AV_PIX_FMT_YUV422P,
721 AVPixelFormat::AV_PIX_FMT_YUV420P,
722 AVPixelFormat::AV_PIX_FMT_YUV411P,
723 AVPixelFormat::AV_PIX_FMT_BGRA,
724 AVPixelFormat::AV_PIX_FMT_ARGB,
725 AVPixelFormat::AV_PIX_FMT_RGBA,
726 AVPixelFormat::AV_PIX_FMT_ABGR,
727 AVPixelFormat::AV_PIX_FMT_GRAY8
731 int wanted_num_audio_streams() const override
736 boost::optional<int> wanted_num_channels_per_stream() const
741 boost::optional<AVMediaType> try_push(AVMediaType type, int stream_index, spl::shared_ptr<AVFrame> av_frame) override
743 if (!has_audio_ && !has_video_)
744 CASPAR_THROW_EXCEPTION(invalid_operation());
746 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && av_frame->data[0])
748 if (channel_layout_ == core::audio_channel_layout::invalid()) // First audio
750 channel_layout_ = get_audio_channel_layout(av_frame->channels, av_frame->channel_layout, L"");
752 // Insert silence samples so that the audio mixer is guaranteed to be filled.
753 auto min_num_samples_per_frame = *boost::min_element(audio_cadence_);
754 auto max_num_samples_per_frame = *boost::max_element(audio_cadence_);
755 auto cadence_safety_samples = max_num_samples_per_frame - min_num_samples_per_frame;
756 audio_samples_.resize(channel_layout_.num_channels * cadence_safety_samples, 0);
759 auto ptr = reinterpret_cast<int32_t*>(av_frame->data[0]);
761 audio_samples_.insert(audio_samples_.end(), ptr, ptr + av_frame->linesize[0] / sizeof(int32_t));
763 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO)
765 video_frames_.push(std::move(av_frame));
772 (channel_layout_ != core::audio_channel_layout::invalid() && audio_samples_.size() >= audio_cadence_.front() * channel_layout_.num_channels);
775 !video_frames_.empty();
778 return AVMediaType::AVMEDIA_TYPE_AUDIO;
781 return AVMediaType::AVMEDIA_TYPE_VIDEO;
783 core::mutable_audio_buffer audio_data;
787 auto begin = audio_samples_.begin();
788 auto end = begin + audio_cadence_.front() * channel_layout_.num_channels;
790 audio_data.insert(audio_data.begin(), begin, end);
791 audio_samples_.erase(begin, end);
792 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_) + 1);
795 if (!has_video_) // Audio only
797 core::mutable_frame audio_only_frame(
799 std::move(audio_data),
801 core::pixel_format_desc(core::pixel_format::invalid),
804 output_frames_.push(core::draw_frame(std::move(audio_only_frame)));
806 return AVMediaType::AVMEDIA_TYPE_AUDIO;
809 auto output_frame = make_frame(this, spl::make_shared_ptr(video_frames_.front()), *factory_, channel_layout_);
811 output_frame.audio_data() = std::move(audio_data);
813 output_frames_.push(core::draw_frame(std::move(output_frame)));
819 // Drain rest, regardless of it being enough or not.
820 while (!video_frames_.empty() || !audio_samples_.empty())
822 core::mutable_audio_buffer audio_data;
824 audio_data.swap(audio_samples_);
826 if (!video_frames_.empty())
828 auto output_frame = make_frame(this, spl::make_shared_ptr(video_frames_.front()), *factory_, channel_layout_);
830 output_frame.audio_data() = std::move(audio_data);
832 output_frames_.push(core::draw_frame(std::move(output_frame)));
836 core::mutable_frame audio_only_frame(
838 std::move(audio_data),
840 core::pixel_format_desc(core::pixel_format::invalid),
843 output_frames_.push(core::draw_frame(std::move(audio_only_frame)));
844 output_frames_.push(core::draw_frame::empty());
849 core::draw_frame try_pop_frame()
851 core::draw_frame frame = core::draw_frame::late();
853 if (!output_frames_.try_pop(frame) && !running_)
854 return core::draw_frame::empty();
860 struct audio_stream_info
862 int num_channels = 0;
863 AVSampleFormat sampleformat = AVSampleFormat::AV_SAMPLE_FMT_NONE;
866 struct video_stream_info
870 AVPixelFormat pixelformat = AVPixelFormat::AV_PIX_FMT_NONE;
871 core::field_mode fieldmode = core::field_mode::progressive;
874 class ffmpeg_pipeline_backend_internal : public ffmpeg_pipeline_backend
876 spl::shared_ptr<diagnostics::graph> graph_;
878 spl::unique_ptr<source> source_ = spl::make_unique<no_source_selected>();
879 std::function<bool (caspar::array<const std::int32_t> data)> try_push_audio_;
880 std::function<bool (caspar::array<const std::uint8_t> data)> try_push_video_;
882 std::vector<audio_stream_info> source_audio_streams_;
883 video_stream_info source_video_stream_;
885 std::string afilter_;
886 std::unique_ptr<audio_filter> audio_filter_;
887 std::string vfilter_;
888 std::unique_ptr<filter> video_filter_;
890 spl::unique_ptr<sink> sink_ = spl::make_unique<no_sink_selected>();
891 std::function<core::draw_frame ()> try_pop_frame_;
893 tbb::atomic<bool> started_;
894 tbb::spin_mutex exception_mutex_;
895 boost::exception_ptr exception_;
896 boost::thread thread_;
898 ffmpeg_pipeline_backend_internal()
901 diagnostics::register_graph(graph_);
904 ~ffmpeg_pipeline_backend_internal()
909 void throw_if_error()
911 boost::lock_guard<tbb::spin_mutex> lock(exception_mutex_);
913 if (exception_ != nullptr)
914 boost::rethrow_exception(exception_);
917 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
919 graph_ = std::move(g);
920 source_->graph(graph_);
921 sink_->graph(graph_);
926 void from_file(std::string filename) override
928 source_ = spl::make_unique<file_source>(std::move(filename));
929 try_push_audio_ = std::function<bool (caspar::array<const std::int32_t>)>();
930 try_push_video_ = std::function<bool (caspar::array<const std::uint8_t>)>();
931 source_->graph(graph_);
934 void from_memory_only_audio(int num_channels, int samplerate) override
936 auto source = spl::make_unique<memory_source>();
937 auto source_ptr = source.get();
938 try_push_audio_ = [this, source_ptr](caspar::array<const std::int32_t> data) { return source_ptr->try_push_audio(std::move(data)); };
939 source->enable_audio(samplerate, num_channels);
941 source_ = std::move(source);
942 source_->graph(graph_);
945 void from_memory_only_video(int width, int height, boost::rational<int> framerate) override
947 auto source = spl::make_unique<memory_source>();
948 auto source_ptr = source.get();
949 try_push_video_ = [this, source_ptr](caspar::array<const std::uint8_t> data) { return source_ptr->try_push_video(std::move(data)); };
950 source->enable_video(width, height, std::move(framerate));
952 source_ = std::move(source);
953 source_->graph(graph_);
956 void from_memory(int num_channels, int samplerate, int width, int height, boost::rational<int> framerate) override
958 auto source = spl::make_unique<memory_source>();
959 auto source_ptr = source.get();
960 try_push_audio_ = [this, source_ptr](caspar::array<const std::int32_t> data) { return source_ptr->try_push_audio(std::move(data)); };
961 try_push_video_ = [this, source_ptr](caspar::array<const std::uint8_t> data) { return source_ptr->try_push_video(std::move(data)); };
962 source->enable_audio(samplerate, num_channels);
963 source->enable_video(width, height, std::move(framerate));
965 source_ = std::move(source);
966 source_->graph(graph_);
969 void start_frame(std::uint32_t frame) override { source_->start_frame(frame); }
970 std::uint32_t start_frame() const override { return source_->start_frame(); }
971 void length(std::uint32_t frames) override { source_->length(frames); }
972 std::uint32_t length() const override { return source_->length(); }
973 void seek(std::uint32_t frame) override { source_->seek(frame); }
974 void loop(bool value) override { source_->loop(value); }
975 bool loop() const override { return source_->loop(); }
976 std::string source_filename() const override { return source_->filename(); }
980 void vfilter(std::string filter) override
982 vfilter_ = std::move(filter);
985 void afilter(std::string filter) override
987 afilter_ = std::move(filter);
990 int width() const override
992 return source_video_stream_.width;
995 int height() const override
997 return source_video_stream_.height;
1000 boost::rational<int> framerate() const override
1002 bool double_rate = filter::is_double_rate(u16(vfilter_));
1004 return double_rate ? source_->framerate() * 2 : source_->framerate();
1007 bool progressive() const override
1014 void to_memory(spl::shared_ptr<core::frame_factory> factory, core::video_format_desc format) override
1016 auto sink = spl::make_unique<memory_sink>(std::move(factory), std::move(format));
1017 auto sink_ptr = sink.get();
1018 try_pop_frame_ = [sink_ptr] { return sink_ptr->try_pop_frame(); };
1020 sink_ = std::move(sink);
1021 sink_->graph(graph_);
1024 void to_file(std::string filename) override
1026 sink_ = spl::make_unique<file_sink>(std::move(filename));
1027 try_pop_frame_ = std::function<core::draw_frame ()>();
1028 sink_->graph(graph_);
1031 void acodec(std::string codec) override { sink_->acodec(std::move(codec)); }
1032 void vcodec(std::string codec) override { sink_->vcodec(std::move(codec)); }
1033 void format(std::string fmt) override { sink_->format(std::move(fmt)); }
1037 void start() override
1040 sink_->start(source_->has_audio(), source_->has_video());
1042 bool quiet = is_logging_quiet_for_thread();
1044 thread_ = boost::thread([=] { run(quiet); });
1047 bool try_push_audio(caspar::array<const std::int32_t> data) override
1051 if (try_push_audio_)
1052 return try_push_audio_(std::move(data));
1057 bool try_push_video(caspar::array<const std::uint8_t> data) override
1061 if (try_push_video_)
1062 return try_push_video_(std::move(data));
1067 core::draw_frame try_pop_frame() override
1071 if (!try_pop_frame_)
1072 CASPAR_THROW_EXCEPTION(invalid_operation());
1074 return try_pop_frame_();
1077 std::uint32_t last_frame() const override
1079 return source_->frame_number();
1082 bool started() const override
1087 void stop() override
1094 if (thread_.joinable())
1099 void run(bool quiet)
1101 ensure_gpf_handler_installed_for_thread(u8(L"ffmpeg-pipeline: " + source_->print() + L" -> " + sink_->print()).c_str());
1102 auto quiet_logging = temporary_enable_quiet_logging_for_thread(quiet);
1106 boost::optional<AVMediaType> result = source_->has_audio() ? AVMediaType::AVMEDIA_TYPE_AUDIO : AVMediaType::AVMEDIA_TYPE_VIDEO;
1108 while (started_ && (source_->has_audio() || source_->has_video()))
1110 auto needed = *result;
1111 auto input_frames_for_streams = source_->get_input_frames_for_streams(needed);
1113 if (!input_frames_for_streams.empty() && input_frames_for_streams.at(0))
1115 for (int input_stream_index = 0; input_stream_index < input_frames_for_streams.size(); ++input_stream_index)
1117 if (needed == AVMediaType::AVMEDIA_TYPE_AUDIO)
1119 initialize_audio_filter_if_needed(input_frames_for_streams);
1120 audio_filter_->push(input_stream_index, std::move(input_frames_for_streams.at(input_stream_index)));
1122 for (int output_stream_index = 0; output_stream_index < sink_->wanted_num_audio_streams(); ++output_stream_index)
1123 for (auto filtered_frame : audio_filter_->poll_all(output_stream_index))
1124 result = sink_->try_push(AVMediaType::AVMEDIA_TYPE_AUDIO, output_stream_index, std::move(filtered_frame));
1126 else if (needed == AVMediaType::AVMEDIA_TYPE_VIDEO)
1128 initialize_video_filter_if_needed(*input_frames_for_streams.at(input_stream_index));
1129 video_filter_->push(std::move(input_frames_for_streams.at(input_stream_index)));
1131 for (auto filtered_frame : video_filter_->poll_all())
1132 result = sink_->try_push(AVMediaType::AVMEDIA_TYPE_VIDEO, 0, std::move(filtered_frame));
1135 CASPAR_THROW_EXCEPTION(not_supported());
1138 else if (source_->eof())
1145 result = boost::none;
1149 graph_->set_tag(caspar::diagnostics::tag_severity::WARNING, "dropped-frame");
1150 result = needed; // Repeat same media type
1156 if (is_logging_quiet_for_thread())
1158 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
1162 CASPAR_LOG_CURRENT_EXCEPTION();
1165 boost::lock_guard<tbb::spin_mutex> lock(exception_mutex_);
1166 exception_ = boost::current_exception();
1169 video_filter_.reset();
1170 audio_filter_.reset();
1176 template<typename T>
1177 void set_if_changed(bool& changed, T& old_value, T new_value)
1179 if (old_value != new_value)
1182 old_value = new_value;
1186 void initialize_audio_filter_if_needed(const std::vector<std::shared_ptr<AVFrame>>& av_frames_per_stream)
1188 bool changed = av_frames_per_stream.size() != source_audio_streams_.size();
1189 source_audio_streams_.resize(av_frames_per_stream.size());
1191 for (int i = 0; i < av_frames_per_stream.size(); ++i)
1193 auto& av_frame = *av_frames_per_stream.at(i);
1194 auto& stream = source_audio_streams_.at(i);
1196 set_if_changed(changed, stream.sampleformat, static_cast<AVSampleFormat>(av_frame.format));
1197 set_if_changed(changed, stream.num_channels, av_frame.channels);
1201 initialize_audio_filter();
1204 void initialize_audio_filter()
1206 std::vector<audio_input_pad> input_pads;
1207 std::vector<audio_output_pad> output_pads;
1209 for (auto& source_audio_stream : source_audio_streams_)
1211 input_pads.emplace_back(
1212 boost::rational<int>(1, source_->samplerate()),
1213 source_->samplerate(),
1214 source_audio_stream.sampleformat,
1215 av_get_default_channel_layout(source_audio_stream.num_channels));
1218 auto total_num_channels = cpplinq::from(source_audio_streams_)
1219 .select([](const audio_stream_info& info) { return info.num_channels; })
1220 .aggregate(0, std::plus<int>());
1222 if (total_num_channels > 1 && sink_->wanted_num_audio_streams() > 1)
1223 CASPAR_THROW_EXCEPTION(invalid_operation()
1224 << msg_info("only one-to-many or many-to-one audio stream conversion supported."));
1226 std::wstring amerge;
1228 if (sink_->wanted_num_audio_streams() == 1 && !sink_->wanted_num_channels_per_stream())
1230 output_pads.emplace_back(
1231 sink_->supported_samplerates(),
1232 sink_->supported_sample_formats(),
1233 std::vector<int64_t>({ av_get_default_channel_layout(total_num_channels) }));
1235 if (source_audio_streams_.size() > 1)
1237 for (int i = 0; i < source_audio_streams_.size(); ++i)
1238 amerge += L"[a:" + boost::lexical_cast<std::wstring>(i) + L"]";
1240 amerge += L"amerge=inputs=" + boost::lexical_cast<std::wstring>(source_audio_streams_.size());
1244 std::wstring afilter = u16(afilter_);
1246 if (!amerge.empty())
1248 afilter = prepend_filter(u16(afilter), amerge);
1249 afilter += L"[aout:0]";
1252 audio_filter_.reset(new audio_filter(input_pads, output_pads, u8(afilter)));
1255 void initialize_video_filter_if_needed(const AVFrame& av_frame)
1257 bool changed = false;
1259 set_if_changed(changed, source_video_stream_.width, av_frame.width);
1260 set_if_changed(changed, source_video_stream_.height, av_frame.height);
1261 set_if_changed(changed, source_video_stream_.pixelformat, static_cast<AVPixelFormat>(av_frame.format));
1263 core::field_mode field_mode = core::field_mode::progressive;
1265 if (av_frame.interlaced_frame)
1266 field_mode = av_frame.top_field_first ? core::field_mode::upper : core::field_mode::lower;
1268 set_if_changed(changed, source_video_stream_.fieldmode, field_mode);
1271 initialize_video_filter();
1274 void initialize_video_filter()
1276 if (source_video_stream_.fieldmode != core::field_mode::progressive && !filter::is_deinterlacing(u16(vfilter_)))
1277 vfilter_ = u8(append_filter(u16(vfilter_), L"YADIF=1:-1"));
1279 if (source_video_stream_.height == 480) // NTSC DV
1281 auto pad_str = L"PAD=" + boost::lexical_cast<std::wstring>(source_video_stream_.width) + L":486:0:2:black";
1282 vfilter_ = u8(append_filter(u16(vfilter_), pad_str));
1285 video_filter_.reset(new filter(
1286 source_video_stream_.width,
1287 source_video_stream_.height,
1288 1 / source_->framerate(),
1289 source_->framerate(),
1290 boost::rational<int>(1, 1), // TODO
1291 source_video_stream_.pixelformat,
1292 sink_->supported_pixel_formats(),
1294 sink_->framerate(framerate());
1298 spl::shared_ptr<struct ffmpeg_pipeline_backend> create_internal_pipeline()
1300 return spl::make_shared<ffmpeg_pipeline_backend_internal>();