2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Helge Norberg, helge.norberg@svt.se
20 * Author: Robert Nagy, ronag89@gmail.com
25 #include "ffmpeg_pipeline_backend.h"
26 #include "ffmpeg_pipeline_backend_internal.h"
27 #include "producer/input/input.h"
28 #include "producer/video/video_decoder.h"
29 #include "producer/audio/audio_decoder.h"
30 #include "producer/filter/audio_filter.h"
31 #include "producer/filter/filter.h"
32 #include "producer/util/util.h"
33 #include "ffmpeg_error.h"
36 #include <common/diagnostics/graph.h>
37 #include <common/os/general_protection_fault.h>
38 #include <common/enum_class.h>
40 #include <core/frame/audio_channel_layout.h>
41 #include <core/frame/frame.h>
42 #include <core/frame/frame_factory.h>
43 #include <core/video_format.h>
50 #include <tbb/atomic.h>
51 #include <tbb/concurrent_queue.h>
52 #include <tbb/spin_mutex.h>
54 #include <boost/thread.hpp>
55 #include <boost/optional.hpp>
56 #include <boost/exception_ptr.hpp>
58 namespace caspar { namespace ffmpeg {
60 std::string to_string(const boost::rational<int>& framerate)
62 return boost::lexical_cast<std::string>(framerate.numerator())
63 + "/" + boost::lexical_cast<std::string>(framerate.denominator()) + " (" + boost::lexical_cast<std::string>(static_cast<double>(framerate.numerator()) / static_cast<double>(framerate.denominator())) + ") fps";
66 std::vector<int> find_audio_cadence(const boost::rational<int>& framerate)
68 static std::map<boost::rational<int>, std::vector<int>> CADENCES_BY_FRAMERATE = []
70 std::map<boost::rational<int>, std::vector<int>> result;
72 for (core::video_format format : enum_constants<core::video_format>())
74 core::video_format_desc desc(format);
75 boost::rational<int> format_rate(desc.time_scale, desc.duration);
77 result.insert(std::make_pair(format_rate, desc.audio_cadence));
83 auto exact_match = CADENCES_BY_FRAMERATE.find(framerate);
85 if (exact_match != CADENCES_BY_FRAMERATE.end())
86 return exact_match->second;
88 boost::rational<int> closest_framerate_diff = std::numeric_limits<int>::max();
89 boost::rational<int> closest_framerate = 0;
91 for (auto format_framerate : CADENCES_BY_FRAMERATE | boost::adaptors::map_keys)
93 auto diff = boost::abs(framerate - format_framerate);
95 if (diff < closest_framerate_diff)
97 closest_framerate_diff = diff;
98 closest_framerate = format_framerate;
102 if (is_logging_quiet_for_thread())
103 CASPAR_LOG(debug) << "No exact audio cadence match found for framerate " << to_string(framerate)
104 << "\nClosest match is " << to_string(closest_framerate)
105 << "\nwhich is a " << to_string(closest_framerate_diff) << " difference.";
107 CASPAR_LOG(warning) << "No exact audio cadence match found for framerate " << to_string(framerate)
108 << "\nClosest match is " << to_string(closest_framerate)
109 << "\nwhich is a " << to_string(closest_framerate_diff) << " difference.";
111 return CADENCES_BY_FRAMERATE[closest_framerate];
116 virtual ~source() { }
118 virtual std::wstring print() const = 0;
119 virtual void start() { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
120 virtual void graph(spl::shared_ptr<caspar::diagnostics::graph> g) { }
121 virtual void stop() { }
122 virtual void start_frame(std::uint32_t frame) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
123 virtual std::uint32_t start_frame() const { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
124 virtual void loop(bool value) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
125 virtual bool loop() const { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
126 virtual void length(std::uint32_t frames) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
127 virtual std::uint32_t length() const { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
128 virtual std::string filename() const { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print())); }
129 virtual void seek(std::uint32_t frame) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
130 virtual bool has_audio() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
131 virtual int samplerate() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
132 virtual bool has_video() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
133 virtual bool eof() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
134 virtual boost::rational<int> framerate() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
135 virtual std::uint32_t frame_number() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
136 virtual std::vector<std::shared_ptr<AVFrame>> get_input_frames_for_streams(AVMediaType type) { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
139 struct no_source_selected : public source
141 std::wstring print() const override
143 return L"[no_source_selected]";
147 class file_source : public source
149 std::wstring filename_;
150 spl::shared_ptr<diagnostics::graph> graph_;
151 std::uint32_t start_frame_ = 0;
152 std::uint32_t length_ = std::numeric_limits<std::uint32_t>::max();
154 mutable boost::mutex pointer_mutex_;
155 std::shared_ptr<input> input_;
156 std::vector<spl::shared_ptr<audio_decoder>> audio_decoders_;
157 std::shared_ptr<video_decoder> video_decoder_;
158 bool started_ = false;
160 file_source(std::string filename)
161 : filename_(u16(filename))
165 std::wstring print() const override
167 return L"[file_source " + filename_ + L"]";
170 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
172 graph_ = std::move(g);
175 void start() override
177 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
178 bool thumbnail_mode = is_logging_quiet_for_thread();
179 input_.reset(new input(graph_, filename_, loop_, start_frame_, length_, thumbnail_mode));
181 for (int i = 0; i < input_->num_audio_streams(); ++i)
185 audio_decoders_.push_back(spl::make_shared<audio_decoder>(*input_, core::video_format::invalid, i));
189 if (is_logging_quiet_for_thread())
191 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
192 CASPAR_LOG(info) << print() << " Failed to open audio-stream. Turn on log level debug to see more information.";
196 CASPAR_LOG_CURRENT_EXCEPTION();
197 CASPAR_LOG(warning) << print() << " Failed to open audio-stream.";
202 if (audio_decoders_.empty())
203 CASPAR_LOG(debug) << print() << " No audio-stream found. Running without audio.";
207 video_decoder_.reset(new video_decoder(*input_, false));
209 catch (averror_stream_not_found&)
211 CASPAR_LOG(debug) << print() << " No video-stream found. Running without video.";
215 if (is_logging_quiet_for_thread())
217 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
218 CASPAR_LOG(info) << print() << " Failed to open video-stream. Running without audio. Turn on log level debug to see more information.";
222 CASPAR_LOG_CURRENT_EXCEPTION();
223 CASPAR_LOG(warning) << print() << " Failed to open video-stream. Running without audio.";
235 void start_frame(std::uint32_t frame) override
237 start_frame_ = frame;
239 auto i = get_input();
244 std::uint32_t start_frame() const override
249 void loop(bool value) override
253 auto i = get_input();
258 bool loop() const override
263 void length(std::uint32_t frames) override
267 auto i = get_input();
272 std::uint32_t length() const override
274 auto v = get_video_decoder();
277 return v->nb_frames();
279 auto a = get_audio_decoders();
282 return a.at(0)->nb_frames(); // Should be ok.
287 std::string filename() const override
289 return u8(filename_);
292 void seek(std::uint32_t frame) override
295 get_input()->seek(frame);
298 bool eof() const override
300 auto i = get_input();
301 return !i || i->eof();
304 bool has_audio() const override
306 return !get_audio_decoders().empty();
309 int samplerate() const override
311 if (get_audio_decoders().empty())
317 bool has_video() const override
319 return static_cast<bool>(get_video_decoder());
322 boost::rational<int> framerate() const override
324 auto decoder = get_video_decoder();
329 return decoder->framerate();
332 std::uint32_t frame_number() const override
334 auto decoder = get_video_decoder();
339 return decoder->file_frame_number();
342 std::vector<std::shared_ptr<AVFrame>> get_input_frames_for_streams(AVMediaType type) override
344 auto a_decoders = get_audio_decoders();
345 auto v_decoder = get_video_decoder();
348 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && !a_decoders.empty())
350 std::vector<std::shared_ptr<AVFrame>> frames;
352 for (auto& a_decoder : a_decoders)
354 std::shared_ptr<AVFrame> frame;
356 for (int i = 0; i < 64; ++i)
358 frame = (*a_decoder)();
360 if (frame && frame->data[0])
366 frames.push_back(std::move(frame));
371 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO && v_decoder)
373 std::shared_ptr<AVFrame> frame;
375 for (int i = 0; i < 128; ++i)
377 frame = (*v_decoder)();
379 if (frame && frame->data[0])
384 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(
385 print() + L" Unhandled media type " + boost::lexical_cast<std::wstring>(type)));
390 void expect_started() const
393 CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" Not started."));
396 std::shared_ptr<input> get_input() const
398 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
402 std::vector<spl::shared_ptr<audio_decoder>> get_audio_decoders() const
404 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
405 return audio_decoders_;
408 std::shared_ptr<video_decoder> get_video_decoder() const
410 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
411 return video_decoder_;
415 class memory_source : public source
417 int samplerate_ = -1;
418 int num_channels_ = -1;
421 boost::rational<int> framerate_ = -1;
423 tbb::atomic<bool> running_;
424 tbb::concurrent_bounded_queue<caspar::array<const int32_t>> audio_frames_;
425 tbb::concurrent_bounded_queue<caspar::array<const uint8_t>> video_frames_;
426 int64_t audio_pts_ = 0;
427 int64_t video_pts_ = 0;
432 video_frames_.set_capacity(1);
433 audio_frames_.set_capacity(1);
441 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
445 std::wstring print() const override
447 return L"[memory_source]";
450 void enable_audio(int samplerate, int num_channels)
452 samplerate_ = samplerate;
453 num_channels_ = num_channels;
456 void enable_video(int width, int height, boost::rational<int> framerate)
462 void start() override
470 video_frames_.try_push(caspar::array<const uint8_t>());
471 audio_frames_.try_push(caspar::array<const int32_t>());
474 bool has_audio() const override
476 return samplerate_ != -1;
479 int samplerate() const override
484 bool has_video() const override
489 bool eof() const override
494 boost::rational<int> framerate() const override
499 bool try_push_audio(caspar::array<const std::int32_t> data)
502 CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" audio not enabled."));
504 if (data.empty() || data.size() % num_channels_ != 0)
505 CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info(print() + L" audio with incorrect number of channels submitted."));
507 return audio_frames_.try_push(std::move(data));
510 bool try_push_video(caspar::array<const std::uint8_t> data)
513 CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" video not enabled."));
515 if (data.size() != width_ * height_ * 4)
516 CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info(print() + L" video with incorrect size submitted."));
518 return video_frames_.try_push(std::move(data));
521 std::vector<std::shared_ptr<AVFrame>> get_input_frames_for_streams(AVMediaType type) override
524 CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not running."));
526 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && has_audio())
528 caspar::array<const std::int32_t> samples;
529 audio_frames_.pop(samples);
534 spl::shared_ptr<AVFrame> av_frame(av_frame_alloc(), [samples](AVFrame* p) { av_frame_free(&p); });
536 av_frame->channels = num_channels_;
537 av_frame->channel_layout = av_get_default_channel_layout(num_channels_);
538 av_frame->sample_rate = samplerate_;
539 av_frame->nb_samples = static_cast<int>(samples.size()) / num_channels_;
540 av_frame->format = AV_SAMPLE_FMT_S32;
541 av_frame->pts = audio_pts_;
543 audio_pts_ += av_frame->nb_samples;
545 FF(av_samples_fill_arrays(
546 av_frame->extended_data,
548 reinterpret_cast<const std::uint8_t*>(&*samples.begin()),
550 av_frame->nb_samples,
551 static_cast<AVSampleFormat>(av_frame->format),
556 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO && has_video())
558 caspar::array<const std::uint8_t> data;
559 video_frames_.pop(data);
564 spl::shared_ptr<AVFrame> av_frame(av_frame_alloc(), [data](AVFrame* p) { av_frame_free(&p); });
565 avcodec_get_frame_defaults(av_frame.get());
567 const auto sample_aspect_ratio = boost::rational<int>(width_, height_);
569 av_frame->format = AV_PIX_FMT_BGRA;
570 av_frame->width = width_;
571 av_frame->height = height_;
572 av_frame->sample_aspect_ratio.num = sample_aspect_ratio.numerator();
573 av_frame->sample_aspect_ratio.den = sample_aspect_ratio.denominator();
574 av_frame->pts = video_pts_;
578 FF(av_image_fill_arrays(
582 static_cast<AVPixelFormat>(av_frame->format),
590 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(
591 print() + L" Unhandled media type " + boost::lexical_cast<std::wstring>(type)));
599 virtual std::wstring print() const = 0;
600 virtual void graph(spl::shared_ptr<caspar::diagnostics::graph> g) { }
601 virtual void acodec(std::string codec) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
602 virtual void vcodec(std::string codec) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
603 virtual void format(std::string fmt) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
604 virtual void framerate(boost::rational<int> framerate) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
605 virtual void start(bool has_audio, bool has_video) { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
606 virtual void stop() { }
607 virtual std::vector<AVSampleFormat> supported_sample_formats() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
608 virtual std::vector<int> supported_samplerates() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
609 virtual std::vector<AVPixelFormat> supported_pixel_formats() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
610 virtual int wanted_num_audio_streams() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
611 virtual boost::optional<int> wanted_num_channels_per_stream() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
612 virtual boost::optional<AVMediaType> try_push(AVMediaType type, int stream_index, spl::shared_ptr<AVFrame> frame) { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
613 virtual void eof() { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
616 struct no_sink_selected : public sink
618 std::wstring print() const override
620 return L"[no_sink_selected]";
624 class file_sink : public sink
626 std::wstring filename_;
627 spl::shared_ptr<diagnostics::graph> graph_;
629 file_sink(std::string filename)
630 : filename_(u16(std::move(filename)))
634 std::wstring print() const override
636 return L"[file_sink " + filename_ + L"]";
639 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
641 graph_ = std::move(g);
645 class memory_sink : public sink
647 spl::shared_ptr<core::frame_factory> factory_;
649 bool has_audio_ = false;
650 bool has_video_ = false;
651 std::vector<int> audio_cadence_;
652 core::audio_channel_layout channel_layout_ = core::audio_channel_layout::invalid();
653 core::mutable_audio_buffer audio_samples_;
655 std::queue<std::shared_ptr<AVFrame>> video_frames_;
657 tbb::concurrent_bounded_queue<core::draw_frame> output_frames_;
658 tbb::atomic<bool> running_;
660 memory_sink(spl::shared_ptr<core::frame_factory> factory, core::video_format_desc format)
661 : factory_(std::move(factory))
662 , audio_cadence_(format.audio_cadence)
664 output_frames_.set_capacity(2);
666 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
667 // This cadence fills the audio mixer most optimally.
668 boost::range::rotate(audio_cadence_, std::end(audio_cadence_) - 1);
676 std::wstring print() const override
678 return L"[memory_sink]";
681 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
685 void framerate(boost::rational<int> framerate) override
687 audio_cadence_ = find_audio_cadence(framerate);
688 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
689 // This cadence fills the audio mixer most optimally.
690 boost::range::rotate(audio_cadence_, std::end(audio_cadence_) - 1);
693 void start(bool has_audio, bool has_video) override
695 has_audio_ = has_audio;
696 has_video_ = has_video;
703 output_frames_.set_capacity(4);
706 std::vector<AVSampleFormat> supported_sample_formats() const override
708 return { AVSampleFormat::AV_SAMPLE_FMT_S32 };
711 std::vector<int> supported_samplerates() const override {
715 std::vector<AVPixelFormat> supported_pixel_formats() const override
718 AVPixelFormat::AV_PIX_FMT_YUVA420P,
719 AVPixelFormat::AV_PIX_FMT_YUV444P,
720 AVPixelFormat::AV_PIX_FMT_YUV422P,
721 AVPixelFormat::AV_PIX_FMT_YUV420P,
722 AVPixelFormat::AV_PIX_FMT_YUV411P,
723 AVPixelFormat::AV_PIX_FMT_BGRA,
724 AVPixelFormat::AV_PIX_FMT_ARGB,
725 AVPixelFormat::AV_PIX_FMT_RGBA,
726 AVPixelFormat::AV_PIX_FMT_ABGR,
727 AVPixelFormat::AV_PIX_FMT_GRAY8
731 int wanted_num_audio_streams() const override
736 boost::optional<int> wanted_num_channels_per_stream() const
741 boost::optional<AVMediaType> try_push(AVMediaType type, int stream_index, spl::shared_ptr<AVFrame> av_frame) override
743 if (!has_audio_ && !has_video_)
744 CASPAR_THROW_EXCEPTION(invalid_operation());
746 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && av_frame->data[0])
748 if (channel_layout_ == core::audio_channel_layout::invalid()) // First audio
750 channel_layout_ = get_audio_channel_layout(av_frame->channels, av_frame->channel_layout, L"");
752 // Insert silence samples so that the audio mixer is guaranteed to be filled.
753 auto min_num_samples_per_frame = *boost::min_element(audio_cadence_);
754 auto max_num_samples_per_frame = *boost::max_element(audio_cadence_);
755 auto cadence_safety_samples = max_num_samples_per_frame - min_num_samples_per_frame;
756 audio_samples_.resize(channel_layout_.num_channels * cadence_safety_samples, 0);
759 auto ptr = reinterpret_cast<int32_t*>(av_frame->data[0]);
761 audio_samples_.insert(audio_samples_.end(), ptr, ptr + av_frame->linesize[0] / sizeof(int32_t));
763 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO)
765 video_frames_.push(std::move(av_frame));
772 (channel_layout_ != core::audio_channel_layout::invalid() && audio_samples_.size() >= audio_cadence_.front() * channel_layout_.num_channels);
775 !video_frames_.empty();
778 return AVMediaType::AVMEDIA_TYPE_AUDIO;
781 return AVMediaType::AVMEDIA_TYPE_VIDEO;
783 core::mutable_audio_buffer audio_data;
787 auto begin = audio_samples_.begin();
788 auto end = begin + audio_cadence_.front() * channel_layout_.num_channels;
790 audio_data.insert(audio_data.begin(), begin, end);
791 audio_samples_.erase(begin, end);
792 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_) + 1);
795 if (!has_video_) // Audio only
797 core::mutable_frame audio_only_frame(
799 std::move(audio_data),
801 core::pixel_format_desc(core::pixel_format::invalid),
804 output_frames_.push(core::draw_frame(std::move(audio_only_frame)));
806 return AVMediaType::AVMEDIA_TYPE_AUDIO;
809 auto output_frame = make_frame(this, spl::make_shared_ptr(video_frames_.front()), *factory_, channel_layout_);
811 output_frame.audio_data() = std::move(audio_data);
813 output_frames_.push(core::draw_frame(std::move(output_frame)));
819 // Drain rest, regardless of it being enough or not.
820 while (!video_frames_.empty() || !audio_samples_.empty())
822 core::mutable_audio_buffer audio_data;
824 audio_data.swap(audio_samples_);
826 if (!video_frames_.empty())
828 auto output_frame = make_frame(this, spl::make_shared_ptr(video_frames_.front()), *factory_, channel_layout_);
830 output_frame.audio_data() = std::move(audio_data);
832 output_frames_.push(core::draw_frame(std::move(output_frame)));
836 core::mutable_frame audio_only_frame(
838 std::move(audio_data),
840 core::pixel_format_desc(core::pixel_format::invalid),
843 output_frames_.push(core::draw_frame(std::move(audio_only_frame)));
844 output_frames_.push(core::draw_frame::empty());
849 core::draw_frame try_pop_frame()
851 core::draw_frame frame = core::draw_frame::late();
853 if (!output_frames_.try_pop(frame) && !running_)
854 return core::draw_frame::empty();
860 struct audio_stream_info
862 int num_channels = 0;
863 AVSampleFormat sampleformat = AVSampleFormat::AV_SAMPLE_FMT_NONE;
864 uint64_t channel_layout = 0;
867 struct video_stream_info
871 AVPixelFormat pixelformat = AVPixelFormat::AV_PIX_FMT_NONE;
872 core::field_mode fieldmode = core::field_mode::progressive;
875 class ffmpeg_pipeline_backend_internal : public ffmpeg_pipeline_backend
877 spl::shared_ptr<diagnostics::graph> graph_;
879 spl::unique_ptr<source> source_ = spl::make_unique<no_source_selected>();
880 std::function<bool (caspar::array<const std::int32_t> data)> try_push_audio_;
881 std::function<bool (caspar::array<const std::uint8_t> data)> try_push_video_;
883 std::vector<audio_stream_info> source_audio_streams_;
884 video_stream_info source_video_stream_;
886 std::string afilter_;
887 std::unique_ptr<audio_filter> audio_filter_;
888 std::string vfilter_;
889 std::unique_ptr<filter> video_filter_;
891 spl::unique_ptr<sink> sink_ = spl::make_unique<no_sink_selected>();
892 std::function<core::draw_frame ()> try_pop_frame_;
894 tbb::atomic<bool> started_;
895 tbb::spin_mutex exception_mutex_;
896 boost::exception_ptr exception_;
897 boost::thread thread_;
899 ffmpeg_pipeline_backend_internal()
902 diagnostics::register_graph(graph_);
905 ~ffmpeg_pipeline_backend_internal()
910 void throw_if_error()
912 boost::lock_guard<tbb::spin_mutex> lock(exception_mutex_);
914 if (exception_ != nullptr)
915 boost::rethrow_exception(exception_);
918 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
920 graph_ = std::move(g);
921 source_->graph(graph_);
922 sink_->graph(graph_);
927 void from_file(std::string filename) override
929 source_ = spl::make_unique<file_source>(std::move(filename));
930 try_push_audio_ = std::function<bool (caspar::array<const std::int32_t>)>();
931 try_push_video_ = std::function<bool (caspar::array<const std::uint8_t>)>();
932 source_->graph(graph_);
935 void from_memory_only_audio(int num_channels, int samplerate) override
937 auto source = spl::make_unique<memory_source>();
938 auto source_ptr = source.get();
939 try_push_audio_ = [this, source_ptr](caspar::array<const std::int32_t> data) { return source_ptr->try_push_audio(std::move(data)); };
940 source->enable_audio(samplerate, num_channels);
942 source_ = std::move(source);
943 source_->graph(graph_);
946 void from_memory_only_video(int width, int height, boost::rational<int> framerate) override
948 auto source = spl::make_unique<memory_source>();
949 auto source_ptr = source.get();
950 try_push_video_ = [this, source_ptr](caspar::array<const std::uint8_t> data) { return source_ptr->try_push_video(std::move(data)); };
951 source->enable_video(width, height, std::move(framerate));
953 source_ = std::move(source);
954 source_->graph(graph_);
957 void from_memory(int num_channels, int samplerate, int width, int height, boost::rational<int> framerate) override
959 auto source = spl::make_unique<memory_source>();
960 auto source_ptr = source.get();
961 try_push_audio_ = [this, source_ptr](caspar::array<const std::int32_t> data) { return source_ptr->try_push_audio(std::move(data)); };
962 try_push_video_ = [this, source_ptr](caspar::array<const std::uint8_t> data) { return source_ptr->try_push_video(std::move(data)); };
963 source->enable_audio(samplerate, num_channels);
964 source->enable_video(width, height, std::move(framerate));
966 source_ = std::move(source);
967 source_->graph(graph_);
970 void start_frame(std::uint32_t frame) override { source_->start_frame(frame); }
971 std::uint32_t start_frame() const override { return source_->start_frame(); }
972 void length(std::uint32_t frames) override { source_->length(frames); }
973 std::uint32_t length() const override { return source_->length(); }
974 void seek(std::uint32_t frame) override { source_->seek(frame); }
975 void loop(bool value) override { source_->loop(value); }
976 bool loop() const override { return source_->loop(); }
977 std::string source_filename() const override { return source_->filename(); }
981 void vfilter(std::string filter) override
983 vfilter_ = std::move(filter);
986 void afilter(std::string filter) override
988 afilter_ = std::move(filter);
991 int width() const override
993 return source_video_stream_.width;
996 int height() const override
998 return source_video_stream_.height;
1001 boost::rational<int> framerate() const override
1003 bool double_rate = filter::is_double_rate(u16(vfilter_));
1005 return double_rate ? source_->framerate() * 2 : source_->framerate();
1008 bool progressive() const override
1015 void to_memory(spl::shared_ptr<core::frame_factory> factory, core::video_format_desc format) override
1017 auto sink = spl::make_unique<memory_sink>(std::move(factory), std::move(format));
1018 auto sink_ptr = sink.get();
1019 try_pop_frame_ = [sink_ptr] { return sink_ptr->try_pop_frame(); };
1021 sink_ = std::move(sink);
1022 sink_->graph(graph_);
1025 void to_file(std::string filename) override
1027 sink_ = spl::make_unique<file_sink>(std::move(filename));
1028 try_pop_frame_ = std::function<core::draw_frame ()>();
1029 sink_->graph(graph_);
1032 void acodec(std::string codec) override { sink_->acodec(std::move(codec)); }
1033 void vcodec(std::string codec) override { sink_->vcodec(std::move(codec)); }
1034 void format(std::string fmt) override { sink_->format(std::move(fmt)); }
1038 void start() override
1041 sink_->start(source_->has_audio(), source_->has_video());
1043 bool quiet = is_logging_quiet_for_thread();
1045 thread_ = boost::thread([=] { run(quiet); });
1048 bool try_push_audio(caspar::array<const std::int32_t> data) override
1052 if (try_push_audio_)
1053 return try_push_audio_(std::move(data));
1058 bool try_push_video(caspar::array<const std::uint8_t> data) override
1062 if (try_push_video_)
1063 return try_push_video_(std::move(data));
1068 core::draw_frame try_pop_frame() override
1072 if (!try_pop_frame_)
1073 CASPAR_THROW_EXCEPTION(invalid_operation());
1075 return try_pop_frame_();
1078 std::uint32_t last_frame() const override
1080 return source_->frame_number();
1083 bool started() const override
1088 void stop() override
1095 if (thread_.joinable())
1100 void run(bool quiet)
1102 ensure_gpf_handler_installed_for_thread(u8(L"ffmpeg-pipeline: " + source_->print() + L" -> " + sink_->print()).c_str());
1103 auto quiet_logging = temporary_enable_quiet_logging_for_thread(quiet);
1107 boost::optional<AVMediaType> result = source_->has_audio() ? AVMediaType::AVMEDIA_TYPE_AUDIO : AVMediaType::AVMEDIA_TYPE_VIDEO;
1109 while (started_ && (source_->has_audio() || source_->has_video()))
1111 auto needed = *result;
1112 auto input_frames_for_streams = source_->get_input_frames_for_streams(needed);
1114 if (!input_frames_for_streams.empty() && input_frames_for_streams.at(0))
1116 for (int input_stream_index = 0; input_stream_index < input_frames_for_streams.size(); ++input_stream_index)
1118 if (needed == AVMediaType::AVMEDIA_TYPE_AUDIO)
1120 initialize_audio_filter_if_needed(input_frames_for_streams);
1121 audio_filter_->push(input_stream_index, std::move(input_frames_for_streams.at(input_stream_index)));
1123 for (int output_stream_index = 0; output_stream_index < sink_->wanted_num_audio_streams(); ++output_stream_index)
1124 for (auto filtered_frame : audio_filter_->poll_all(output_stream_index))
1125 result = sink_->try_push(AVMediaType::AVMEDIA_TYPE_AUDIO, output_stream_index, std::move(filtered_frame));
1127 else if (needed == AVMediaType::AVMEDIA_TYPE_VIDEO)
1129 initialize_video_filter_if_needed(*input_frames_for_streams.at(input_stream_index));
1130 video_filter_->push(std::move(input_frames_for_streams.at(input_stream_index)));
1132 for (auto filtered_frame : video_filter_->poll_all())
1133 result = sink_->try_push(AVMediaType::AVMEDIA_TYPE_VIDEO, 0, std::move(filtered_frame));
1136 CASPAR_THROW_EXCEPTION(not_supported());
1139 else if (source_->eof())
1146 result = boost::none;
1150 graph_->set_tag(caspar::diagnostics::tag_severity::WARNING, "dropped-frame");
1151 result = needed; // Repeat same media type
1157 if (is_logging_quiet_for_thread())
1159 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
1163 CASPAR_LOG_CURRENT_EXCEPTION();
1166 boost::lock_guard<tbb::spin_mutex> lock(exception_mutex_);
1167 exception_ = boost::current_exception();
1170 video_filter_.reset();
1171 audio_filter_.reset();
1177 template<typename T>
1178 void set_if_changed(bool& changed, T& old_value, T new_value)
1180 if (old_value != new_value)
1183 old_value = new_value;
1187 void initialize_audio_filter_if_needed(const std::vector<std::shared_ptr<AVFrame>>& av_frames_per_stream)
1189 bool changed = av_frames_per_stream.size() != source_audio_streams_.size();
1190 source_audio_streams_.resize(av_frames_per_stream.size());
1192 for (int i = 0; i < av_frames_per_stream.size(); ++i)
1194 auto& av_frame = *av_frames_per_stream.at(i);
1195 auto& stream = source_audio_streams_.at(i);
1197 auto channel_layout = av_frame.channel_layout == 0
1198 ? av_get_default_channel_layout(av_frame.channels)
1199 : av_frame.channel_layout;
1201 set_if_changed(changed, stream.sampleformat, static_cast<AVSampleFormat>(av_frame.format));
1202 set_if_changed(changed, stream.num_channels, av_frame.channels);
1203 set_if_changed(changed, stream.channel_layout, channel_layout);
1207 initialize_audio_filter();
1210 void initialize_audio_filter()
1212 std::vector<audio_input_pad> input_pads;
1213 std::vector<audio_output_pad> output_pads;
1215 for (auto& source_audio_stream : source_audio_streams_)
1217 input_pads.emplace_back(
1218 boost::rational<int>(1, source_->samplerate()),
1219 source_->samplerate(),
1220 source_audio_stream.sampleformat,
1221 source_audio_stream.channel_layout);
1224 auto total_num_channels = cpplinq::from(source_audio_streams_)
1225 .select([](const audio_stream_info& info) { return info.num_channels; })
1226 .aggregate(0, std::plus<int>());
1228 if (total_num_channels > 1 && sink_->wanted_num_audio_streams() > 1)
1229 CASPAR_THROW_EXCEPTION(invalid_operation()
1230 << msg_info("only one-to-many or many-to-one audio stream conversion supported."));
1232 std::wstring amerge;
1234 if (sink_->wanted_num_audio_streams() == 1 && !sink_->wanted_num_channels_per_stream())
1236 output_pads.emplace_back(
1237 sink_->supported_samplerates(),
1238 sink_->supported_sample_formats(),
1239 std::vector<int64_t>({ av_get_default_channel_layout(total_num_channels) }));
1241 if (source_audio_streams_.size() > 1)
1243 for (int i = 0; i < source_audio_streams_.size(); ++i)
1244 amerge += L"[a:" + boost::lexical_cast<std::wstring>(i) + L"]";
1246 amerge += L"amerge=inputs=" + boost::lexical_cast<std::wstring>(source_audio_streams_.size());
1250 std::wstring afilter = u16(afilter_);
1252 if (!amerge.empty())
1254 afilter = prepend_filter(u16(afilter), amerge);
1255 afilter += L"[aout:0]";
1258 audio_filter_.reset(new audio_filter(input_pads, output_pads, u8(afilter)));
1261 void initialize_video_filter_if_needed(const AVFrame& av_frame)
1263 bool changed = false;
1265 set_if_changed(changed, source_video_stream_.width, av_frame.width);
1266 set_if_changed(changed, source_video_stream_.height, av_frame.height);
1267 set_if_changed(changed, source_video_stream_.pixelformat, static_cast<AVPixelFormat>(av_frame.format));
1269 core::field_mode field_mode = core::field_mode::progressive;
1271 if (av_frame.interlaced_frame)
1272 field_mode = av_frame.top_field_first ? core::field_mode::upper : core::field_mode::lower;
1274 set_if_changed(changed, source_video_stream_.fieldmode, field_mode);
1277 initialize_video_filter();
1280 void initialize_video_filter()
1282 if (source_video_stream_.fieldmode != core::field_mode::progressive && !filter::is_deinterlacing(u16(vfilter_)))
1283 vfilter_ = u8(append_filter(u16(vfilter_), L"YADIF=1:-1"));
1285 if (source_video_stream_.height == 480) // NTSC DV
1287 auto pad_str = L"PAD=" + boost::lexical_cast<std::wstring>(source_video_stream_.width) + L":486:0:2:black";
1288 vfilter_ = u8(append_filter(u16(vfilter_), pad_str));
1291 video_filter_.reset(new filter(
1292 source_video_stream_.width,
1293 source_video_stream_.height,
1294 1 / source_->framerate(),
1295 source_->framerate(),
1296 boost::rational<int>(1, 1), // TODO
1297 source_video_stream_.pixelformat,
1298 sink_->supported_pixel_formats(),
1300 sink_->framerate(framerate());
1304 spl::shared_ptr<struct ffmpeg_pipeline_backend> create_internal_pipeline()
1306 return spl::make_shared<ffmpeg_pipeline_backend_internal>();