2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Helge Norberg, helge.norberg@svt.se
20 * Author: Robert Nagy, ronag89@gmail.com
25 #include "ffmpeg_pipeline_backend.h"
26 #include "ffmpeg_pipeline_backend_internal.h"
27 #include "producer/input/input.h"
28 #include "producer/video/video_decoder.h"
29 #include "producer/audio/audio_decoder.h"
30 #include "producer/filter/audio_filter.h"
31 #include "producer/filter/filter.h"
32 #include "producer/util/util.h"
33 #include "ffmpeg_error.h"
36 #include <common/diagnostics/graph.h>
37 #include <common/os/general_protection_fault.h>
38 #include <common/enum_class.h>
40 #include <core/frame/audio_channel_layout.h>
41 #include <core/frame/frame.h>
42 #include <core/frame/frame_factory.h>
43 #include <core/video_format.h>
50 #include <tbb/atomic.h>
51 #include <tbb/concurrent_queue.h>
52 #include <tbb/spin_mutex.h>
54 #include <boost/thread.hpp>
55 #include <boost/optional.hpp>
56 #include <boost/exception_ptr.hpp>
58 namespace caspar { namespace ffmpeg {
60 std::string to_string(const boost::rational<int>& framerate)
62 return boost::lexical_cast<std::string>(framerate.numerator())
63 + "/" + boost::lexical_cast<std::string>(framerate.denominator()) + " (" + boost::lexical_cast<std::string>(static_cast<double>(framerate.numerator()) / static_cast<double>(framerate.denominator())) + ") fps";
66 std::vector<int> find_audio_cadence(const boost::rational<int>& framerate)
68 static std::map<boost::rational<int>, std::vector<int>> CADENCES_BY_FRAMERATE = []
70 std::map<boost::rational<int>, std::vector<int>> result;
72 for (core::video_format format : enum_constants<core::video_format>())
74 core::video_format_desc desc(format);
75 boost::rational<int> format_rate(desc.time_scale, desc.duration);
77 result.insert(std::make_pair(format_rate, desc.audio_cadence));
83 auto exact_match = CADENCES_BY_FRAMERATE.find(framerate);
85 if (exact_match != CADENCES_BY_FRAMERATE.end())
86 return exact_match->second;
88 boost::rational<int> closest_framerate_diff = std::numeric_limits<int>::max();
89 boost::rational<int> closest_framerate = 0;
91 for (auto format_framerate : CADENCES_BY_FRAMERATE | boost::adaptors::map_keys)
93 auto diff = boost::abs(framerate - format_framerate);
95 if (diff < closest_framerate_diff)
97 closest_framerate_diff = diff;
98 closest_framerate = format_framerate;
102 if (is_logging_quiet_for_thread())
103 CASPAR_LOG(debug) << "No exact audio cadence match found for framerate " << to_string(framerate)
104 << "\nClosest match is " << to_string(closest_framerate)
105 << "\nwhich is a " << to_string(closest_framerate_diff) << " difference.";
107 CASPAR_LOG(warning) << "No exact audio cadence match found for framerate " << to_string(framerate)
108 << "\nClosest match is " << to_string(closest_framerate)
109 << "\nwhich is a " << to_string(closest_framerate_diff) << " difference.";
111 return CADENCES_BY_FRAMERATE[closest_framerate];
116 virtual ~source() { }
118 virtual std::wstring print() const = 0;
119 virtual void start() { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
120 virtual void graph(spl::shared_ptr<caspar::diagnostics::graph> g) { }
121 virtual void stop() { }
122 virtual void start_frame(std::uint32_t frame) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
123 virtual std::uint32_t start_frame() const { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
124 virtual void loop(bool value) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
125 virtual bool loop() const { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
126 virtual void length(std::uint32_t frames) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
127 virtual std::uint32_t length() const { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
128 virtual std::string filename() const { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print())); }
129 virtual void seek(std::uint32_t frame) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
130 virtual bool has_audio() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
131 virtual int samplerate() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
132 virtual bool has_video() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
133 virtual bool eof() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
134 virtual boost::rational<int> framerate() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
135 virtual std::uint32_t frame_number() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
136 virtual std::vector<std::shared_ptr<AVFrame>> get_input_frames_for_streams(AVMediaType type) { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
139 struct no_source_selected : public source
141 std::wstring print() const override
143 return L"[no_source_selected]";
147 class file_source : public source
149 std::wstring filename_;
150 spl::shared_ptr<diagnostics::graph> graph_;
151 std::uint32_t start_frame_ = 0;
152 std::uint32_t length_ = std::numeric_limits<std::uint32_t>::max();
154 mutable boost::mutex pointer_mutex_;
155 std::shared_ptr<input> input_;
156 std::vector<spl::shared_ptr<audio_decoder>> audio_decoders_;
157 std::shared_ptr<video_decoder> video_decoder_;
158 bool started_ = false;
160 file_source(std::string filename)
161 : filename_(u16(filename))
165 std::wstring print() const override
167 return L"[file_source " + filename_ + L"]";
170 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
172 graph_ = std::move(g);
175 void start() override
177 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
178 bool thumbnail_mode = is_logging_quiet_for_thread();
179 input_.reset(new input(graph_, filename_, loop_, start_frame_, length_, thumbnail_mode));
181 for (int i = 0; i < input_->num_audio_streams(); ++i)
185 audio_decoders_.push_back(spl::make_shared<audio_decoder>(*input_, core::video_format_desc(), i));
189 if (is_logging_quiet_for_thread())
191 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
192 CASPAR_LOG(info) << print() << " Failed to open audio-stream. Turn on log level debug to see more information.";
196 CASPAR_LOG_CURRENT_EXCEPTION();
197 CASPAR_LOG(warning) << print() << " Failed to open audio-stream.";
202 if (audio_decoders_.empty())
203 CASPAR_LOG(debug) << print() << " No audio-stream found. Running without audio.";
207 video_decoder_.reset(new video_decoder(*input_, false));
209 catch (averror_stream_not_found&)
211 CASPAR_LOG(debug) << print() << " No video-stream found. Running without video.";
215 if (is_logging_quiet_for_thread())
217 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
218 CASPAR_LOG(info) << print() << " Failed to open video-stream. Running without audio. Turn on log level debug to see more information.";
222 CASPAR_LOG_CURRENT_EXCEPTION();
223 CASPAR_LOG(warning) << print() << " Failed to open video-stream. Running without audio.";
235 void start_frame(std::uint32_t frame) override
237 start_frame_ = frame;
239 auto i = get_input();
244 std::uint32_t start_frame() const override
249 void loop(bool value) override
253 auto i = get_input();
258 bool loop() const override
263 void length(std::uint32_t frames) override
267 auto i = get_input();
272 std::uint32_t length() const override
274 auto v = get_video_decoder();
277 return v->nb_frames();
279 auto a = get_audio_decoders();
282 return a.at(0)->nb_frames(); // Should be ok.
287 std::string filename() const override
289 return u8(filename_);
292 void seek(std::uint32_t frame) override
295 get_input()->seek(frame);
298 bool eof() const override
300 auto i = get_input();
301 return !i || i->eof();
304 bool has_audio() const override
306 return !get_audio_decoders().empty();
309 int samplerate() const override
311 if (get_audio_decoders().empty())
317 bool has_video() const override
319 return static_cast<bool>(get_video_decoder());
322 boost::rational<int> framerate() const override
324 auto decoder = get_video_decoder();
329 return decoder->framerate();
332 std::uint32_t frame_number() const override
334 auto decoder = get_video_decoder();
339 return decoder->file_frame_number();
342 std::vector<std::shared_ptr<AVFrame>> get_input_frames_for_streams(AVMediaType type) override
344 auto a_decoders = get_audio_decoders();
345 auto v_decoder = get_video_decoder();
348 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && !a_decoders.empty())
350 std::vector<std::shared_ptr<AVFrame>> frames;
352 for (auto& a_decoder : a_decoders)
354 std::shared_ptr<AVFrame> frame;
356 for (int i = 0; i < 64; ++i)
358 frame = (*a_decoder)();
360 if (frame && frame->data[0])
366 frames.push_back(std::move(frame));
371 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO && v_decoder)
373 std::shared_ptr<AVFrame> frame;
375 for (int i = 0; i < 128; ++i)
377 frame = (*v_decoder)();
379 if (frame && frame->data[0])
384 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(
385 print() + L" Unhandled media type " + boost::lexical_cast<std::wstring>(type)));
390 void expect_started() const
393 CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" Not started."));
396 std::shared_ptr<input> get_input() const
398 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
402 std::vector<spl::shared_ptr<audio_decoder>> get_audio_decoders() const
404 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
405 return audio_decoders_;
408 std::shared_ptr<video_decoder> get_video_decoder() const
410 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
411 return video_decoder_;
415 class memory_source : public source
417 int samplerate_ = -1;
418 int num_channels_ = -1;
421 boost::rational<int> framerate_ = -1;
423 tbb::atomic<bool> running_;
424 tbb::concurrent_bounded_queue<caspar::array<const int32_t>> audio_frames_;
425 tbb::concurrent_bounded_queue<caspar::array<const uint8_t>> video_frames_;
426 int64_t audio_pts_ = 0;
427 int64_t video_pts_ = 0;
432 video_frames_.set_capacity(1);
433 audio_frames_.set_capacity(1);
441 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
445 std::wstring print() const override
447 return L"[memory_source]";
450 void enable_audio(int samplerate, int num_channels)
452 samplerate_ = samplerate;
453 num_channels_ = num_channels;
456 void enable_video(int width, int height, boost::rational<int> framerate)
462 void start() override
470 video_frames_.try_push(caspar::array<const uint8_t>());
471 audio_frames_.try_push(caspar::array<const int32_t>());
474 bool has_audio() const override
476 return samplerate_ != -1;
479 int samplerate() const override
484 bool has_video() const override
489 bool eof() const override
494 boost::rational<int> framerate() const override
499 bool try_push_audio(caspar::array<const std::int32_t> data)
502 CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" audio not enabled."));
504 if (data.empty() || data.size() % num_channels_ != 0)
505 CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info(print() + L" audio with incorrect number of channels submitted."));
507 return audio_frames_.try_push(std::move(data));
510 bool try_push_video(caspar::array<const std::uint8_t> data)
513 CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" video not enabled."));
515 if (data.size() != width_ * height_ * 4)
516 CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info(print() + L" video with incorrect size submitted."));
518 return video_frames_.try_push(std::move(data));
521 std::vector<std::shared_ptr<AVFrame>> get_input_frames_for_streams(AVMediaType type) override
524 CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not running."));
526 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && has_audio())
528 caspar::array<const std::int32_t> samples;
529 audio_frames_.pop(samples);
534 spl::shared_ptr<AVFrame> av_frame(av_frame_alloc(), [samples](AVFrame* p) { av_frame_free(&p); });
536 av_frame->channels = num_channels_;
537 av_frame->channel_layout = av_get_default_channel_layout(num_channels_);
538 av_frame->sample_rate = samplerate_;
539 av_frame->nb_samples = static_cast<int>(samples.size()) / num_channels_;
540 av_frame->format = AV_SAMPLE_FMT_S32;
541 av_frame->pts = audio_pts_;
543 audio_pts_ += av_frame->nb_samples;
545 FF(av_samples_fill_arrays(
546 av_frame->extended_data,
548 reinterpret_cast<const std::uint8_t*>(&*samples.begin()),
550 av_frame->nb_samples,
551 static_cast<AVSampleFormat>(av_frame->format),
556 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO && has_video())
558 caspar::array<const std::uint8_t> data;
559 video_frames_.pop(data);
564 spl::shared_ptr<AVFrame> av_frame(av_frame_alloc(), [data](AVFrame* p) { av_frame_free(&p); });
565 avcodec_get_frame_defaults(av_frame.get());
567 const auto sample_aspect_ratio = boost::rational<int>(width_, height_);
569 av_frame->format = AV_PIX_FMT_BGRA;
570 av_frame->width = width_;
571 av_frame->height = height_;
572 av_frame->sample_aspect_ratio.num = sample_aspect_ratio.numerator();
573 av_frame->sample_aspect_ratio.den = sample_aspect_ratio.denominator();
574 av_frame->pts = video_pts_;
578 FF(av_image_fill_arrays(
582 static_cast<AVPixelFormat>(av_frame->format),
590 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(
591 print() + L" Unhandled media type " + boost::lexical_cast<std::wstring>(type)));
599 virtual std::wstring print() const = 0;
600 virtual void graph(spl::shared_ptr<caspar::diagnostics::graph> g) { }
601 virtual void acodec(std::string codec) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
602 virtual void vcodec(std::string codec) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
603 virtual void format(std::string fmt) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
604 virtual void framerate(boost::rational<int> framerate) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
605 virtual void start(bool has_audio, bool has_video) { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
606 virtual void stop() { }
607 virtual std::vector<AVSampleFormat> supported_sample_formats() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
608 virtual std::vector<int> supported_samplerates() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
609 virtual std::vector<AVPixelFormat> supported_pixel_formats() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
610 virtual int wanted_num_audio_streams() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
611 virtual boost::optional<int> wanted_num_channels_per_stream() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
612 virtual boost::optional<AVMediaType> try_push(AVMediaType type, int stream_index, spl::shared_ptr<AVFrame> frame) { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
613 virtual void eof() { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
616 struct no_sink_selected : public sink
618 std::wstring print() const override
620 return L"[no_sink_selected]";
624 class file_sink : public sink
626 std::wstring filename_;
627 spl::shared_ptr<diagnostics::graph> graph_;
629 file_sink(std::string filename)
630 : filename_(u16(std::move(filename)))
634 std::wstring print() const override
636 return L"[file_sink " + filename_ + L"]";
639 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
641 graph_ = std::move(g);
645 class memory_sink : public sink
647 spl::shared_ptr<core::frame_factory> factory_;
649 bool has_audio_ = false;
650 bool has_video_ = false;
651 std::vector<int> audio_cadence_;
652 core::audio_channel_layout channel_layout_ = core::audio_channel_layout::invalid();
653 core::mutable_audio_buffer audio_samples_;
655 std::queue<std::shared_ptr<AVFrame>> video_frames_;
657 tbb::concurrent_bounded_queue<core::draw_frame> output_frames_;
658 tbb::atomic<bool> running_;
660 memory_sink(spl::shared_ptr<core::frame_factory> factory, core::video_format_desc format)
661 : factory_(std::move(factory))
662 , audio_cadence_(format.audio_cadence)
664 output_frames_.set_capacity(2);
666 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
667 // This cadence fills the audio mixer most optimally.
668 boost::range::rotate(audio_cadence_, std::end(audio_cadence_) - 1);
676 std::wstring print() const override
678 return L"[memory_sink]";
681 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
685 void framerate(boost::rational<int> framerate) override
687 audio_cadence_ = find_audio_cadence(framerate);
688 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
689 // This cadence fills the audio mixer most optimally.
690 boost::range::rotate(audio_cadence_, std::end(audio_cadence_) - 1);
693 void start(bool has_audio, bool has_video) override
695 has_audio_ = has_audio;
696 has_video_ = has_video;
707 std::vector<AVSampleFormat> supported_sample_formats() const override
709 return { AVSampleFormat::AV_SAMPLE_FMT_S32 };
712 std::vector<int> supported_samplerates() const override {
716 std::vector<AVPixelFormat> supported_pixel_formats() const override
719 AVPixelFormat::AV_PIX_FMT_YUVA420P,
720 AVPixelFormat::AV_PIX_FMT_YUV444P,
721 AVPixelFormat::AV_PIX_FMT_YUV422P,
722 AVPixelFormat::AV_PIX_FMT_YUV420P,
723 AVPixelFormat::AV_PIX_FMT_YUV411P,
724 AVPixelFormat::AV_PIX_FMT_BGRA,
725 AVPixelFormat::AV_PIX_FMT_ARGB,
726 AVPixelFormat::AV_PIX_FMT_RGBA,
727 AVPixelFormat::AV_PIX_FMT_ABGR,
728 AVPixelFormat::AV_PIX_FMT_GRAY8
732 int wanted_num_audio_streams() const override
737 boost::optional<int> wanted_num_channels_per_stream() const
742 boost::optional<AVMediaType> try_push(AVMediaType type, int stream_index, spl::shared_ptr<AVFrame> av_frame) override
744 if (!has_audio_ && !has_video_)
745 CASPAR_THROW_EXCEPTION(invalid_operation());
747 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && av_frame->data[0])
749 if (channel_layout_ == core::audio_channel_layout::invalid()) // First audio
751 channel_layout_ = get_audio_channel_layout(av_frame->channels, av_frame->channel_layout, L"");
753 // Insert silence samples so that the audio mixer is guaranteed to be filled.
754 auto min_num_samples_per_frame = *boost::min_element(audio_cadence_);
755 auto max_num_samples_per_frame = *boost::max_element(audio_cadence_);
756 auto cadence_safety_samples = max_num_samples_per_frame - min_num_samples_per_frame;
757 audio_samples_.resize(channel_layout_.num_channels * cadence_safety_samples, 0);
760 auto ptr = reinterpret_cast<int32_t*>(av_frame->data[0]);
762 audio_samples_.insert(audio_samples_.end(), ptr, ptr + av_frame->linesize[0] / sizeof(int32_t));
764 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO)
766 video_frames_.push(std::move(av_frame));
773 (channel_layout_ != core::audio_channel_layout::invalid() && audio_samples_.size() >= audio_cadence_.front() * channel_layout_.num_channels);
776 !video_frames_.empty();
779 return AVMediaType::AVMEDIA_TYPE_AUDIO;
782 return AVMediaType::AVMEDIA_TYPE_VIDEO;
784 core::mutable_audio_buffer audio_data;
788 auto begin = audio_samples_.begin();
789 auto end = begin + audio_cadence_.front() * channel_layout_.num_channels;
791 audio_data.insert(audio_data.begin(), begin, end);
792 audio_samples_.erase(begin, end);
793 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_) + 1);
796 if (!has_video_) // Audio only
798 core::mutable_frame audio_only_frame(
800 std::move(audio_data),
802 core::pixel_format_desc(core::pixel_format::invalid),
805 output_frames_.push(core::draw_frame(std::move(audio_only_frame)));
807 return AVMediaType::AVMEDIA_TYPE_AUDIO;
810 auto output_frame = make_frame(this, spl::make_shared_ptr(video_frames_.front()), *factory_, channel_layout_);
812 output_frame.audio_data() = std::move(audio_data);
814 output_frames_.push(core::draw_frame(std::move(output_frame)));
820 // Drain rest, regardless of it being enough or not.
821 while (!video_frames_.empty() || !audio_samples_.empty())
823 core::mutable_audio_buffer audio_data;
825 audio_data.swap(audio_samples_);
827 if (!video_frames_.empty())
829 auto output_frame = make_frame(this, spl::make_shared_ptr(video_frames_.front()), *factory_, channel_layout_);
831 output_frame.audio_data() = std::move(audio_data);
833 output_frames_.push(core::draw_frame(std::move(output_frame)));
837 core::mutable_frame audio_only_frame(
839 std::move(audio_data),
841 core::pixel_format_desc(core::pixel_format::invalid),
844 output_frames_.push(core::draw_frame(std::move(audio_only_frame)));
845 output_frames_.push(core::draw_frame::empty());
850 core::draw_frame try_pop_frame()
852 core::draw_frame frame = core::draw_frame::late();
854 if (!output_frames_.try_pop(frame) && !running_)
855 return core::draw_frame::empty();
861 struct audio_stream_info
863 int num_channels = 0;
864 AVSampleFormat sampleformat = AVSampleFormat::AV_SAMPLE_FMT_NONE;
865 uint64_t channel_layout = 0;
868 struct video_stream_info
872 AVPixelFormat pixelformat = AVPixelFormat::AV_PIX_FMT_NONE;
873 core::field_mode fieldmode = core::field_mode::progressive;
876 class ffmpeg_pipeline_backend_internal : public ffmpeg_pipeline_backend
878 spl::shared_ptr<diagnostics::graph> graph_;
880 spl::unique_ptr<source> source_ = spl::make_unique<no_source_selected>();
881 std::function<bool (caspar::array<const std::int32_t> data)> try_push_audio_;
882 std::function<bool (caspar::array<const std::uint8_t> data)> try_push_video_;
884 std::vector<audio_stream_info> source_audio_streams_;
885 video_stream_info source_video_stream_;
887 std::string afilter_;
888 std::unique_ptr<audio_filter> audio_filter_;
889 std::string vfilter_;
890 std::unique_ptr<filter> video_filter_;
892 spl::unique_ptr<sink> sink_ = spl::make_unique<no_sink_selected>();
893 std::function<core::draw_frame ()> try_pop_frame_;
895 tbb::atomic<bool> started_;
896 tbb::spin_mutex exception_mutex_;
897 boost::exception_ptr exception_;
898 boost::thread thread_;
900 ffmpeg_pipeline_backend_internal()
903 diagnostics::register_graph(graph_);
906 ~ffmpeg_pipeline_backend_internal()
911 void throw_if_error()
913 boost::lock_guard<tbb::spin_mutex> lock(exception_mutex_);
915 if (exception_ != nullptr)
916 boost::rethrow_exception(exception_);
919 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
921 graph_ = std::move(g);
922 source_->graph(graph_);
923 sink_->graph(graph_);
928 void from_file(std::string filename) override
930 source_ = spl::make_unique<file_source>(std::move(filename));
931 try_push_audio_ = std::function<bool (caspar::array<const std::int32_t>)>();
932 try_push_video_ = std::function<bool (caspar::array<const std::uint8_t>)>();
933 source_->graph(graph_);
936 void from_memory_only_audio(int num_channels, int samplerate) override
938 auto source = spl::make_unique<memory_source>();
939 auto source_ptr = source.get();
940 try_push_audio_ = [this, source_ptr](caspar::array<const std::int32_t> data) { return source_ptr->try_push_audio(std::move(data)); };
941 source->enable_audio(samplerate, num_channels);
943 source_ = std::move(source);
944 source_->graph(graph_);
947 void from_memory_only_video(int width, int height, boost::rational<int> framerate) override
949 auto source = spl::make_unique<memory_source>();
950 auto source_ptr = source.get();
951 try_push_video_ = [this, source_ptr](caspar::array<const std::uint8_t> data) { return source_ptr->try_push_video(std::move(data)); };
952 source->enable_video(width, height, std::move(framerate));
954 source_ = std::move(source);
955 source_->graph(graph_);
958 void from_memory(int num_channels, int samplerate, int width, int height, boost::rational<int> framerate) override
960 auto source = spl::make_unique<memory_source>();
961 auto source_ptr = source.get();
962 try_push_audio_ = [this, source_ptr](caspar::array<const std::int32_t> data) { return source_ptr->try_push_audio(std::move(data)); };
963 try_push_video_ = [this, source_ptr](caspar::array<const std::uint8_t> data) { return source_ptr->try_push_video(std::move(data)); };
964 source->enable_audio(samplerate, num_channels);
965 source->enable_video(width, height, std::move(framerate));
967 source_ = std::move(source);
968 source_->graph(graph_);
971 void start_frame(std::uint32_t frame) override { source_->start_frame(frame); }
972 std::uint32_t start_frame() const override { return source_->start_frame(); }
973 void length(std::uint32_t frames) override { source_->length(frames); }
974 std::uint32_t length() const override { return source_->length(); }
975 void seek(std::uint32_t frame) override { source_->seek(frame); }
976 void loop(bool value) override { source_->loop(value); }
977 bool loop() const override { return source_->loop(); }
978 std::string source_filename() const override { return source_->filename(); }
982 void vfilter(std::string filter) override
984 vfilter_ = std::move(filter);
987 void afilter(std::string filter) override
989 afilter_ = std::move(filter);
992 int width() const override
994 return source_video_stream_.width;
997 int height() const override
999 return source_video_stream_.height;
1002 boost::rational<int> framerate() const override
1004 bool double_rate = filter::is_double_rate(u16(vfilter_));
1006 return double_rate ? source_->framerate() * 2 : source_->framerate();
1009 bool progressive() const override
1016 void to_memory(spl::shared_ptr<core::frame_factory> factory, core::video_format_desc format) override
1018 auto sink = spl::make_unique<memory_sink>(std::move(factory), std::move(format));
1019 auto sink_ptr = sink.get();
1020 try_pop_frame_ = [sink_ptr] { return sink_ptr->try_pop_frame(); };
1022 sink_ = std::move(sink);
1023 sink_->graph(graph_);
1026 void to_file(std::string filename) override
1028 sink_ = spl::make_unique<file_sink>(std::move(filename));
1029 try_pop_frame_ = std::function<core::draw_frame ()>();
1030 sink_->graph(graph_);
1033 void acodec(std::string codec) override { sink_->acodec(std::move(codec)); }
1034 void vcodec(std::string codec) override { sink_->vcodec(std::move(codec)); }
1035 void format(std::string fmt) override { sink_->format(std::move(fmt)); }
1039 void start() override
1042 sink_->start(source_->has_audio(), source_->has_video());
1044 bool quiet = is_logging_quiet_for_thread();
1046 thread_ = boost::thread([=] { run(quiet); });
1049 bool try_push_audio(caspar::array<const std::int32_t> data) override
1053 if (try_push_audio_)
1054 return try_push_audio_(std::move(data));
1059 bool try_push_video(caspar::array<const std::uint8_t> data) override
1063 if (try_push_video_)
1064 return try_push_video_(std::move(data));
1069 core::draw_frame try_pop_frame() override
1073 if (!try_pop_frame_)
1074 CASPAR_THROW_EXCEPTION(invalid_operation());
1076 return try_pop_frame_();
1079 std::uint32_t last_frame() const override
1081 return source_->frame_number();
1084 bool started() const override
1089 void stop() override
1096 if (thread_.joinable())
1101 void run(bool quiet)
1103 ensure_gpf_handler_installed_for_thread(u8(L"ffmpeg-pipeline: " + source_->print() + L" -> " + sink_->print()).c_str());
1104 auto quiet_logging = temporary_enable_quiet_logging_for_thread(quiet);
1108 boost::optional<AVMediaType> result = source_->has_audio() ? AVMediaType::AVMEDIA_TYPE_AUDIO : AVMediaType::AVMEDIA_TYPE_VIDEO;
1110 while (started_ && (source_->has_audio() || source_->has_video()))
1112 auto needed = *result;
1113 auto input_frames_for_streams = source_->get_input_frames_for_streams(needed);
1115 if (!input_frames_for_streams.empty() && input_frames_for_streams.at(0))
1117 for (int input_stream_index = 0; input_stream_index < input_frames_for_streams.size(); ++input_stream_index)
1119 if (needed == AVMediaType::AVMEDIA_TYPE_AUDIO)
1121 initialize_audio_filter_if_needed(input_frames_for_streams);
1122 audio_filter_->push(input_stream_index, std::move(input_frames_for_streams.at(input_stream_index)));
1124 for (int output_stream_index = 0; output_stream_index < sink_->wanted_num_audio_streams(); ++output_stream_index)
1125 for (auto filtered_frame : audio_filter_->poll_all(output_stream_index))
1126 result = sink_->try_push(AVMediaType::AVMEDIA_TYPE_AUDIO, output_stream_index, std::move(filtered_frame));
1128 else if (needed == AVMediaType::AVMEDIA_TYPE_VIDEO)
1130 initialize_video_filter_if_needed(*input_frames_for_streams.at(input_stream_index));
1131 video_filter_->push(std::move(input_frames_for_streams.at(input_stream_index)));
1133 for (auto filtered_frame : video_filter_->poll_all())
1134 result = sink_->try_push(AVMediaType::AVMEDIA_TYPE_VIDEO, 0, std::move(filtered_frame));
1137 CASPAR_THROW_EXCEPTION(not_supported());
1140 else if (source_->eof())
1147 result = boost::none;
1151 graph_->set_tag(caspar::diagnostics::tag_severity::WARNING, "dropped-frame");
1152 result = needed; // Repeat same media type
1158 if (is_logging_quiet_for_thread())
1160 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
1164 CASPAR_LOG_CURRENT_EXCEPTION();
1167 boost::lock_guard<tbb::spin_mutex> lock(exception_mutex_);
1168 exception_ = boost::current_exception();
1171 video_filter_.reset();
1172 audio_filter_.reset();
1178 template<typename T>
1179 void set_if_changed(bool& changed, T& old_value, T new_value)
1181 if (old_value != new_value)
1184 old_value = new_value;
1188 void initialize_audio_filter_if_needed(const std::vector<std::shared_ptr<AVFrame>>& av_frames_per_stream)
1190 bool changed = av_frames_per_stream.size() != source_audio_streams_.size();
1191 source_audio_streams_.resize(av_frames_per_stream.size());
1193 for (int i = 0; i < av_frames_per_stream.size(); ++i)
1195 auto& av_frame = *av_frames_per_stream.at(i);
1196 auto& stream = source_audio_streams_.at(i);
1198 auto channel_layout = av_frame.channel_layout == 0
1199 ? av_get_default_channel_layout(av_frame.channels)
1200 : av_frame.channel_layout;
1202 set_if_changed(changed, stream.sampleformat, static_cast<AVSampleFormat>(av_frame.format));
1203 set_if_changed(changed, stream.num_channels, av_frame.channels);
1204 set_if_changed(changed, stream.channel_layout, channel_layout);
1208 initialize_audio_filter();
1211 void initialize_audio_filter()
1213 std::vector<audio_input_pad> input_pads;
1214 std::vector<audio_output_pad> output_pads;
1216 for (auto& source_audio_stream : source_audio_streams_)
1218 input_pads.emplace_back(
1219 boost::rational<int>(1, source_->samplerate()),
1220 source_->samplerate(),
1221 source_audio_stream.sampleformat,
1222 source_audio_stream.channel_layout);
1225 auto total_num_channels = cpplinq::from(source_audio_streams_)
1226 .select([](const audio_stream_info& info) { return info.num_channels; })
1227 .aggregate(0, std::plus<int>());
1229 if (total_num_channels > 1 && sink_->wanted_num_audio_streams() > 1)
1230 CASPAR_THROW_EXCEPTION(invalid_operation()
1231 << msg_info("only one-to-many or many-to-one audio stream conversion supported."));
1233 std::wstring amerge;
1235 if (sink_->wanted_num_audio_streams() == 1 && !sink_->wanted_num_channels_per_stream())
1237 output_pads.emplace_back(
1238 sink_->supported_samplerates(),
1239 sink_->supported_sample_formats(),
1240 std::vector<int64_t>({ av_get_default_channel_layout(total_num_channels) }));
1242 if (source_audio_streams_.size() > 1)
1244 for (int i = 0; i < source_audio_streams_.size(); ++i)
1245 amerge += L"[a:" + boost::lexical_cast<std::wstring>(i) + L"]";
1247 amerge += L"amerge=inputs=" + boost::lexical_cast<std::wstring>(source_audio_streams_.size());
1251 std::wstring afilter = u16(afilter_);
1253 if (!amerge.empty())
1255 afilter = prepend_filter(u16(afilter), amerge);
1256 afilter += L"[aout:0]";
1259 audio_filter_.reset(new audio_filter(input_pads, output_pads, u8(afilter)));
1262 void initialize_video_filter_if_needed(const AVFrame& av_frame)
1264 bool changed = false;
1266 set_if_changed(changed, source_video_stream_.width, av_frame.width);
1267 set_if_changed(changed, source_video_stream_.height, av_frame.height);
1268 set_if_changed(changed, source_video_stream_.pixelformat, static_cast<AVPixelFormat>(av_frame.format));
1270 core::field_mode field_mode = core::field_mode::progressive;
1272 if (av_frame.interlaced_frame)
1273 field_mode = av_frame.top_field_first ? core::field_mode::upper : core::field_mode::lower;
1275 set_if_changed(changed, source_video_stream_.fieldmode, field_mode);
1278 initialize_video_filter();
1281 void initialize_video_filter()
1283 if (source_video_stream_.fieldmode != core::field_mode::progressive && !filter::is_deinterlacing(u16(vfilter_)))
1284 vfilter_ = u8(append_filter(u16(vfilter_), L"YADIF=1:-1"));
1286 if (source_video_stream_.height == 480) // NTSC DV
1288 auto pad_str = L"PAD=" + boost::lexical_cast<std::wstring>(source_video_stream_.width) + L":486:0:2:black";
1289 vfilter_ = u8(append_filter(u16(vfilter_), pad_str));
1292 video_filter_.reset(new filter(
1293 source_video_stream_.width,
1294 source_video_stream_.height,
1295 1 / source_->framerate(),
1296 source_->framerate(),
1297 boost::rational<int>(1, 1), // TODO
1298 source_video_stream_.pixelformat,
1299 sink_->supported_pixel_formats(),
1301 sink_->framerate(framerate());
1305 spl::shared_ptr<struct ffmpeg_pipeline_backend> create_internal_pipeline()
1307 return spl::make_shared<ffmpeg_pipeline_backend_internal>();