2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Helge Norberg, helge.norberg@svt.se
20 * Author: Robert Nagy, ronag89@gmail.com
25 #include "ffmpeg_pipeline_backend.h"
26 #include "ffmpeg_pipeline_backend_internal.h"
27 #include "producer/input/input.h"
28 #include "producer/video/video_decoder.h"
29 #include "producer/audio/audio_decoder.h"
30 #include "producer/filter/audio_filter.h"
31 #include "producer/filter/filter.h"
32 #include "producer/util/util.h"
33 #include "ffmpeg_error.h"
36 #include <common/diagnostics/graph.h>
37 #include <common/os/general_protection_fault.h>
38 #include <common/enum_class.h>
40 #include <core/frame/audio_channel_layout.h>
41 #include <core/frame/frame.h>
42 #include <core/frame/frame_factory.h>
43 #include <core/video_format.h>
50 #include <tbb/atomic.h>
51 #include <tbb/concurrent_queue.h>
52 #include <tbb/spin_mutex.h>
54 #include <boost/thread.hpp>
55 #include <boost/optional.hpp>
56 #include <boost/exception_ptr.hpp>
58 namespace caspar { namespace ffmpeg {
60 std::string to_string(const boost::rational<int>& framerate)
62 return boost::lexical_cast<std::string>(framerate.numerator())
63 + "/" + boost::lexical_cast<std::string>(framerate.denominator()) + " (" + boost::lexical_cast<std::string>(static_cast<double>(framerate.numerator()) / static_cast<double>(framerate.denominator())) + ") fps";
66 std::vector<int> find_audio_cadence(const boost::rational<int>& framerate)
68 static std::map<boost::rational<int>, std::vector<int>> CADENCES_BY_FRAMERATE = []
70 std::map<boost::rational<int>, std::vector<int>> result;
72 for (core::video_format format : enum_constants<core::video_format>())
74 core::video_format_desc desc(format);
75 boost::rational<int> format_rate(desc.time_scale, desc.duration);
77 result.insert(std::make_pair(format_rate, desc.audio_cadence));
83 auto exact_match = CADENCES_BY_FRAMERATE.find(framerate);
85 if (exact_match != CADENCES_BY_FRAMERATE.end())
86 return exact_match->second;
88 boost::rational<int> closest_framerate_diff = std::numeric_limits<int>::max();
89 boost::rational<int> closest_framerate = 0;
91 for (auto format_framerate : CADENCES_BY_FRAMERATE | boost::adaptors::map_keys)
93 auto diff = boost::abs(framerate - format_framerate);
95 if (diff < closest_framerate_diff)
97 closest_framerate_diff = diff;
98 closest_framerate = format_framerate;
102 if (is_logging_quiet_for_thread())
103 CASPAR_LOG(debug) << "No exact audio cadence match found for framerate " << to_string(framerate)
104 << "\nClosest match is " << to_string(closest_framerate)
105 << "\nwhich is a " << to_string(closest_framerate_diff) << " difference.";
107 CASPAR_LOG(warning) << "No exact audio cadence match found for framerate " << to_string(framerate)
108 << "\nClosest match is " << to_string(closest_framerate)
109 << "\nwhich is a " << to_string(closest_framerate_diff) << " difference.";
111 return CADENCES_BY_FRAMERATE[closest_framerate];
116 virtual ~source() { }
118 virtual std::wstring print() const = 0;
119 virtual void start() { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
120 virtual void graph(spl::shared_ptr<caspar::diagnostics::graph> g) { }
121 virtual void stop() { }
122 virtual void start_frame(std::uint32_t frame) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
123 virtual std::uint32_t start_frame() const { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
124 virtual void loop(bool value) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
125 virtual bool loop() const { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
126 virtual void length(std::uint32_t frames) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
127 virtual std::uint32_t length() const { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
128 virtual std::string filename() const { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print())); }
129 virtual void seek(std::uint32_t frame) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
130 virtual bool has_audio() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
131 virtual int samplerate() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
132 virtual bool has_video() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
133 virtual bool eof() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
134 virtual boost::rational<int> framerate() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
135 virtual std::uint32_t frame_number() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
136 virtual std::vector<std::shared_ptr<AVFrame>> get_input_frames_for_streams(AVMediaType type) { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
139 struct no_source_selected : public source
141 std::wstring print() const override
143 return L"[no_source_selected]";
147 class file_source : public source
149 std::wstring filename_;
150 spl::shared_ptr<diagnostics::graph> graph_;
151 std::uint32_t start_frame_ = 0;
152 std::uint32_t length_ = std::numeric_limits<std::uint32_t>::max();
154 mutable boost::mutex pointer_mutex_;
155 std::shared_ptr<input> input_;
156 std::vector<spl::shared_ptr<audio_decoder>> audio_decoders_;
157 std::shared_ptr<video_decoder> video_decoder_;
158 bool started_ = false;
160 file_source(std::string filename)
161 : filename_(u16(filename))
165 std::wstring print() const override
167 return L"[file_source " + filename_ + L"]";
170 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
172 graph_ = std::move(g);
175 void start() override
177 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
178 bool thumbnail_mode = is_logging_quiet_for_thread();
179 input_.reset(new input(graph_, filename_, loop_, start_frame_, length_, thumbnail_mode));
181 for (int i = 0; i < input_->num_audio_streams(); ++i)
185 audio_decoders_.push_back(spl::make_shared<audio_decoder>(*input_, core::video_format::invalid, i));
189 if (is_logging_quiet_for_thread())
191 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
192 CASPAR_LOG(info) << print() << " Failed to open audio-stream. Turn on log level debug to see more information.";
196 CASPAR_LOG_CURRENT_EXCEPTION();
197 CASPAR_LOG(warning) << print() << " Failed to open audio-stream.";
202 if (audio_decoders_.empty())
203 CASPAR_LOG(debug) << print() << " No audio-stream found. Running without audio.";
207 video_decoder_.reset(new video_decoder(*input_, false));
209 catch (averror_stream_not_found&)
211 CASPAR_LOG(debug) << print() << " No video-stream found. Running without video.";
215 if (is_logging_quiet_for_thread())
217 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
218 CASPAR_LOG(info) << print() << " Failed to open video-stream. Running without audio. Turn on log level debug to see more information.";
222 CASPAR_LOG_CURRENT_EXCEPTION();
223 CASPAR_LOG(warning) << print() << " Failed to open video-stream. Running without audio.";
235 void start_frame(std::uint32_t frame) override
237 start_frame_ = frame;
239 auto i = get_input();
244 std::uint32_t start_frame() const override
249 void loop(bool value) override
253 auto i = get_input();
258 bool loop() const override
263 void length(std::uint32_t frames) override
267 auto i = get_input();
272 std::uint32_t length() const override
274 auto v = get_video_decoder();
277 return v->nb_frames();
279 auto a = get_audio_decoders();
282 return a.at(0)->nb_frames(); // Should be ok.
287 std::string filename() const override
289 return u8(filename_);
292 void seek(std::uint32_t frame) override
295 get_input()->seek(frame);
298 bool eof() const override
300 auto i = get_input();
301 return !i || i->eof();
304 bool has_audio() const override
306 return !get_audio_decoders().empty();
309 int samplerate() const override
311 if (get_audio_decoders().empty())
317 bool has_video() const override
319 return static_cast<bool>(get_video_decoder());
322 boost::rational<int> framerate() const override
324 auto decoder = get_video_decoder();
329 return decoder->framerate();
332 std::uint32_t frame_number() const override
334 auto decoder = get_video_decoder();
339 return decoder->file_frame_number();
342 std::vector<std::shared_ptr<AVFrame>> get_input_frames_for_streams(AVMediaType type) override
344 auto a_decoders = get_audio_decoders();
345 auto v_decoder = get_video_decoder();
348 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && !a_decoders.empty())
350 std::vector<std::shared_ptr<AVFrame>> frames;
352 for (auto& a_decoder : a_decoders)
354 std::shared_ptr<AVFrame> frame;
356 for (int i = 0; i < 64; ++i)
358 frame = (*a_decoder)();
360 if (frame == flush() || (frame && frame->data[0]))
366 frames.push_back(std::move(frame));
371 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO && v_decoder)
373 std::shared_ptr<AVFrame> frame;
375 for (int i = 0; i < 128; ++i)
377 frame = (*v_decoder)();
379 if (frame == flush() || (frame && frame->data[0]))
384 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(
385 print() + L" Unhandled media type " + boost::lexical_cast<std::wstring>(type)));
390 void expect_started() const
393 CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" Not started."));
396 std::shared_ptr<input> get_input() const
398 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
402 std::vector<spl::shared_ptr<audio_decoder>> get_audio_decoders() const
404 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
405 return audio_decoders_;
408 std::shared_ptr<video_decoder> get_video_decoder() const
410 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
411 return video_decoder_;
415 class memory_source : public source
417 int samplerate_ = -1;
418 int num_channels_ = -1;
421 boost::rational<int> framerate_ = -1;
423 tbb::atomic<bool> running_;
424 tbb::concurrent_bounded_queue<caspar::array<const int32_t>> audio_frames_;
425 tbb::concurrent_bounded_queue<caspar::array<const uint8_t>> video_frames_;
426 int64_t audio_pts_ = 0;
427 int64_t video_pts_ = 0;
432 video_frames_.set_capacity(1);
433 audio_frames_.set_capacity(1);
441 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
445 std::wstring print() const override
447 return L"[memory_source]";
450 void enable_audio(int samplerate, int num_channels)
452 samplerate_ = samplerate;
453 num_channels_ = num_channels;
456 void enable_video(int width, int height, boost::rational<int> framerate)
462 void start() override
470 video_frames_.try_push(caspar::array<const uint8_t>());
471 audio_frames_.try_push(caspar::array<const int32_t>());
474 bool has_audio() const override
476 return samplerate_ != -1;
479 int samplerate() const override
484 bool has_video() const override
489 bool eof() const override
494 boost::rational<int> framerate() const override
499 bool try_push_audio(caspar::array<const std::int32_t> data)
502 CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" audio not enabled."));
504 if (data.empty() || data.size() % num_channels_ != 0)
505 CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info(print() + L" audio with incorrect number of channels submitted."));
507 return audio_frames_.try_push(std::move(data));
510 bool try_push_video(caspar::array<const std::uint8_t> data)
513 CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" video not enabled."));
515 if (data.size() != width_ * height_ * 4)
516 CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info(print() + L" video with incorrect size submitted."));
518 return video_frames_.try_push(std::move(data));
521 std::vector<std::shared_ptr<AVFrame>> get_input_frames_for_streams(AVMediaType type) override
524 CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not running."));
526 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && has_audio())
528 caspar::array<const std::int32_t> samples;
529 audio_frames_.pop(samples);
534 spl::shared_ptr<AVFrame> av_frame(av_frame_alloc(), [samples](AVFrame* p) { av_frame_free(&p); });
536 av_frame->channels = num_channels_;
537 av_frame->channel_layout = av_get_default_channel_layout(num_channels_);
538 av_frame->sample_rate = samplerate_;
539 av_frame->nb_samples = static_cast<int>(samples.size()) / num_channels_;
540 av_frame->format = AV_SAMPLE_FMT_S32;
541 av_frame->pts = audio_pts_;
543 audio_pts_ += av_frame->nb_samples;
545 FF(av_samples_fill_arrays(
546 av_frame->extended_data,
548 reinterpret_cast<const std::uint8_t*>(&*samples.begin()),
550 av_frame->nb_samples,
551 static_cast<AVSampleFormat>(av_frame->format),
556 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO && has_video())
558 caspar::array<const std::uint8_t> data;
559 video_frames_.pop(data);
564 spl::shared_ptr<AVFrame> av_frame(av_frame_alloc(), [data](AVFrame* p) { av_frame_free(&p); });
565 avcodec_get_frame_defaults(av_frame.get());
567 const auto sample_aspect_ratio = boost::rational<int>(width_, height_);
569 av_frame->format = AV_PIX_FMT_BGRA;
570 av_frame->width = width_;
571 av_frame->height = height_;
572 av_frame->sample_aspect_ratio.num = sample_aspect_ratio.numerator();
573 av_frame->sample_aspect_ratio.den = sample_aspect_ratio.denominator();
574 av_frame->pts = video_pts_;
578 FF(av_image_fill_arrays(
582 static_cast<AVPixelFormat>(av_frame->format),
590 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(
591 print() + L" Unhandled media type " + boost::lexical_cast<std::wstring>(type)));
599 virtual std::wstring print() const = 0;
600 virtual void graph(spl::shared_ptr<caspar::diagnostics::graph> g) { }
601 virtual void acodec(std::string codec) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
602 virtual void vcodec(std::string codec) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
603 virtual void format(std::string fmt) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
604 virtual void framerate(boost::rational<int> framerate) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
605 virtual void start(bool has_audio, bool has_video) { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
606 virtual void stop() { }
607 virtual void flush_all() { }
608 virtual std::vector<AVSampleFormat> supported_sample_formats() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
609 virtual std::vector<int> supported_samplerates() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
610 virtual std::vector<AVPixelFormat> supported_pixel_formats() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
611 virtual int wanted_num_audio_streams() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
612 virtual boost::optional<int> wanted_num_channels_per_stream() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
613 virtual boost::optional<AVMediaType> try_push(AVMediaType type, int stream_index, spl::shared_ptr<AVFrame> frame) { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
614 virtual void eof() { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
617 struct no_sink_selected : public sink
619 std::wstring print() const override
621 return L"[no_sink_selected]";
625 class file_sink : public sink
627 std::wstring filename_;
628 spl::shared_ptr<diagnostics::graph> graph_;
630 file_sink(std::string filename)
631 : filename_(u16(std::move(filename)))
635 std::wstring print() const override
637 return L"[file_sink " + filename_ + L"]";
640 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
642 graph_ = std::move(g);
646 class memory_sink : public sink
648 spl::shared_ptr<core::frame_factory> factory_;
650 bool has_audio_ = false;
651 bool has_video_ = false;
652 std::vector<int> audio_cadence_;
653 core::audio_channel_layout channel_layout_ = core::audio_channel_layout::invalid();
654 core::mutable_audio_buffer audio_samples_;
656 std::queue<std::shared_ptr<AVFrame>> video_frames_;
657 std::shared_ptr<AVFrame> last_video_frame_;
659 tbb::concurrent_bounded_queue<core::draw_frame> output_frames_;
660 tbb::atomic<bool> running_;
662 memory_sink(spl::shared_ptr<core::frame_factory> factory, core::video_format_desc format)
663 : factory_(std::move(factory))
664 , audio_cadence_(format.audio_cadence)
666 output_frames_.set_capacity(2);
668 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
669 // This cadence fills the audio mixer most optimally.
670 boost::range::rotate(audio_cadence_, std::end(audio_cadence_) - 1);
678 std::wstring print() const override
680 return L"[memory_sink]";
683 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
687 void framerate(boost::rational<int> framerate) override
689 audio_cadence_ = find_audio_cadence(framerate);
690 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
691 // This cadence fills the audio mixer most optimally.
692 boost::range::rotate(audio_cadence_, std::end(audio_cadence_) - 1);
695 void start(bool has_audio, bool has_video) override
697 has_audio_ = has_audio;
698 has_video_ = has_video;
705 output_frames_.set_capacity(4);
708 std::vector<AVSampleFormat> supported_sample_formats() const override
710 return { AVSampleFormat::AV_SAMPLE_FMT_S32 };
713 std::vector<int> supported_samplerates() const override {
717 std::vector<AVPixelFormat> supported_pixel_formats() const override
720 AVPixelFormat::AV_PIX_FMT_YUVA420P,
721 AVPixelFormat::AV_PIX_FMT_YUV444P,
722 AVPixelFormat::AV_PIX_FMT_YUV422P,
723 AVPixelFormat::AV_PIX_FMT_YUV420P,
724 AVPixelFormat::AV_PIX_FMT_YUV411P,
725 AVPixelFormat::AV_PIX_FMT_BGRA,
726 AVPixelFormat::AV_PIX_FMT_ARGB,
727 AVPixelFormat::AV_PIX_FMT_RGBA,
728 AVPixelFormat::AV_PIX_FMT_ABGR,
729 AVPixelFormat::AV_PIX_FMT_GRAY8
733 int wanted_num_audio_streams() const override
738 boost::optional<int> wanted_num_channels_per_stream() const
743 void flush_all() override
745 audio_samples_.clear();
747 while (!video_frames_.empty())
751 boost::optional<AVMediaType> try_push(AVMediaType type, int stream_index, spl::shared_ptr<AVFrame> av_frame) override
753 if (!has_audio_ && !has_video_)
754 CASPAR_THROW_EXCEPTION(invalid_operation());
756 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && av_frame->data[0])
758 if (channel_layout_ == core::audio_channel_layout::invalid()) // First audio
760 channel_layout_ = get_audio_channel_layout(av_frame->channels, av_frame->channel_layout, L"");
762 // Insert silence samples so that the audio mixer is guaranteed to be filled.
763 auto min_num_samples_per_frame = *boost::min_element(audio_cadence_);
764 auto max_num_samples_per_frame = *boost::max_element(audio_cadence_);
765 auto cadence_safety_samples = max_num_samples_per_frame - min_num_samples_per_frame;
766 audio_samples_.resize(channel_layout_.num_channels * cadence_safety_samples, 0);
769 auto ptr = reinterpret_cast<int32_t*>(av_frame->data[0]);
771 audio_samples_.insert(audio_samples_.end(), ptr, ptr + av_frame->linesize[0] / sizeof(int32_t));
773 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO)
775 video_frames_.push(std::move(av_frame));
782 (channel_layout_ != core::audio_channel_layout::invalid() && audio_samples_.size() >= audio_cadence_.front() * channel_layout_.num_channels);
785 !video_frames_.empty();
788 return AVMediaType::AVMEDIA_TYPE_AUDIO;
791 return AVMediaType::AVMEDIA_TYPE_VIDEO;
793 core::mutable_audio_buffer audio_data;
797 auto begin = audio_samples_.begin();
798 auto end = begin + audio_cadence_.front() * channel_layout_.num_channels;
800 audio_data.insert(audio_data.begin(), begin, end);
801 audio_samples_.erase(begin, end);
802 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_) + 1);
805 if (!has_video_) // Audio only
807 core::mutable_frame audio_only_frame(
809 std::move(audio_data),
811 core::pixel_format_desc(core::pixel_format::invalid),
814 output_frames_.push(core::draw_frame(std::move(audio_only_frame)));
816 return AVMediaType::AVMEDIA_TYPE_AUDIO;
819 auto output_frame = make_frame(this, spl::make_shared_ptr(video_frames_.front()), *factory_, channel_layout_);
820 last_video_frame_ = video_frames_.front();
822 output_frame.audio_data() = std::move(audio_data);
824 output_frames_.push(core::draw_frame(std::move(output_frame)));
830 // Drain rest, regardless of it being enough or not.
831 while (!video_frames_.empty() || !audio_samples_.empty())
833 core::mutable_audio_buffer audio_data;
835 audio_data.swap(audio_samples_);
837 if (video_frames_.empty() && !audio_data.empty() && last_video_frame_) // More audio samples than video
839 video_frames_.push(last_video_frame_);
842 if (!video_frames_.empty())
844 auto output_frame = make_frame(this, spl::make_shared_ptr(video_frames_.front()), *factory_, channel_layout_);
845 last_video_frame_ = video_frames_.front();
847 output_frame.audio_data() = std::move(audio_data);
849 output_frames_.push(core::draw_frame(std::move(output_frame)));
853 core::mutable_frame audio_only_frame(
855 std::move(audio_data),
857 core::pixel_format_desc(core::pixel_format::invalid),
860 output_frames_.push(core::draw_frame(std::move(audio_only_frame)));
861 output_frames_.push(core::draw_frame::empty());
866 core::draw_frame try_pop_frame()
868 core::draw_frame frame = core::draw_frame::late();
870 if (!output_frames_.try_pop(frame) && !running_)
871 return core::draw_frame::empty();
877 struct audio_stream_info
879 int num_channels = 0;
880 AVSampleFormat sampleformat = AVSampleFormat::AV_SAMPLE_FMT_NONE;
881 uint64_t channel_layout = 0;
884 struct video_stream_info
888 AVPixelFormat pixelformat = AVPixelFormat::AV_PIX_FMT_NONE;
889 core::field_mode fieldmode = core::field_mode::progressive;
892 class ffmpeg_pipeline_backend_internal : public ffmpeg_pipeline_backend
894 spl::shared_ptr<diagnostics::graph> graph_;
896 spl::unique_ptr<source> source_ = spl::make_unique<no_source_selected>();
897 std::function<bool (caspar::array<const std::int32_t> data)> try_push_audio_;
898 std::function<bool (caspar::array<const std::uint8_t> data)> try_push_video_;
900 std::vector<audio_stream_info> source_audio_streams_;
901 video_stream_info source_video_stream_;
903 std::string afilter_;
904 std::unique_ptr<audio_filter> audio_filter_;
905 std::string vfilter_;
906 std::unique_ptr<filter> video_filter_;
908 spl::unique_ptr<sink> sink_ = spl::make_unique<no_sink_selected>();
909 std::function<core::draw_frame ()> try_pop_frame_;
911 tbb::atomic<bool> started_;
912 tbb::spin_mutex exception_mutex_;
913 boost::exception_ptr exception_;
914 boost::thread thread_;
916 ffmpeg_pipeline_backend_internal()
919 diagnostics::register_graph(graph_);
922 ~ffmpeg_pipeline_backend_internal()
927 void throw_if_error()
929 boost::lock_guard<tbb::spin_mutex> lock(exception_mutex_);
931 if (exception_ != nullptr)
932 boost::rethrow_exception(exception_);
935 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
937 graph_ = std::move(g);
938 source_->graph(graph_);
939 sink_->graph(graph_);
944 void from_file(std::string filename) override
946 source_ = spl::make_unique<file_source>(std::move(filename));
947 try_push_audio_ = std::function<bool (caspar::array<const std::int32_t>)>();
948 try_push_video_ = std::function<bool (caspar::array<const std::uint8_t>)>();
949 source_->graph(graph_);
952 void from_memory_only_audio(int num_channels, int samplerate) override
954 auto source = spl::make_unique<memory_source>();
955 auto source_ptr = source.get();
956 try_push_audio_ = [this, source_ptr](caspar::array<const std::int32_t> data) { return source_ptr->try_push_audio(std::move(data)); };
957 source->enable_audio(samplerate, num_channels);
959 source_ = std::move(source);
960 source_->graph(graph_);
963 void from_memory_only_video(int width, int height, boost::rational<int> framerate) override
965 auto source = spl::make_unique<memory_source>();
966 auto source_ptr = source.get();
967 try_push_video_ = [this, source_ptr](caspar::array<const std::uint8_t> data) { return source_ptr->try_push_video(std::move(data)); };
968 source->enable_video(width, height, std::move(framerate));
970 source_ = std::move(source);
971 source_->graph(graph_);
974 void from_memory(int num_channels, int samplerate, int width, int height, boost::rational<int> framerate) override
976 auto source = spl::make_unique<memory_source>();
977 auto source_ptr = source.get();
978 try_push_audio_ = [this, source_ptr](caspar::array<const std::int32_t> data) { return source_ptr->try_push_audio(std::move(data)); };
979 try_push_video_ = [this, source_ptr](caspar::array<const std::uint8_t> data) { return source_ptr->try_push_video(std::move(data)); };
980 source->enable_audio(samplerate, num_channels);
981 source->enable_video(width, height, std::move(framerate));
983 source_ = std::move(source);
984 source_->graph(graph_);
987 void start_frame(std::uint32_t frame) override { source_->start_frame(frame); }
988 std::uint32_t start_frame() const override { return source_->start_frame(); }
989 void length(std::uint32_t frames) override { source_->length(frames); }
990 std::uint32_t length() const override { return source_->length(); }
991 void seek(std::uint32_t frame) override { source_->seek(frame); }
992 void loop(bool value) override { source_->loop(value); }
993 bool loop() const override { return source_->loop(); }
994 std::string source_filename() const override { return source_->filename(); }
998 void vfilter(std::string filter) override
1000 vfilter_ = std::move(filter);
1003 void afilter(std::string filter) override
1005 afilter_ = std::move(filter);
1008 int width() const override
1010 return source_video_stream_.width;
1013 int height() const override
1015 return source_video_stream_.height;
1018 boost::rational<int> framerate() const override
1020 bool double_rate = filter::is_double_rate(u16(vfilter_));
1022 return double_rate ? source_->framerate() * 2 : source_->framerate();
1025 bool progressive() const override
1032 void to_memory(spl::shared_ptr<core::frame_factory> factory, core::video_format_desc format) override
1034 auto sink = spl::make_unique<memory_sink>(std::move(factory), std::move(format));
1035 auto sink_ptr = sink.get();
1036 try_pop_frame_ = [sink_ptr] { return sink_ptr->try_pop_frame(); };
1038 sink_ = std::move(sink);
1039 sink_->graph(graph_);
1042 void to_file(std::string filename) override
1044 sink_ = spl::make_unique<file_sink>(std::move(filename));
1045 try_pop_frame_ = std::function<core::draw_frame ()>();
1046 sink_->graph(graph_);
1049 void acodec(std::string codec) override { sink_->acodec(std::move(codec)); }
1050 void vcodec(std::string codec) override { sink_->vcodec(std::move(codec)); }
1051 void format(std::string fmt) override { sink_->format(std::move(fmt)); }
1055 void start() override
1058 sink_->start(source_->has_audio(), source_->has_video());
1060 bool quiet = is_logging_quiet_for_thread();
1062 thread_ = boost::thread([=] { run(quiet); });
1065 bool try_push_audio(caspar::array<const std::int32_t> data) override
1069 if (try_push_audio_)
1070 return try_push_audio_(std::move(data));
1075 bool try_push_video(caspar::array<const std::uint8_t> data) override
1079 if (try_push_video_)
1080 return try_push_video_(std::move(data));
1085 core::draw_frame try_pop_frame() override
1089 if (!try_pop_frame_)
1090 CASPAR_THROW_EXCEPTION(invalid_operation());
1092 return try_pop_frame_();
1095 std::uint32_t last_frame() const override
1097 return source_->frame_number();
1100 bool started() const override
1105 void stop() override
1112 if (thread_.joinable())
1117 void run(bool quiet)
1119 ensure_gpf_handler_installed_for_thread(u8(L"ffmpeg-pipeline: " + source_->print() + L" -> " + sink_->print()).c_str());
1120 auto quiet_logging = temporary_enable_quiet_logging_for_thread(quiet);
1124 boost::optional<AVMediaType> result = source_->has_audio() ? AVMediaType::AVMEDIA_TYPE_AUDIO : AVMediaType::AVMEDIA_TYPE_VIDEO;
1126 while (started_ && (source_->has_audio() || source_->has_video()))
1128 auto needed = *result;
1129 auto input_frames_for_streams = source_->get_input_frames_for_streams(needed);
1130 bool flush_all = !input_frames_for_streams.empty() && input_frames_for_streams.at(0) == flush();
1136 if (source_->has_audio() && source_->has_video())
1137 result = needed == AVMediaType::AVMEDIA_TYPE_AUDIO ? AVMediaType::AVMEDIA_TYPE_VIDEO : AVMediaType::AVMEDIA_TYPE_AUDIO;
1142 bool got_requested_media_type = !input_frames_for_streams.empty() && input_frames_for_streams.at(0);
1144 if (got_requested_media_type)
1146 for (int input_stream_index = 0; input_stream_index < input_frames_for_streams.size(); ++input_stream_index)
1148 if (needed == AVMediaType::AVMEDIA_TYPE_AUDIO)
1150 initialize_audio_filter_if_needed(input_frames_for_streams);
1151 audio_filter_->push(input_stream_index, std::move(input_frames_for_streams.at(input_stream_index)));
1153 for (int output_stream_index = 0; output_stream_index < sink_->wanted_num_audio_streams(); ++output_stream_index)
1154 for (auto filtered_frame : audio_filter_->poll_all(output_stream_index))
1155 result = sink_->try_push(AVMediaType::AVMEDIA_TYPE_AUDIO, output_stream_index, std::move(filtered_frame));
1157 else if (needed == AVMediaType::AVMEDIA_TYPE_VIDEO)
1159 initialize_video_filter_if_needed(*input_frames_for_streams.at(input_stream_index));
1160 video_filter_->push(std::move(input_frames_for_streams.at(input_stream_index)));
1162 for (auto filtered_frame : video_filter_->poll_all())
1163 result = sink_->try_push(AVMediaType::AVMEDIA_TYPE_VIDEO, 0, std::move(filtered_frame));
1166 CASPAR_THROW_EXCEPTION(not_supported());
1169 else if (source_->eof())
1176 result = boost::none;
1180 graph_->set_tag(caspar::diagnostics::tag_severity::WARNING, "dropped-frame");
1181 result = needed; // Repeat same media type
1187 if (is_logging_quiet_for_thread())
1189 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
1193 CASPAR_LOG_CURRENT_EXCEPTION();
1196 boost::lock_guard<tbb::spin_mutex> lock(exception_mutex_);
1197 exception_ = boost::current_exception();
1200 video_filter_.reset();
1201 audio_filter_.reset();
1207 template<typename T>
1208 void set_if_changed(bool& changed, T& old_value, T new_value)
1210 if (old_value != new_value)
1213 old_value = new_value;
1217 void initialize_audio_filter_if_needed(const std::vector<std::shared_ptr<AVFrame>>& av_frames_per_stream)
1219 bool changed = av_frames_per_stream.size() != source_audio_streams_.size();
1220 source_audio_streams_.resize(av_frames_per_stream.size());
1222 for (int i = 0; i < av_frames_per_stream.size(); ++i)
1224 auto& av_frame = *av_frames_per_stream.at(i);
1225 auto& stream = source_audio_streams_.at(i);
1227 auto channel_layout = av_frame.channel_layout == 0
1228 ? av_get_default_channel_layout(av_frame.channels)
1229 : av_frame.channel_layout;
1231 set_if_changed(changed, stream.sampleformat, static_cast<AVSampleFormat>(av_frame.format));
1232 set_if_changed(changed, stream.num_channels, av_frame.channels);
1233 set_if_changed(changed, stream.channel_layout, channel_layout);
1237 initialize_audio_filter();
1240 void initialize_audio_filter()
1242 std::vector<audio_input_pad> input_pads;
1243 std::vector<audio_output_pad> output_pads;
1245 for (auto& source_audio_stream : source_audio_streams_)
1247 input_pads.emplace_back(
1248 boost::rational<int>(1, source_->samplerate()),
1249 source_->samplerate(),
1250 source_audio_stream.sampleformat,
1251 source_audio_stream.channel_layout);
1254 auto total_num_channels = cpplinq::from(source_audio_streams_)
1255 .select([](const audio_stream_info& info) { return info.num_channels; })
1256 .aggregate(0, std::plus<int>());
1258 if (total_num_channels > 1 && sink_->wanted_num_audio_streams() > 1)
1259 CASPAR_THROW_EXCEPTION(invalid_operation()
1260 << msg_info("only one-to-many or many-to-one audio stream conversion supported."));
1262 std::wstring amerge;
1264 if (sink_->wanted_num_audio_streams() == 1 && !sink_->wanted_num_channels_per_stream())
1266 output_pads.emplace_back(
1267 sink_->supported_samplerates(),
1268 sink_->supported_sample_formats(),
1269 std::vector<int64_t>({ av_get_default_channel_layout(total_num_channels) }));
1271 if (source_audio_streams_.size() > 1)
1273 for (int i = 0; i < source_audio_streams_.size(); ++i)
1274 amerge += L"[a:" + boost::lexical_cast<std::wstring>(i) + L"]";
1276 amerge += L"amerge=inputs=" + boost::lexical_cast<std::wstring>(source_audio_streams_.size());
1280 std::wstring afilter = u16(afilter_);
1282 if (!amerge.empty())
1284 afilter = prepend_filter(u16(afilter), amerge);
1285 afilter += L"[aout:0]";
1288 audio_filter_.reset(new audio_filter(input_pads, output_pads, u8(afilter)));
1291 void initialize_video_filter_if_needed(const AVFrame& av_frame)
1293 bool changed = false;
1295 set_if_changed(changed, source_video_stream_.width, av_frame.width);
1296 set_if_changed(changed, source_video_stream_.height, av_frame.height);
1297 set_if_changed(changed, source_video_stream_.pixelformat, static_cast<AVPixelFormat>(av_frame.format));
1299 core::field_mode field_mode = core::field_mode::progressive;
1301 if (av_frame.interlaced_frame)
1302 field_mode = av_frame.top_field_first ? core::field_mode::upper : core::field_mode::lower;
1304 set_if_changed(changed, source_video_stream_.fieldmode, field_mode);
1307 initialize_video_filter();
1310 void initialize_video_filter()
1312 if (source_video_stream_.fieldmode != core::field_mode::progressive && !filter::is_deinterlacing(u16(vfilter_)))
1313 vfilter_ = u8(append_filter(u16(vfilter_), L"YADIF=1:-1"));
1315 if (source_video_stream_.height == 480) // NTSC DV
1317 auto pad_str = L"PAD=" + boost::lexical_cast<std::wstring>(source_video_stream_.width) + L":486:0:2:black";
1318 vfilter_ = u8(append_filter(u16(vfilter_), pad_str));
1321 video_filter_.reset(new filter(
1322 source_video_stream_.width,
1323 source_video_stream_.height,
1324 1 / source_->framerate(),
1325 source_->framerate(),
1326 boost::rational<int>(1, 1), // TODO
1327 source_video_stream_.pixelformat,
1328 sink_->supported_pixel_formats(),
1330 sink_->framerate(framerate());
1334 spl::shared_ptr<struct ffmpeg_pipeline_backend> create_internal_pipeline()
1336 return spl::make_shared<ffmpeg_pipeline_backend_internal>();