2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Helge Norberg, helge.norberg@svt.se
24 #include "ffmpeg_pipeline_backend.h"
25 #include "ffmpeg_pipeline_backend_internal.h"
26 #include "producer/input/input.h"
27 #include "producer/video/video_decoder.h"
28 #include "producer/audio/audio_decoder.h"
29 #include "producer/filter/audio_filter.h"
30 #include "producer/filter/filter.h"
31 #include "producer/util/util.h"
32 #include "ffmpeg_error.h"
35 #include <common/diagnostics/graph.h>
36 #include <common/os/general_protection_fault.h>
37 #include <common/enum_class.h>
39 #include <core/frame/audio_channel_layout.h>
40 #include <core/frame/frame.h>
41 #include <core/frame/frame_factory.h>
42 #include <core/video_format.h>
49 #include <tbb/atomic.h>
50 #include <tbb/concurrent_queue.h>
52 #include <boost/thread.hpp>
53 #include <boost/optional.hpp>
55 namespace caspar { namespace ffmpeg {
57 std::string to_string(const boost::rational<int>& framerate)
59 return boost::lexical_cast<std::string>(framerate.numerator())
60 + "/" + boost::lexical_cast<std::string>(framerate.denominator()) + " (" + boost::lexical_cast<std::string>(static_cast<double>(framerate.numerator()) / static_cast<double>(framerate.denominator())) + ") fps";
63 std::vector<int> find_audio_cadence(const boost::rational<int>& framerate)
65 static std::map<boost::rational<int>, std::vector<int>> CADENCES_BY_FRAMERATE = []
67 std::map<boost::rational<int>, std::vector<int>> result;
69 for (core::video_format format : enum_constants<core::video_format>())
71 core::video_format_desc desc(format);
72 boost::rational<int> format_rate(desc.time_scale, desc.duration);
74 result.insert(std::make_pair(format_rate, desc.audio_cadence));
80 auto exact_match = CADENCES_BY_FRAMERATE.find(framerate);
82 if (exact_match != CADENCES_BY_FRAMERATE.end())
83 return exact_match->second;
85 boost::rational<int> closest_framerate_diff = std::numeric_limits<int>::max();
86 boost::rational<int> closest_framerate = 0;
88 for (auto format_framerate : CADENCES_BY_FRAMERATE | boost::adaptors::map_keys)
90 auto diff = boost::abs(framerate - format_framerate);
92 if (diff < closest_framerate_diff)
94 closest_framerate_diff = diff;
95 closest_framerate = format_framerate;
99 if (is_logging_quiet_for_thread())
100 CASPAR_LOG(debug) << "No exact audio cadence match found for framerate " << to_string(framerate)
101 << "\nClosest match is " << to_string(closest_framerate)
102 << "\nwhich is a " << to_string(closest_framerate_diff) << " difference.";
104 CASPAR_LOG(warning) << "No exact audio cadence match found for framerate " << to_string(framerate)
105 << "\nClosest match is " << to_string(closest_framerate)
106 << "\nwhich is a " << to_string(closest_framerate_diff) << " difference.";
108 return CADENCES_BY_FRAMERATE[closest_framerate];
113 virtual ~source() { }
115 virtual std::wstring print() const = 0;
116 virtual void start() { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
117 virtual void graph(spl::shared_ptr<caspar::diagnostics::graph> g) { }
118 virtual void stop() { }
119 virtual void start_frame(std::uint32_t frame) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
120 virtual std::uint32_t start_frame() const { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
121 virtual void loop(bool value) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
122 virtual bool loop() const { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
123 virtual void length(std::uint32_t frames) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
124 virtual std::uint32_t length() const { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
125 virtual std::string filename() const { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print())); }
126 virtual void seek(std::uint32_t frame) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not seekable.")); }
127 virtual bool has_audio() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
128 virtual int samplerate() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
129 virtual bool has_video() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
130 virtual bool eof() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
131 virtual boost::rational<int> framerate() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
132 virtual std::uint32_t frame_number() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
133 virtual std::shared_ptr<AVFrame> get_input_frame(AVMediaType type) { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
136 struct no_source_selected : public source
138 std::wstring print() const override
140 return L"[no_source_selected]";
144 class file_source : public source
146 std::wstring filename_;
147 spl::shared_ptr<diagnostics::graph> graph_;
148 std::uint32_t start_frame_ = 0;
149 std::uint32_t length_ = std::numeric_limits<std::uint32_t>::max();
151 mutable boost::mutex pointer_mutex_;
152 std::shared_ptr<input> input_;
153 std::shared_ptr<audio_decoder> audio_decoder_;
154 std::shared_ptr<video_decoder> video_decoder_;
155 bool started_ = false;
157 file_source(std::string filename)
158 : filename_(u16(filename))
162 std::wstring print() const override
164 return L"[file_source " + filename_ + L"]";
167 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
169 graph_ = std::move(g);
172 void start() override
174 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
175 bool thumbnail_mode = is_logging_quiet_for_thread();
176 input_.reset(new input(graph_, filename_, loop_, start_frame_, length_, thumbnail_mode));
180 audio_decoder_.reset(new audio_decoder(*input_, core::video_format_desc()));
182 catch (averror_stream_not_found&)
184 CASPAR_LOG(debug) << print() << " No audio-stream found. Running without audio.";
188 if (is_logging_quiet_for_thread())
190 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
191 CASPAR_LOG(info) << print() << " Failed to open audio-stream. Running without audio. Turn on log level debug to see more information.";
195 CASPAR_LOG_CURRENT_EXCEPTION();
196 CASPAR_LOG(warning) << print() << " Failed to open audio-stream. Running without audio.";
202 video_decoder_.reset(new video_decoder(*input_, false));
204 catch (averror_stream_not_found&)
206 CASPAR_LOG(debug) << print() << " No video-stream found. Running without video.";
210 if (is_logging_quiet_for_thread())
212 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
213 CASPAR_LOG(info) << print() << " Failed to open video-stream. Running without audio. Turn on log level debug to see more information.";
217 CASPAR_LOG_CURRENT_EXCEPTION();
218 CASPAR_LOG(warning) << print() << " Failed to open video-stream. Running without audio.";
230 void start_frame(std::uint32_t frame) override
232 start_frame_ = frame;
234 auto i = get_input();
239 std::uint32_t start_frame() const override
244 void loop(bool value) override
248 auto i = get_input();
253 bool loop() const override
258 void length(std::uint32_t frames) override
262 auto i = get_input();
267 std::uint32_t length() const override
269 auto v = get_video_decoder();
272 return v->nb_frames();
274 auto a = get_audio_decoder();
277 return a->nb_frames();
282 std::string filename() const override
284 return u8(filename_);
287 void seek(std::uint32_t frame) override
290 get_input()->seek(frame);
293 bool eof() const override
295 auto i = get_input();
296 return !i || i->eof();
299 bool has_audio() const override
301 return static_cast<bool>(get_audio_decoder());
304 int samplerate() const override
306 auto decoder = get_audio_decoder();
314 bool has_video() const override
316 return static_cast<bool>(get_video_decoder());
319 boost::rational<int> framerate() const override
321 auto decoder = get_video_decoder();
326 return decoder->framerate();
329 std::uint32_t frame_number() const override
331 auto decoder = get_video_decoder();
336 return decoder->file_frame_number();
339 std::shared_ptr<AVFrame> get_input_frame(AVMediaType type) override
341 auto a_decoder = get_audio_decoder();
342 auto v_decoder = get_video_decoder();
345 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && a_decoder)
347 std::shared_ptr<AVFrame> frame;
349 for (int i = 0; i < 64; ++i)
351 frame = (*a_decoder)();
353 if (frame && frame->data[0])
354 return spl::make_shared_ptr(frame);
357 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO && v_decoder)
359 std::shared_ptr<AVFrame> frame;
361 for (int i = 0; i < 128; ++i)
363 frame = (*v_decoder)();
365 if (frame && frame->data[0])
366 return spl::make_shared_ptr(frame);
370 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(
371 print() + L" Unhandled media type " + boost::lexical_cast<std::wstring>(type)));
376 void expect_started() const
379 CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" Not started."));
382 std::shared_ptr<input> get_input() const
384 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
388 std::shared_ptr<audio_decoder> get_audio_decoder() const
390 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
391 return audio_decoder_;
394 std::shared_ptr<video_decoder> get_video_decoder() const
396 boost::lock_guard<boost::mutex> lock(pointer_mutex_);
397 return video_decoder_;
401 class memory_source : public source
403 int samplerate_ = -1;
404 int num_channels_ = -1;
407 boost::rational<int> framerate_ = -1;
409 tbb::atomic<bool> running_;
410 tbb::concurrent_bounded_queue<caspar::array<const int32_t>> audio_frames_;
411 tbb::concurrent_bounded_queue<caspar::array<const uint8_t>> video_frames_;
412 int64_t audio_pts_ = 0;
413 int64_t video_pts_ = 0;
418 video_frames_.set_capacity(1);
419 audio_frames_.set_capacity(1);
427 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
431 std::wstring print() const override
433 return L"[memory_source]";
436 void enable_audio(int samplerate, int num_channels)
438 samplerate_ = samplerate;
439 num_channels_ = num_channels;
442 void enable_video(int width, int height, boost::rational<int> framerate)
448 void start() override
456 video_frames_.try_push(caspar::array<const uint8_t>());
457 audio_frames_.try_push(caspar::array<const int32_t>());
460 bool has_audio() const override
462 return samplerate_ != -1;
465 int samplerate() const override
470 bool has_video() const override
475 bool eof() const override
480 boost::rational<int> framerate() const override
485 bool try_push_audio(caspar::array<const std::int32_t> data)
488 CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" audio not enabled."));
490 if (data.empty() || data.size() % num_channels_ != 0)
491 CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info(print() + L" audio with incorrect number of channels submitted."));
493 return audio_frames_.try_push(std::move(data));
496 bool try_push_video(caspar::array<const std::uint8_t> data)
499 CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" video not enabled."));
501 if (data.size() != width_ * height_ * 4)
502 CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info(print() + L" video with incorrect size submitted."));
504 return video_frames_.try_push(std::move(data));
507 std::shared_ptr<AVFrame> get_input_frame(AVMediaType type) override
510 CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not running."));
512 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && has_audio())
514 caspar::array<const std::int32_t> samples;
515 audio_frames_.pop(samples);
520 spl::shared_ptr<AVFrame> av_frame(av_frame_alloc(), [samples](AVFrame* p) { av_frame_free(&p); });
522 av_frame->channels = num_channels_;
523 av_frame->channel_layout = av_get_default_channel_layout(num_channels_);
524 av_frame->sample_rate = samplerate_;
525 av_frame->nb_samples = static_cast<int>(samples.size()) / num_channels_;
526 av_frame->format = AV_SAMPLE_FMT_S32;
527 av_frame->pts = audio_pts_;
529 audio_pts_ += av_frame->nb_samples;
531 FF(av_samples_fill_arrays(
532 av_frame->extended_data,
534 reinterpret_cast<const std::uint8_t*>(&*samples.begin()),
536 av_frame->nb_samples,
537 static_cast<AVSampleFormat>(av_frame->format),
542 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO && has_video())
544 caspar::array<const std::uint8_t> data;
545 video_frames_.pop(data);
550 spl::shared_ptr<AVFrame> av_frame(av_frame_alloc(), [data](AVFrame* p) { av_frame_free(&p); });
551 avcodec_get_frame_defaults(av_frame.get());
553 const auto sample_aspect_ratio = boost::rational<int>(width_, height_);
555 av_frame->format = AV_PIX_FMT_BGRA;
556 av_frame->width = width_;
557 av_frame->height = height_;
558 av_frame->sample_aspect_ratio.num = sample_aspect_ratio.numerator();
559 av_frame->sample_aspect_ratio.den = sample_aspect_ratio.denominator();
560 av_frame->pts = video_pts_;
564 FF(av_image_fill_arrays(
568 static_cast<AVPixelFormat>(av_frame->format),
576 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(
577 print() + L" Unhandled media type " + boost::lexical_cast<std::wstring>(type)));
585 virtual std::wstring print() const = 0;
586 virtual void graph(spl::shared_ptr<caspar::diagnostics::graph> g) { }
587 virtual void acodec(std::string codec) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
588 virtual void vcodec(std::string codec) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
589 virtual void format(std::string fmt) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
590 virtual void framerate(boost::rational<int> framerate) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); }
591 virtual void start(bool has_audio, bool has_video) { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
592 virtual void stop() { }
593 virtual std::vector<AVSampleFormat> supported_sample_formats() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
594 virtual std::vector<int> supported_samplerates() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
595 virtual std::vector<AVPixelFormat> supported_pixel_formats() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
596 virtual boost::optional<AVMediaType> try_push(AVMediaType type, spl::shared_ptr<AVFrame> frame) { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
597 virtual void eof() { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); }
600 struct no_sink_selected : public sink
602 std::wstring print() const override
604 return L"[no_sink_selected]";
608 class file_sink : public sink
610 std::wstring filename_;
611 spl::shared_ptr<diagnostics::graph> graph_;
613 file_sink(std::string filename)
614 : filename_(u16(std::move(filename)))
618 std::wstring print() const override
620 return L"[file_sink " + filename_ + L"]";
623 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
625 graph_ = std::move(g);
629 class memory_sink : public sink
631 spl::shared_ptr<core::frame_factory> factory_;
633 bool has_audio_ = false;
634 bool has_video_ = false;
635 std::vector<int> audio_cadence_;
636 core::audio_channel_layout channel_layout_ = core::audio_channel_layout::invalid();
637 core::mutable_audio_buffer audio_samples_;
639 std::queue<std::shared_ptr<AVFrame>> video_frames_;
641 tbb::concurrent_bounded_queue<core::draw_frame> output_frames_;
642 tbb::atomic<bool> running_;
644 memory_sink(spl::shared_ptr<core::frame_factory> factory, core::video_format_desc format)
645 : factory_(std::move(factory))
646 , audio_cadence_(format.audio_cadence)
648 output_frames_.set_capacity(2);
650 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
651 // This cadence fills the audio mixer most optimally.
652 boost::range::rotate(audio_cadence_, std::end(audio_cadence_) - 1);
660 std::wstring print() const override
662 return L"[memory_sink]";
665 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
669 void framerate(boost::rational<int> framerate) override
671 audio_cadence_ = find_audio_cadence(framerate);
672 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
673 // This cadence fills the audio mixer most optimally.
674 boost::range::rotate(audio_cadence_, std::end(audio_cadence_) - 1);
677 void start(bool has_audio, bool has_video) override
679 has_audio_ = has_audio;
680 has_video_ = has_video;
691 std::vector<AVSampleFormat> supported_sample_formats() const override
693 return { AVSampleFormat::AV_SAMPLE_FMT_S32 };
696 std::vector<int> supported_samplerates() const override {
700 std::vector<AVPixelFormat> supported_pixel_formats() const override
703 AVPixelFormat::AV_PIX_FMT_YUVA420P,
704 AVPixelFormat::AV_PIX_FMT_YUV444P,
705 AVPixelFormat::AV_PIX_FMT_YUV422P,
706 AVPixelFormat::AV_PIX_FMT_YUV420P,
707 AVPixelFormat::AV_PIX_FMT_YUV411P,
708 AVPixelFormat::AV_PIX_FMT_BGRA,
709 AVPixelFormat::AV_PIX_FMT_ARGB,
710 AVPixelFormat::AV_PIX_FMT_RGBA,
711 AVPixelFormat::AV_PIX_FMT_ABGR,
712 AVPixelFormat::AV_PIX_FMT_GRAY8
716 boost::optional<AVMediaType> try_push(AVMediaType type, spl::shared_ptr<AVFrame> av_frame) override
718 if (!has_audio_ && !has_video_)
719 CASPAR_THROW_EXCEPTION(invalid_operation());
721 if (type == AVMediaType::AVMEDIA_TYPE_AUDIO && av_frame->data[0])
723 if (channel_layout_ == core::audio_channel_layout::invalid()) // First audio
725 channel_layout_ = get_audio_channel_layout(av_frame->channels, av_frame->channel_layout, L"");
727 // Insert silence samples so that the audio mixer is guaranteed to be filled.
728 auto min_num_samples_per_frame = *boost::min_element(audio_cadence_);
729 auto max_num_samples_per_frame = *boost::max_element(audio_cadence_);
730 auto cadence_safety_samples = max_num_samples_per_frame - min_num_samples_per_frame;
731 audio_samples_.resize(channel_layout_.num_channels * cadence_safety_samples, 0);
734 auto ptr = reinterpret_cast<int32_t*>(av_frame->data[0]);
736 audio_samples_.insert(audio_samples_.end(), ptr, ptr + av_frame->linesize[0] / sizeof(int32_t));
738 else if (type == AVMediaType::AVMEDIA_TYPE_VIDEO)
740 video_frames_.push(std::move(av_frame));
747 (channel_layout_ != core::audio_channel_layout::invalid() && audio_samples_.size() >= audio_cadence_.front() * channel_layout_.num_channels);
750 !video_frames_.empty();
753 return AVMediaType::AVMEDIA_TYPE_AUDIO;
756 return AVMediaType::AVMEDIA_TYPE_VIDEO;
758 core::mutable_audio_buffer audio_data;
762 auto begin = audio_samples_.begin();
763 auto end = begin + audio_cadence_.front() * channel_layout_.num_channels;
765 audio_data.insert(audio_data.begin(), begin, end);
766 audio_samples_.erase(begin, end);
767 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_) + 1);
770 if (!has_video_) // Audio only
772 core::mutable_frame audio_only_frame(
774 std::move(audio_data),
776 core::pixel_format_desc(core::pixel_format::invalid),
779 output_frames_.push(core::draw_frame(std::move(audio_only_frame)));
781 return AVMediaType::AVMEDIA_TYPE_AUDIO;
784 auto output_frame = make_frame(this, spl::make_shared_ptr(video_frames_.front()), *factory_, channel_layout_);
786 output_frame.audio_data() = std::move(audio_data);
788 output_frames_.push(core::draw_frame(std::move(output_frame)));
794 // Drain rest, regardless of it being enough or not.
795 while (!video_frames_.empty() || !audio_samples_.empty())
797 core::mutable_audio_buffer audio_data;
799 audio_data.swap(audio_samples_);
801 if (!video_frames_.empty())
803 auto output_frame = make_frame(this, spl::make_shared_ptr(video_frames_.front()), *factory_, channel_layout_);
805 output_frame.audio_data() = std::move(audio_data);
807 output_frames_.push(core::draw_frame(std::move(output_frame)));
811 core::mutable_frame audio_only_frame(
813 std::move(audio_data),
815 core::pixel_format_desc(core::pixel_format::invalid),
818 output_frames_.push(core::draw_frame(std::move(audio_only_frame)));
819 output_frames_.push(core::draw_frame::empty());
824 core::draw_frame try_pop_frame()
826 core::draw_frame frame = core::draw_frame::late();
828 if (!output_frames_.try_pop(frame) && !running_)
829 return core::draw_frame::empty();
835 class ffmpeg_pipeline_backend_internal : public ffmpeg_pipeline_backend
837 spl::shared_ptr<diagnostics::graph> graph_;
839 spl::unique_ptr<source> source_ = spl::make_unique<no_source_selected>();
840 std::function<bool (caspar::array<const std::int32_t> data)> try_push_audio_;
841 std::function<bool (caspar::array<const std::uint8_t> data)> try_push_video_;
843 int source_num_channels_ = 0;
844 AVSampleFormat source_sampleformat_ = AVSampleFormat::AV_SAMPLE_FMT_NONE;
845 int source_width_ = 0;
846 int source_height_ = 0;
847 AVPixelFormat source_pixelformat_ = AVPixelFormat::AV_PIX_FMT_NONE;
848 core::field_mode source_fieldmode_ = core::field_mode::progressive;
850 std::string afilter_;
851 std::unique_ptr<audio_filter> audio_filter_;
852 std::string vfilter_;
853 std::unique_ptr<filter> video_filter_;
855 spl::unique_ptr<sink> sink_ = spl::make_unique<no_sink_selected>();
856 std::function<core::draw_frame ()> try_pop_frame_;
858 tbb::atomic<bool> started_;
859 boost::thread thread_;
861 ffmpeg_pipeline_backend_internal()
864 diagnostics::register_graph(graph_);
867 ~ffmpeg_pipeline_backend_internal()
872 void graph(spl::shared_ptr<caspar::diagnostics::graph> g) override
874 graph_ = std::move(g);
875 source_->graph(graph_);
876 sink_->graph(graph_);
881 void from_file(std::string filename) override
883 source_ = spl::make_unique<file_source>(std::move(filename));
884 try_push_audio_ = std::function<bool (caspar::array<const std::int32_t>)>();
885 try_push_video_ = std::function<bool (caspar::array<const std::uint8_t>)>();
886 source_->graph(graph_);
889 void from_memory_only_audio(int num_channels, int samplerate) override
891 auto source = spl::make_unique<memory_source>();
892 auto source_ptr = source.get();
893 try_push_audio_ = [this, source_ptr](caspar::array<const std::int32_t> data) { return source_ptr->try_push_audio(std::move(data)); };
894 source->enable_audio(samplerate, num_channels);
896 source_ = std::move(source);
897 source_->graph(graph_);
900 void from_memory_only_video(int width, int height, boost::rational<int> framerate) override
902 auto source = spl::make_unique<memory_source>();
903 auto source_ptr = source.get();
904 try_push_video_ = [this, source_ptr](caspar::array<const std::uint8_t> data) { return source_ptr->try_push_video(std::move(data)); };
905 source->enable_video(width, height, std::move(framerate));
907 source_ = std::move(source);
908 source_->graph(graph_);
911 void from_memory(int num_channels, int samplerate, int width, int height, boost::rational<int> framerate) override
913 auto source = spl::make_unique<memory_source>();
914 auto source_ptr = source.get();
915 try_push_audio_ = [this, source_ptr](caspar::array<const std::int32_t> data) { return source_ptr->try_push_audio(std::move(data)); };
916 try_push_video_ = [this, source_ptr](caspar::array<const std::uint8_t> data) { return source_ptr->try_push_video(std::move(data)); };
917 source->enable_audio(samplerate, num_channels);
918 source->enable_video(width, height, std::move(framerate));
920 source_ = std::move(source);
921 source_->graph(graph_);
924 void start_frame(std::uint32_t frame) override { source_->start_frame(frame); }
925 std::uint32_t start_frame() const override { return source_->start_frame(); }
926 void length(std::uint32_t frames) override { source_->length(frames); }
927 std::uint32_t length() const override { return source_->length(); }
928 void seek(std::uint32_t frame) override { source_->seek(frame); }
929 void loop(bool value) override { source_->loop(value); }
930 bool loop() const override { return source_->loop(); }
931 std::string source_filename() const override { return source_->filename(); }
935 void vfilter(std::string filter) override
937 vfilter_ = std::move(filter);
940 void afilter(std::string filter) override
942 afilter_ = std::move(filter);
945 int width() const override
947 return source_width_;
950 int height() const override
952 return source_height_;
955 boost::rational<int> framerate() const override
957 bool double_rate = filter::is_double_rate(u16(vfilter_));
959 return double_rate ? source_->framerate() * 2 : source_->framerate();
962 bool progressive() const override
969 void to_memory(spl::shared_ptr<core::frame_factory> factory, core::video_format_desc format) override
971 auto sink = spl::make_unique<memory_sink>(std::move(factory), std::move(format));
972 auto sink_ptr = sink.get();
973 try_pop_frame_ = [sink_ptr] { return sink_ptr->try_pop_frame(); };
975 sink_ = std::move(sink);
976 sink_->graph(graph_);
979 void to_file(std::string filename) override
981 sink_ = spl::make_unique<file_sink>(std::move(filename));
982 try_pop_frame_ = std::function<core::draw_frame ()>();
983 sink_->graph(graph_);
986 void acodec(std::string codec) override { sink_->acodec(std::move(codec)); }
987 void vcodec(std::string codec) override { sink_->vcodec(std::move(codec)); }
988 void format(std::string fmt) override { sink_->format(std::move(fmt)); }
992 void start() override
995 sink_->start(source_->has_audio(), source_->has_video());
997 bool quiet = is_logging_quiet_for_thread();
999 thread_ = boost::thread([=] { run(quiet); });
1002 bool try_push_audio(caspar::array<const std::int32_t> data) override
1004 if (try_push_audio_)
1005 return try_push_audio_(std::move(data));
1010 bool try_push_video(caspar::array<const std::uint8_t> data) override
1012 if (try_push_video_)
1013 return try_push_video_(std::move(data));
1018 core::draw_frame try_pop_frame() override
1020 if (!try_pop_frame_)
1021 CASPAR_THROW_EXCEPTION(invalid_operation());
1023 return try_pop_frame_();
1026 std::uint32_t last_frame() const override
1028 return source_->frame_number();
1031 bool started() const override
1036 void stop() override
1043 if (thread_.joinable())
1048 void run(bool quiet)
1050 ensure_gpf_handler_installed_for_thread(u8(L"ffmpeg-pipeline: " + source_->print() + L" -> " + sink_->print()).c_str());
1051 auto quiet_logging = temporary_enable_quiet_logging_for_thread(quiet);
1055 boost::optional<AVMediaType> result = source_->has_audio() ? AVMediaType::AVMEDIA_TYPE_AUDIO : AVMediaType::AVMEDIA_TYPE_VIDEO;
1057 while (started_ && (source_->has_audio() || source_->has_video()))
1059 auto needed = *result;
1060 auto input_frame = source_->get_input_frame(needed);
1064 if (needed == AVMediaType::AVMEDIA_TYPE_AUDIO)
1066 result = sink_->try_push(AVMediaType::AVMEDIA_TYPE_AUDIO, spl::make_shared_ptr(std::move(input_frame)));
1068 else if (needed == AVMediaType::AVMEDIA_TYPE_VIDEO)
1070 initialize_video_filter_if_needed(*input_frame);
1071 video_filter_->push(std::move(input_frame));
1073 for (auto filtered_frame : video_filter_->poll_all())
1075 result = sink_->try_push(AVMediaType::AVMEDIA_TYPE_VIDEO, std::move(filtered_frame));
1079 CASPAR_THROW_EXCEPTION(not_supported());
1081 else if (source_->eof())
1088 result = boost::none;
1092 graph_->set_tag(caspar::diagnostics::tag_severity::WARNING, "dropped-frame");
1093 result = needed; // Repeat same media type
1099 if (is_logging_quiet_for_thread())
1101 CASPAR_LOG_CURRENT_EXCEPTION_AT_LEVEL(debug);
1105 CASPAR_LOG_CURRENT_EXCEPTION();
1109 video_filter_.reset();
1110 audio_filter_.reset();
1116 template<typename T>
1117 void set_if_changed(bool& changed, T& old_value, T new_value)
1119 if (old_value != new_value)
1122 old_value = new_value;
1126 void initialize_audio_filter_if_needed(const AVFrame& av_frame)
1128 bool changed = false;
1130 set_if_changed(changed, source_sampleformat_, static_cast<AVSampleFormat>(av_frame.format));
1131 set_if_changed(changed, source_num_channels_, av_frame.channels);
1134 initialize_audio_filter();
1137 void initialize_audio_filter()
1139 audio_filter_.reset(new audio_filter(
1140 boost::rational<int>(1, source_->samplerate()),
1141 source_->samplerate(),
1142 source_sampleformat_,
1143 av_get_default_channel_layout(source_num_channels_),
1144 sink_->supported_samplerates(),
1145 sink_->supported_sample_formats(),
1150 void initialize_video_filter_if_needed(const AVFrame& av_frame)
1152 bool changed = false;
1154 set_if_changed(changed, source_width_, av_frame.width);
1155 set_if_changed(changed, source_height_, av_frame.height);
1156 set_if_changed(changed, source_pixelformat_, static_cast<AVPixelFormat>(av_frame.format));
1158 core::field_mode field_mode = core::field_mode::progressive;
1160 if (av_frame.interlaced_frame)
1161 field_mode = av_frame.top_field_first ? core::field_mode::upper : core::field_mode::lower;
1163 set_if_changed(changed, source_fieldmode_, field_mode);
1166 initialize_video_filter();
1169 void initialize_video_filter()
1171 if (source_fieldmode_ != core::field_mode::progressive && !filter::is_deinterlacing(u16(vfilter_)))
1172 vfilter_ = u8(append_filter(u16(vfilter_), L"YADIF=1:-1"));
1174 if (source_height_ == 480) // NTSC DV
1176 auto pad_str = L"PAD=" + boost::lexical_cast<std::wstring>(source_width_) + L":486:0:2:black";
1177 vfilter_ = u8(append_filter(u16(vfilter_), pad_str));
1180 video_filter_.reset(new filter(
1183 1 / source_->framerate(),
1184 source_->framerate(),
1185 boost::rational<int>(1, 1), // TODO
1186 source_pixelformat_,
1187 sink_->supported_pixel_formats(),
1189 sink_->framerate(framerate());
1193 spl::shared_ptr<struct ffmpeg_pipeline_backend> create_internal_pipeline()
1195 return spl::make_shared<ffmpeg_pipeline_backend_internal>();