2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Robert Nagy, ronag89@gmail.com
22 #include "../../StdAfx.h"
24 #include "frame_muxer.h"
26 #include "../filter/filter.h"
27 #include "../util/util.h"
28 #include "../../ffmpeg.h"
30 #include <core/producer/frame_producer.h>
31 #include <core/frame/draw_frame.h>
32 #include <core/frame/frame_transform.h>
33 #include <core/frame/pixel_format.h>
34 #include <core/frame/frame_factory.h>
35 #include <core/frame/frame.h>
36 #include <core/frame/audio_channel_layout.h>
38 #include <common/env.h>
39 #include <common/except.h>
40 #include <common/log.h>
43 #pragma warning (push)
44 #pragma warning (disable : 4244)
48 #define __STDC_CONSTANT_MACROS
49 #define __STDC_LIMIT_MACROS
50 #include <libavcodec/avcodec.h>
51 #include <libavformat/avformat.h>
57 #include <common/assert.h>
58 #include <boost/range/algorithm_ext/push_back.hpp>
59 #include <boost/algorithm/string/predicate.hpp>
60 #include <boost/thread/mutex.hpp>
61 #include <boost/optional.hpp>
67 using namespace caspar::core;
69 namespace caspar { namespace ffmpeg {
70 struct av_frame_format
73 std::array<int, AV_NUM_DATA_POINTERS> line_sizes;
77 av_frame_format(const AVFrame& frame)
78 : pix_format(frame.format)
80 , height(frame.height)
82 boost::copy(frame.linesize, line_sizes.begin());
85 bool operator==(const av_frame_format& other) const
87 return pix_format == other.pix_format
88 && line_sizes == other.line_sizes
89 && width == other.width
90 && height == other.height;
93 bool operator!=(const av_frame_format& other) const
95 return !(*this == other);
99 struct frame_muxer::impl : boost::noncopyable
101 std::queue<std::queue<core::mutable_frame>> video_streams_;
102 std::queue<core::mutable_audio_buffer> audio_streams_;
103 std::queue<core::draw_frame> frame_buffer_;
104 display_mode display_mode_ = display_mode::invalid;
105 const boost::rational<int> in_framerate_;
106 const video_format_desc format_desc_;
107 const audio_channel_layout audio_channel_layout_;
109 std::vector<int> audio_cadence_ = format_desc_.audio_cadence;
111 spl::shared_ptr<core::frame_factory> frame_factory_;
112 boost::optional<av_frame_format> previously_filtered_frame_;
114 std::unique_ptr<filter> filter_;
115 const std::wstring filter_str_;
116 const bool multithreaded_filter_;
117 bool force_deinterlacing_ = env::properties().get(L"configuration.force-deinterlace", false);
119 mutable boost::mutex out_framerate_mutex_;
120 boost::rational<int> out_framerate_;
123 boost::rational<int> in_framerate,
124 const spl::shared_ptr<core::frame_factory>& frame_factory,
125 const core::video_format_desc& format_desc,
126 const core::audio_channel_layout& channel_layout,
127 const std::wstring& filter_str,
128 bool multithreaded_filter)
129 : in_framerate_(in_framerate)
130 , format_desc_(format_desc)
131 , audio_channel_layout_(channel_layout)
132 , frame_factory_(frame_factory)
133 , filter_str_(filter_str)
134 , multithreaded_filter_(multithreaded_filter)
136 video_streams_.push(std::queue<core::mutable_frame>());
137 audio_streams_.push(core::mutable_audio_buffer());
139 set_out_framerate(in_framerate_);
142 void push(const std::shared_ptr<AVFrame>& video_frame)
147 av_frame_format current_frame_format(*video_frame);
149 if (previously_filtered_frame_ && video_frame->data[0] && *previously_filtered_frame_ != current_frame_format)
151 // Fixes bug where avfilter crashes server on some DV files (starts in YUV420p but changes to YUV411p after the first frame).
152 if (ffmpeg::is_logging_quiet_for_thread())
153 CASPAR_LOG(debug) << L"[frame_muxer] Frame format has changed. Resetting display mode.";
155 CASPAR_LOG(info) << L"[frame_muxer] Frame format has changed. Resetting display mode.";
157 display_mode_ = display_mode::invalid;
159 previously_filtered_frame_ = boost::none;
162 if (video_frame == flush_video())
164 video_streams_.push(std::queue<core::mutable_frame>());
166 else if (video_frame == empty_video())
168 video_streams_.back().push(frame_factory_->create_frame(this, core::pixel_format::invalid, audio_channel_layout_));
169 display_mode_ = display_mode::simple;
173 if (!filter_ || display_mode_ == display_mode::invalid)
174 update_display_mode(video_frame);
178 filter_->push(video_frame);
179 previously_filtered_frame_ = current_frame_format;
181 for (auto& av_frame : filter_->poll_all())
182 video_streams_.back().push(make_frame(this, av_frame, *frame_factory_, audio_channel_layout_));
186 if (video_streams_.back().size() > 32)
187 CASPAR_THROW_EXCEPTION(invalid_operation() << source_info("frame_muxer") << msg_info("video-stream overflow. This can be caused by incorrect frame-rate. Check clip meta-data."));
190 void push(const std::shared_ptr<core::mutable_audio_buffer>& audio)
195 if (audio == flush_audio())
197 audio_streams_.push(core::mutable_audio_buffer());
199 else if (audio == empty_audio())
201 boost::range::push_back(audio_streams_.back(), core::mutable_audio_buffer(audio_cadence_.front() * audio_channel_layout_.num_channels, 0));
205 boost::range::push_back(audio_streams_.back(), *audio);
208 if (audio_streams_.back().size() > 32 * audio_cadence_.front() * audio_channel_layout_.num_channels)
209 BOOST_THROW_EXCEPTION(invalid_operation() << source_info("frame_muxer") << msg_info("audio-stream overflow. This can be caused by incorrect frame-rate. Check clip meta-data."));
212 bool video_ready() const
214 return video_streams_.size() > 1 || (video_streams_.size() >= audio_streams_.size() && video_ready2());
217 bool audio_ready() const
219 return audio_streams_.size() > 1 || (audio_streams_.size() >= video_streams_.size() && audio_ready2());
222 bool video_ready2() const
224 return video_streams_.front().size() >= 1;
227 bool audio_ready2() const
229 return audio_streams_.front().size() >= audio_cadence_.front() * audio_channel_layout_.num_channels;
232 core::draw_frame poll()
234 if (!frame_buffer_.empty())
236 auto frame = frame_buffer_.front();
241 if (video_streams_.size() > 1 && audio_streams_.size() > 1 && (!video_ready2() || !audio_ready2()))
243 if (!video_streams_.front().empty() || !audio_streams_.front().empty())
244 CASPAR_LOG(trace) << "Truncating: " << video_streams_.front().size() << L" video-frames, " << audio_streams_.front().size() << L" audio-samples.";
246 video_streams_.pop();
247 audio_streams_.pop();
250 if (!video_ready2() || !audio_ready2() || display_mode_ == display_mode::invalid)
251 return core::draw_frame::empty();
253 auto frame = pop_video();
254 frame.audio_data() = pop_audio();
256 frame_buffer_.push(core::draw_frame(std::move(frame)));
258 return frame_buffer_.empty() ? core::draw_frame::empty() : poll();
261 core::mutable_frame pop_video()
263 auto frame = std::move(video_streams_.front().front());
264 video_streams_.front().pop();
268 core::mutable_audio_buffer pop_audio()
270 CASPAR_VERIFY(audio_streams_.front().size() >= audio_cadence_.front() * audio_channel_layout_.num_channels);
272 auto begin = audio_streams_.front().begin();
273 auto end = begin + (audio_cadence_.front() * audio_channel_layout_.num_channels);
275 core::mutable_audio_buffer samples(begin, end);
276 audio_streams_.front().erase(begin, end);
278 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_) + 1);
283 uint32_t calc_nb_frames(uint32_t nb_frames) const
285 uint64_t nb_frames2 = nb_frames;
287 if(filter_ && filter_->is_double_rate()) // Take into account transformations in filter.
290 return static_cast<uint32_t>(nb_frames2);
293 boost::rational<int> out_framerate() const
295 boost::lock_guard<boost::mutex> lock(out_framerate_mutex_);
297 return out_framerate_;
300 void update_display_mode(const std::shared_ptr<AVFrame>& frame)
302 std::wstring filter_str = filter_str_;
304 display_mode_ = display_mode::simple;
306 auto mode = get_mode(*frame);
307 if (mode == core::field_mode::progressive && frame->height < 720 && in_framerate_ < 50) // SD frames are interlaced. Probably incorrect meta-data. Fix it.
308 mode = core::field_mode::upper;
310 if (filter::is_deinterlacing(filter_str_))
312 display_mode_ = display_mode::simple;
314 else if (mode != core::field_mode::progressive)
316 if (force_deinterlacing_)
318 display_mode_ = display_mode::deinterlace_bob;
322 bool output_also_interlaced = format_desc_.field_mode != core::field_mode::progressive;
323 bool interlaced_output_compatible =
324 output_also_interlaced
326 (frame->height == 480 && format_desc_.height == 486) // don't deinterlace for NTSC DV
327 || frame->height == format_desc_.height
329 && in_framerate_ == format_desc_.framerate;
331 display_mode_ = interlaced_output_compatible ? display_mode::simple : display_mode::deinterlace_bob;
335 if (display_mode_ == display_mode::deinterlace_bob)
336 filter_str = append_filter(filter_str, L"YADIF=1:-1");
338 auto out_framerate = in_framerate_;
340 if (filter::is_double_rate(filter_str))
343 if (frame->height == 480) // NTSC DV
345 auto pad_str = L"PAD=" + boost::lexical_cast<std::wstring>(frame->width) + L":486:0:2:black";
346 filter_str = append_filter(filter_str, pad_str);
349 filter_.reset (new filter(
354 boost::rational<int>(frame->sample_aspect_ratio.num, frame->sample_aspect_ratio.den),
355 static_cast<AVPixelFormat>(frame->format),
356 std::vector<AVPixelFormat>(),
359 set_out_framerate(out_framerate);
361 auto in_fps = static_cast<double>(in_framerate_.numerator()) / static_cast<double>(in_framerate_.denominator());
363 if (ffmpeg::is_logging_quiet_for_thread())
364 CASPAR_LOG(debug) << L"[frame_muxer] " << display_mode_ << L" " << print_mode(frame->width, frame->height, in_fps, frame->interlaced_frame > 0);
366 CASPAR_LOG(info) << L"[frame_muxer] " << display_mode_ << L" " << print_mode(frame->width, frame->height, in_fps, frame->interlaced_frame > 0);
371 while (video_ready() && audio_ready() && display_mode_ != display_mode::invalid)
373 auto frame1 = pop_video();
374 frame1.audio_data() = pop_audio();
376 frame_buffer_.push(core::draw_frame(std::move(frame1)));
380 void set_out_framerate(boost::rational<int> out_framerate)
382 boost::lock_guard<boost::mutex> lock(out_framerate_mutex_);
384 bool changed = out_framerate != out_framerate_;
385 out_framerate_ = std::move(out_framerate);
388 update_audio_cadence();
391 void update_audio_cadence()
393 audio_cadence_ = find_audio_cadence(out_framerate_);
395 // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601)
396 // This cadence fills the audio mixer most optimally.
397 boost::range::rotate(audio_cadence_, std::end(audio_cadence_) - 1);
401 frame_muxer::frame_muxer(
402 boost::rational<int> in_framerate,
403 const spl::shared_ptr<core::frame_factory>& frame_factory,
404 const core::video_format_desc& format_desc,
405 const core::audio_channel_layout& channel_layout,
406 const std::wstring& filter,
407 bool multithreaded_filter)
408 : impl_(new impl(in_framerate, frame_factory, format_desc, channel_layout, filter, multithreaded_filter)){}
409 void frame_muxer::push(const std::shared_ptr<AVFrame>& video){impl_->push(video);}
410 void frame_muxer::push(const std::shared_ptr<core::mutable_audio_buffer>& audio){impl_->push(audio);}
411 core::draw_frame frame_muxer::poll(){return impl_->poll();}
412 uint32_t frame_muxer::calc_nb_frames(uint32_t nb_frames) const {return impl_->calc_nb_frames(nb_frames);}
413 bool frame_muxer::video_ready() const{return impl_->video_ready();}
414 bool frame_muxer::audio_ready() const{return impl_->audio_ready();}
415 boost::rational<int> frame_muxer::out_framerate() const { return impl_->out_framerate(); }