X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=modules%2Fffmpeg%2Fproducer%2Fmuxer%2Fframe_muxer.cpp;h=bf4850ddc8bbfdf98da2b1b228ba0375ba7e75a7;hb=726897adbf881d3b75f171fff24f2b917ba5f05a;hp=8f3dcae317fa5c381e687b425eb43ea850c3362a;hpb=9d09ee300c8c5b6ac026c60dd0d74ab7ceb2d02b;p=casparcg diff --git a/modules/ffmpeg/producer/muxer/frame_muxer.cpp b/modules/ffmpeg/producer/muxer/frame_muxer.cpp index 8f3dcae31..bf4850ddc 100644 --- a/modules/ffmpeg/producer/muxer/frame_muxer.cpp +++ b/modules/ffmpeg/producer/muxer/frame_muxer.cpp @@ -1,342 +1,416 @@ -#include "../../StdAfx.h" - -#include "frame_muxer.h" - -#include "display_mode.h" - -#include "../filter/filter.h" -#include "../util/util.h" - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#if defined(_MSC_VER) -#pragma warning (push) -#pragma warning (disable : 4244) -#endif -extern "C" -{ - #define __STDC_CONSTANT_MACROS - #define __STDC_LIMIT_MACROS - #include - #include -} -#if defined(_MSC_VER) -#pragma warning (pop) -#endif - -#include -#include - -#include -#include -#include - -using namespace caspar::core; - -namespace caspar { namespace ffmpeg { - -struct frame_muxer::implementation : boost::noncopyable -{ - std::queue>> video_streams_; - std::queue audio_streams_; - std::queue> frame_buffer_; - display_mode::type display_mode_; - const double in_fps_; - const video_format_desc format_desc_; - bool auto_transcode_; - - std::vector audio_cadence_; - - size_t audio_sample_count_; - size_t video_frame_count_; - - safe_ptr frame_factory_; - - filter filter_; - std::wstring filter_str_; - - implementation(double in_fps, const safe_ptr& frame_factory, const std::wstring& filter_str) - : display_mode_(display_mode::invalid) - , in_fps_(in_fps) - , format_desc_(frame_factory->get_video_format_desc()) - , auto_transcode_(env::properties().get("configuration.auto-transcode", true)) - , audio_cadence_(format_desc_.audio_cadence) - , audio_sample_count_(0) - , video_frame_count_(0) - , frame_factory_(frame_factory) - , filter_str_(filter_str) - { - video_streams_.push(std::queue>()); - audio_streams_.push(core::audio_buffer()); - boost::range::sort(audio_cadence_); - boost::range::reverse(audio_cadence_); - } - - void push(const std::shared_ptr& video_frame, int hints) - { - if(!video_frame) - return; - - if(video_frame == flush_video()) - { - video_frame_count_ = 0; - video_streams_.push(std::queue>()); - } - else if(video_frame == empty_video()) - { - video_streams_.back().push(make_safe(this)); - ++video_frame_count_; - display_mode_ = display_mode::simple; - } - else - { - if(display_mode_ == display_mode::invalid) - initialize_display_mode(*video_frame); - - if(hints & core::frame_producer::ALPHA_HINT) - video_frame->format = make_alpha_format(video_frame->format); - - auto format = video_frame->format; - if(video_frame->format == CASPAR_PIX_FMT_LUMA) // CASPAR_PIX_FMT_LUMA is not valid for filter, change it to GRAY8 - video_frame->format = PIX_FMT_GRAY8; - - filter_.push(video_frame); - BOOST_FOREACH(auto& av_frame, filter_.poll_all()) - { - if(video_frame->format == PIX_FMT_GRAY8 && format == CASPAR_PIX_FMT_LUMA) - av_frame->format = format; - - video_streams_.back().push(make_write_frame(this, av_frame, frame_factory_, hints)); - ++video_frame_count_; - } - } - - if(video_streams_.back().size() > 32) - BOOST_THROW_EXCEPTION(invalid_operation() << source_info("frame_muxer") << msg_info("video-stream overflow. This can be caused by incorrect frame-rate. Check clip meta-data.")); - } - - void push(const std::shared_ptr& audio) - { - if(!audio) - return; - - if(audio == flush_audio()) - { - audio_sample_count_ = 0; - audio_streams_.push(core::audio_buffer()); - } - else if(audio == empty_audio()) - { - boost::range::push_back(audio_streams_.back(), core::audio_buffer(audio_cadence_.front(), 0)); - audio_sample_count_ += audio->size(); - } - else - { - boost::range::push_back(audio_streams_.back(), *audio); - audio_sample_count_ += audio->size(); - } - - if(audio_streams_.back().size() > 32*audio_cadence_.front()) - BOOST_THROW_EXCEPTION(invalid_operation() << source_info("frame_muxer") << msg_info("audio-stream overflow. This can be caused by incorrect frame-rate. Check clip meta-data.")); - } - - bool video_ready() const - { - return video_streams_.size() > 1 || (video_streams_.size() >= audio_streams_.size() && video_ready2()); - } - - bool audio_ready() const - { - return audio_streams_.size() > 1 || (audio_streams_.size() >= video_streams_.size() && audio_ready2()); - } - - bool video_ready2() const - { - switch(display_mode_) - { - case display_mode::deinterlace_bob_reinterlace: - case display_mode::interlace: - case display_mode::half: - return video_streams_.front().size() >= 2; - default: - return video_streams_.front().size() >= 1; - } - } - - bool audio_ready2() const - { - switch(display_mode_) - { - case display_mode::duplicate: - return audio_streams_.front().size()/2 >= audio_cadence_.front(); - default: - return audio_streams_.front().size() >= audio_cadence_.front(); - } - } - - std::shared_ptr poll() - { - if(!frame_buffer_.empty()) - { - auto frame = frame_buffer_.front(); - frame_buffer_.pop(); - return frame; - } - - if(video_streams_.size() > 1 && audio_streams_.size() > 1 && (!video_ready2() || !audio_ready2())) - { - if(!video_streams_.front().empty() || !audio_streams_.front().empty()) - CASPAR_LOG(debug) << "Truncating: " << video_streams_.front().size() << L" video-frames, " << audio_streams_.front().size() << L" audio-samples."; - - video_streams_.pop(); - audio_streams_.pop(); - } - - if(!video_ready2() || !audio_ready2()) - return nullptr; - - auto frame1 = pop_video(); - frame1->audio_data() = pop_audio(); - - switch(display_mode_) - { - case display_mode::simple: - case display_mode::deinterlace_bob: - case display_mode::deinterlace: - { - frame_buffer_.push(frame1); - break; - } - case display_mode::interlace: - case display_mode::deinterlace_bob_reinterlace: - { - auto frame2 = pop_video(); - - frame_buffer_.push(core::basic_frame::interlace(frame1, frame2, format_desc_.field_mode)); - break; - } - case display_mode::duplicate: - { - auto frame2 = make_safe(*frame1); - frame2->audio_data() = pop_audio(); - - frame_buffer_.push(frame1); - frame_buffer_.push(frame2); - break; - } - case display_mode::half: - { - pop_video(); // Throw away - - frame_buffer_.push(frame1); - break; - } - default: - BOOST_THROW_EXCEPTION(invalid_operation() << msg_info("invalid display-mode")); - } - - return frame_buffer_.empty() ? nullptr : poll(); - } - - safe_ptr pop_video() - { - auto frame = video_streams_.front().front(); - video_streams_.front().pop(); - return frame; - } - - core::audio_buffer pop_audio() - { - CASPAR_VERIFY(audio_streams_.front().size() >= audio_cadence_.front()); - - auto begin = audio_streams_.front().begin(); - auto end = begin + audio_cadence_.front(); - - core::audio_buffer samples(begin, end); - audio_streams_.front().erase(begin, end); - - boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1); - - return samples; - } - - void initialize_display_mode(const AVFrame& frame) - { - display_mode_ = display_mode::simple; - if(auto_transcode_) - { - auto mode = get_mode(frame); - auto fps = in_fps_; - - if(is_deinterlacing(filter_str_)) - mode = core::field_mode::progressive; - - if(is_double_rate(filter_str_)) - fps *= 2; - - display_mode_ = get_display_mode(mode, fps, format_desc_.field_mode, format_desc_.fps); - - if((frame.height != 480 || format_desc_.height != 486) && - display_mode_ == display_mode::simple && mode != core::field_mode::progressive && format_desc_.field_mode != core::field_mode::progressive && - frame.height != static_cast(format_desc_.height)) - { - display_mode_ = display_mode::deinterlace_bob_reinterlace; // The frame will most likely be scaled, we need to deinterlace->reinterlace - } - - if(display_mode_ == display_mode::deinterlace) - filter_str_ = append_filter(filter_str_, L"YADIF=0:-1"); - else if(display_mode_ == display_mode::deinterlace_bob || display_mode_ == display_mode::deinterlace_bob_reinterlace) - filter_str_ = append_filter(filter_str_, L"YADIF=1:-1"); - } - - if(display_mode_ == display_mode::invalid) - { - CASPAR_LOG(warning) << L"[frame_muxer] Auto-transcode: Failed to detect display-mode."; - display_mode_ = display_mode::simple; - } - - filter_ = filter(filter_str_); - - CASPAR_LOG(info) << "[frame_muxer] " << display_mode::print(display_mode_) << L" " << print_mode(frame.width, frame.height, in_fps_, frame.interlaced_frame > 0); - } - - int64_t calc_nb_frames(int64_t nb_frames) const - { - switch(display_mode_) // Take into account transformation in run. - { - case display_mode::deinterlace_bob_reinterlace: - case display_mode::interlace: - case display_mode::half: - nb_frames /= 2; - break; - case display_mode::duplicate: - nb_frames *= 2; - break; - } - - if(is_double_rate(widen(filter_.filter_str()))) // Take into account transformations in filter. - nb_frames *= 2; - - return nb_frames; - } -}; - -frame_muxer::frame_muxer(double in_fps, const safe_ptr& frame_factory, const std::wstring& filter) - : impl_(new implementation(in_fps, frame_factory, filter)){} -void frame_muxer::push(const std::shared_ptr& video_frame, int hints){impl_->push(video_frame, hints);} -void frame_muxer::push(const std::shared_ptr& audio_samples){return impl_->push(audio_samples);} -std::shared_ptr frame_muxer::poll(){return impl_->poll();} -int64_t frame_muxer::calc_nb_frames(int64_t nb_frames) const {return impl_->calc_nb_frames(nb_frames);} -bool frame_muxer::video_ready() const{return impl_->video_ready();} -bool frame_muxer::audio_ready() const{return impl_->audio_ready();} - -}} \ No newline at end of file +/* +* Copyright (c) 2011 Sveriges Television AB +* +* This file is part of CasparCG (www.casparcg.com). +* +* CasparCG is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* CasparCG is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with CasparCG. If not, see . +* +* Author: Robert Nagy, ronag89@gmail.com +*/ + +#include "../../StdAfx.h" + +#include "frame_muxer.h" + +#include "../filter/filter.h" +#include "../util/util.h" +#include "../../ffmpeg.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning (push) +#pragma warning (disable : 4244) +#endif +extern "C" +{ + #define __STDC_CONSTANT_MACROS + #define __STDC_LIMIT_MACROS + #include + #include +} +#if defined(_MSC_VER) +#pragma warning (pop) +#endif + +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace caspar::core; + +namespace caspar { namespace ffmpeg { +struct av_frame_format +{ + int pix_format; + std::array line_sizes; + int width; + int height; + + av_frame_format(const AVFrame& frame) + : pix_format(frame.format) + , width(frame.width) + , height(frame.height) + { + boost::copy(frame.linesize, line_sizes.begin()); + } + + bool operator==(const av_frame_format& other) const + { + return pix_format == other.pix_format + && line_sizes == other.line_sizes + && width == other.width + && height == other.height; + } + + bool operator!=(const av_frame_format& other) const + { + return !(*this == other); + } +}; + +struct frame_muxer::impl : boost::noncopyable +{ + std::queue> video_streams_; + std::queue audio_streams_; + std::queue frame_buffer_; + display_mode display_mode_ = display_mode::invalid; + const boost::rational in_framerate_; + const video_format_desc format_desc_; + const audio_channel_layout audio_channel_layout_; + + std::vector audio_cadence_ = format_desc_.audio_cadence; + + spl::shared_ptr frame_factory_; + boost::optional previously_filtered_frame_; + + std::unique_ptr filter_; + const std::wstring filter_str_; + const bool multithreaded_filter_; + bool force_deinterlacing_ = env::properties().get(L"configuration.force-deinterlace", false); + + mutable boost::mutex out_framerate_mutex_; + boost::rational out_framerate_; + + impl( + boost::rational in_framerate, + const spl::shared_ptr& frame_factory, + const core::video_format_desc& format_desc, + const core::audio_channel_layout& channel_layout, + const std::wstring& filter_str, + bool multithreaded_filter) + : in_framerate_(in_framerate) + , format_desc_(format_desc) + , audio_channel_layout_(channel_layout) + , frame_factory_(frame_factory) + , filter_str_(filter_str) + , multithreaded_filter_(multithreaded_filter) + { + video_streams_.push(std::queue()); + audio_streams_.push(core::mutable_audio_buffer()); + + set_out_framerate(in_framerate_); + } + + void push(const std::shared_ptr& video_frame) + { + if (!video_frame) + return; + + av_frame_format current_frame_format(*video_frame); + + if (previously_filtered_frame_ && video_frame->data[0] && *previously_filtered_frame_ != current_frame_format) + { + // Fixes bug where avfilter crashes server on some DV files (starts in YUV420p but changes to YUV411p after the first frame). + if (ffmpeg::is_logging_quiet_for_thread()) + CASPAR_LOG(debug) << L"[frame_muxer] Frame format has changed. Resetting display mode."; + else + CASPAR_LOG(info) << L"[frame_muxer] Frame format has changed. Resetting display mode."; + + display_mode_ = display_mode::invalid; + filter_.reset(); + previously_filtered_frame_ = boost::none; + } + + if (video_frame == flush_video()) + { + video_streams_.push(std::queue()); + } + else if (video_frame == empty_video()) + { + video_streams_.back().push(frame_factory_->create_frame(this, core::pixel_format::invalid, audio_channel_layout_)); + display_mode_ = display_mode::simple; + } + else + { + if (!filter_ || display_mode_ == display_mode::invalid) + update_display_mode(video_frame); + + if (filter_) + { + filter_->push(video_frame); + previously_filtered_frame_ = current_frame_format; + + for (auto& av_frame : filter_->poll_all()) + video_streams_.back().push(make_frame(this, av_frame, *frame_factory_, audio_channel_layout_)); + } + } + + if (video_streams_.back().size() > 32) + CASPAR_THROW_EXCEPTION(invalid_operation() << source_info("frame_muxer") << msg_info("video-stream overflow. This can be caused by incorrect frame-rate. Check clip meta-data.")); + } + + void push(const std::shared_ptr& audio) + { + if (!audio) + return; + + if (audio == flush_audio()) + { + audio_streams_.push(core::mutable_audio_buffer()); + } + else if (audio == empty_audio()) + { + boost::range::push_back(audio_streams_.back(), core::mutable_audio_buffer(audio_cadence_.front() * audio_channel_layout_.num_channels, 0)); + } + else + { + boost::range::push_back(audio_streams_.back(), *audio); + } + + if (audio_streams_.back().size() > 32 * audio_cadence_.front() * audio_channel_layout_.num_channels) + BOOST_THROW_EXCEPTION(invalid_operation() << source_info("frame_muxer") << msg_info("audio-stream overflow. This can be caused by incorrect frame-rate. Check clip meta-data.")); + } + + bool video_ready() const + { + return video_streams_.size() > 1 || (video_streams_.size() >= audio_streams_.size() && video_ready2()); + } + + bool audio_ready() const + { + return audio_streams_.size() > 1 || (audio_streams_.size() >= video_streams_.size() && audio_ready2()); + } + + bool video_ready2() const + { + return video_streams_.front().size() >= 1; + } + + bool audio_ready2() const + { + return audio_streams_.front().size() >= audio_cadence_.front() * audio_channel_layout_.num_channels; + } + + core::draw_frame poll() + { + if (!frame_buffer_.empty()) + { + auto frame = frame_buffer_.front(); + frame_buffer_.pop(); + return frame; + } + + if (video_streams_.size() > 1 && audio_streams_.size() > 1 && (!video_ready2() || !audio_ready2())) + { + if (!video_streams_.front().empty() || !audio_streams_.front().empty()) + CASPAR_LOG(trace) << "Truncating: " << video_streams_.front().size() << L" video-frames, " << audio_streams_.front().size() << L" audio-samples."; + + video_streams_.pop(); + audio_streams_.pop(); + } + + if (!video_ready2() || !audio_ready2() || display_mode_ == display_mode::invalid) + return core::draw_frame::empty(); + + auto frame = pop_video(); + frame.audio_data() = pop_audio(); + + frame_buffer_.push(core::draw_frame(std::move(frame))); + + return frame_buffer_.empty() ? core::draw_frame::empty() : poll(); + } + + core::mutable_frame pop_video() + { + auto frame = std::move(video_streams_.front().front()); + video_streams_.front().pop(); + return frame; + } + + core::mutable_audio_buffer pop_audio() + { + CASPAR_VERIFY(audio_streams_.front().size() >= audio_cadence_.front() * audio_channel_layout_.num_channels); + + auto begin = audio_streams_.front().begin(); + auto end = begin + (audio_cadence_.front() * audio_channel_layout_.num_channels); + + core::mutable_audio_buffer samples(begin, end); + audio_streams_.front().erase(begin, end); + + boost::range::rotate(audio_cadence_, std::begin(audio_cadence_) + 1); + + return samples; + } + + uint32_t calc_nb_frames(uint32_t nb_frames) const + { + uint64_t nb_frames2 = nb_frames; + + if(filter_ && filter_->is_double_rate()) // Take into account transformations in filter. + nb_frames2 *= 2; + + return static_cast(nb_frames2); + } + + boost::rational out_framerate() const + { + boost::lock_guard lock(out_framerate_mutex_); + + return out_framerate_; + } +private: + void update_display_mode(const std::shared_ptr& frame) + { + std::wstring filter_str = filter_str_; + + display_mode_ = display_mode::simple; + + auto mode = get_mode(*frame); + if (mode == core::field_mode::progressive && frame->height < 720 && in_framerate_ < 50) // SD frames are interlaced. Probably incorrect meta-data. Fix it. + mode = core::field_mode::upper; + + if (filter::is_deinterlacing(filter_str_)) + { + display_mode_ = display_mode::simple; + } + else if (mode != core::field_mode::progressive) + { + if (force_deinterlacing_) + { + display_mode_ = display_mode::deinterlace_bob; + } + else + { + bool output_also_interlaced = format_desc_.field_mode != core::field_mode::progressive; + bool interlaced_output_compatible = + output_also_interlaced + && ( + (frame->height == 480 && format_desc_.height == 486) // don't deinterlace for NTSC DV + || frame->height == format_desc_.height + ) + && in_framerate_ == format_desc_.framerate; + + display_mode_ = interlaced_output_compatible ? display_mode::simple : display_mode::deinterlace_bob; + } + } + + if (display_mode_ == display_mode::deinterlace_bob) + filter_str = append_filter(filter_str, L"YADIF=1:-1"); + + auto out_framerate = in_framerate_; + + if (filter::is_double_rate(filter_str)) + out_framerate *= 2; + + if (frame->height == 480) // NTSC DV + { + auto pad_str = L"PAD=" + boost::lexical_cast(frame->width) + L":486:0:2:black"; + filter_str = append_filter(filter_str, pad_str); + } + + filter_.reset (new filter( + frame->width, + frame->height, + 1 / in_framerate_, + in_framerate_, + boost::rational(frame->sample_aspect_ratio.num, frame->sample_aspect_ratio.den), + static_cast(frame->format), + std::vector(), + u8(filter_str))); + + set_out_framerate(out_framerate); + + auto in_fps = static_cast(in_framerate_.numerator()) / static_cast(in_framerate_.denominator()); + + if (ffmpeg::is_logging_quiet_for_thread()) + CASPAR_LOG(debug) << L"[frame_muxer] " << display_mode_ << L" " << print_mode(frame->width, frame->height, in_fps, frame->interlaced_frame > 0); + else + CASPAR_LOG(info) << L"[frame_muxer] " << display_mode_ << L" " << print_mode(frame->width, frame->height, in_fps, frame->interlaced_frame > 0); + } + + void merge() + { + while (video_ready() && audio_ready() && display_mode_ != display_mode::invalid) + { + auto frame1 = pop_video(); + frame1.audio_data() = pop_audio(); + + frame_buffer_.push(core::draw_frame(std::move(frame1))); + } + } + + void set_out_framerate(boost::rational out_framerate) + { + boost::lock_guard lock(out_framerate_mutex_); + + bool changed = out_framerate != out_framerate_; + out_framerate_ = std::move(out_framerate); + + if (changed) + update_audio_cadence(); + } + + void update_audio_cadence() + { + audio_cadence_ = find_audio_cadence(out_framerate_); + + // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601) + // This cadence fills the audio mixer most optimally. + boost::range::rotate(audio_cadence_, std::end(audio_cadence_) - 1); + } +}; + +frame_muxer::frame_muxer( + boost::rational in_framerate, + const spl::shared_ptr& frame_factory, + const core::video_format_desc& format_desc, + const core::audio_channel_layout& channel_layout, + const std::wstring& filter, + bool multithreaded_filter) + : impl_(new impl(in_framerate, frame_factory, format_desc, channel_layout, filter, multithreaded_filter)){} +void frame_muxer::push(const std::shared_ptr& video){impl_->push(video);} +void frame_muxer::push(const std::shared_ptr& audio){impl_->push(audio);} +core::draw_frame frame_muxer::poll(){return impl_->poll();} +uint32_t frame_muxer::calc_nb_frames(uint32_t nb_frames) const {return impl_->calc_nb_frames(nb_frames);} +bool frame_muxer::video_ready() const{return impl_->video_ready();} +bool frame_muxer::audio_ready() const{return impl_->audio_ready();} +boost::rational frame_muxer::out_framerate() const { return impl_->out_framerate(); } +}}