X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=modules%2Fffmpeg%2Fproducer%2Fmuxer%2Fframe_muxer.cpp;h=7cb40f809ca76da73c952dac9422906b9833b464;hb=fed3739ab45e3fa57470dbe2a50c5f316c0307f3;hp=bd2925ea294fc6838b00e381b54489f6839eb746;hpb=126f680b450e8d8cfe71b1b410df05bf584a2403;p=casparcg diff --git a/modules/ffmpeg/producer/muxer/frame_muxer.cpp b/modules/ffmpeg/producer/muxer/frame_muxer.cpp index bd2925ea2..7cb40f809 100644 --- a/modules/ffmpeg/producer/muxer/frame_muxer.cpp +++ b/modules/ffmpeg/producer/muxer/frame_muxer.cpp @@ -1,386 +1,407 @@ -/* -* Copyright (c) 2011 Sveriges Television AB -* -* This file is part of CasparCG (www.casparcg.com). -* -* CasparCG is free software: you can redistribute it and/or modify -* it under the terms of the GNU General Public License as published by -* the Free Software Foundation, either version 3 of the License, or -* (at your option) any later version. -* -* CasparCG is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with CasparCG. If not, see . -* -* Author: Robert Nagy, ronag89@gmail.com -*/ - -#include "../../StdAfx.h" - -#include "frame_muxer.h" - -#include "../filter/filter.h" -#include "../util/util.h" - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#if defined(_MSC_VER) -#pragma warning (push) -#pragma warning (disable : 4244) -#endif -extern "C" -{ - #define __STDC_CONSTANT_MACROS - #define __STDC_LIMIT_MACROS - #include - #include -} -#if defined(_MSC_VER) -#pragma warning (pop) -#endif - -#include -#include -#include -#include - -#include -#include -#include - -using namespace caspar::core; - -namespace caspar { namespace ffmpeg { - -struct frame_muxer::impl : boost::noncopyable -{ - std::queue>> video_streams_; - std::queue audio_streams_; - std::queue> frame_buffer_; - display_mode display_mode_; - const double in_fps_; - const video_format_desc format_desc_; - bool auto_transcode_; - bool auto_deinterlace_; - - std::vector audio_cadence_; - - safe_ptr frame_factory_; - - filter filter_; - const std::wstring filter_str_; - bool force_deinterlacing_; - - impl(double in_fps, const safe_ptr& frame_factory, const std::wstring& filter_str) - : display_mode_(display_mode::invalid) - , in_fps_(in_fps) - , format_desc_(frame_factory->get_video_format_desc()) - , auto_transcode_(env::properties().get(L"configuration.auto-transcode", true)) - , auto_deinterlace_(env::properties().get(L"configuration.auto-deinterlace", true)) - , audio_cadence_(format_desc_.audio_cadence) - , frame_factory_(frame_factory) - , filter_str_(filter_str) - , force_deinterlacing_(false) - { - video_streams_.push(std::queue>()); - audio_streams_.push(core::audio_buffer()); - - // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601) - // This cadence fills the audio mixer most optimally. - boost::range::rotate(audio_cadence_, std::end(audio_cadence_)-1); - } - - void push(const std::shared_ptr& video_frame, int flags) - { - if(!video_frame) - return; - - if(video_frame == flush_video()) - { - video_streams_.push(std::queue>()); - } - else if(video_frame == empty_video()) - { - video_streams_.back().push(make_safe(this)); - display_mode_ = display_mode::simple; - } - else - { - bool DEINTERLACE_FLAG = (flags & core::frame_producer::flags::deinterlace) != 0; - - if(auto_deinterlace_ && force_deinterlacing_ != DEINTERLACE_FLAG) - { - force_deinterlacing_ = DEINTERLACE_FLAG; - display_mode_ = display_mode::invalid; - } - - if(display_mode_ == display_mode::invalid) - update_display_mode(video_frame, force_deinterlacing_); - - if(flags & core::frame_producer::flags::alpha_only) - video_frame->format = make_alpha_format(video_frame->format); - - auto format = video_frame->format; - if(video_frame->format == CASPAR_PIX_FMT_LUMA) // CASPAR_PIX_FMT_LUMA is not valid for filter, change it to GRAY8 - video_frame->format = PIX_FMT_GRAY8; - - filter_.push(video_frame); - BOOST_FOREACH(auto& av_frame, filter_.poll_all()) - { - if(video_frame->format == PIX_FMT_GRAY8 && format == CASPAR_PIX_FMT_LUMA) - av_frame->format = format; - - video_streams_.back().push(make_write_frame(this, av_frame, frame_factory_, flags)); - } - } - - if(video_streams_.back().size() > 32) - BOOST_THROW_EXCEPTION(invalid_operation() << source_info("frame_muxer") << msg_info("video-stream overflow. This can be caused by incorrect frame-rate. Check clip meta-data.")); - } - - void push(const std::shared_ptr& audio) - { - if(!audio) - return; - - if(audio == flush_audio()) - { - audio_streams_.push(core::audio_buffer()); - } - else if(audio == empty_audio()) - { - boost::range::push_back(audio_streams_.back(), core::audio_buffer(audio_cadence_.front(), 0)); - } - else - { - boost::range::push_back(audio_streams_.back(), *audio); - } - - if(audio_streams_.back().size() > static_cast(32*audio_cadence_.front())) - BOOST_THROW_EXCEPTION(invalid_operation() << source_info("frame_muxer") << msg_info("audio-stream overflow. This can be caused by incorrect frame-rate. Check clip meta-data.")); - } - - bool video_ready() const - { - return video_streams_.size() > 1 || (video_streams_.size() >= audio_streams_.size() && video_ready2()); - } - - bool audio_ready() const - { - return audio_streams_.size() > 1 || (audio_streams_.size() >= video_streams_.size() && audio_ready2()); - } - - bool video_ready2() const - { - switch(display_mode_) - { - case display_mode::deinterlace_bob_reinterlace: - case display_mode::interlace: - case display_mode::half: - return video_streams_.front().size() >= 2; - default: - return video_streams_.front().size() >= 1; - } - } - - bool audio_ready2() const - { - switch(display_mode_) - { - case display_mode::duplicate: - return audio_streams_.front().size()/2 >= static_cast(audio_cadence_.front()); - default: - return audio_streams_.front().size() >= static_cast(audio_cadence_.front()); - } - } - - std::shared_ptr poll() - { - if(!frame_buffer_.empty()) - { - auto frame = frame_buffer_.front(); - frame_buffer_.pop(); - return frame; - } - - if(video_streams_.size() > 1 && audio_streams_.size() > 1 && (!video_ready2() || !audio_ready2())) - { - if(!video_streams_.front().empty() || !audio_streams_.front().empty()) - CASPAR_LOG(trace) << "Truncating: " << video_streams_.front().size() << L" video-frames, " << audio_streams_.front().size() << L" audio-samples."; - - video_streams_.pop(); - audio_streams_.pop(); - } - - if(!video_ready2() || !audio_ready2() || display_mode_ == display_mode::invalid) - return nullptr; - - auto frame1 = pop_video(); - frame1->audio_data() = pop_audio(); - - switch(display_mode_) - { - case display_mode::simple: - case display_mode::deinterlace_bob: - case display_mode::deinterlace: - { - frame_buffer_.push(frame1); - break; - } - case display_mode::interlace: - case display_mode::deinterlace_bob_reinterlace: - { - auto frame2 = pop_video(); - - frame_buffer_.push(core::basic_frame::interlace(frame1, frame2, format_desc_.field_mode)); - break; - } - case display_mode::duplicate: - { - auto frame2 = make_safe(*frame1); - frame2->audio_data() = pop_audio(); - - frame_buffer_.push(frame1); - frame_buffer_.push(frame2); - break; - } - case display_mode::half: - { - pop_video(); // Throw away - - frame_buffer_.push(frame1); - break; - } - } - - return frame_buffer_.empty() ? nullptr : poll(); - } - - safe_ptr pop_video() - { - auto frame = video_streams_.front().front(); - video_streams_.front().pop(); - return frame; - } - - core::audio_buffer pop_audio() - { - CASPAR_VERIFY(audio_streams_.front().size() >= static_cast(audio_cadence_.front())); - - auto begin = audio_streams_.front().begin(); - auto end = begin + audio_cadence_.front(); - - core::audio_buffer samples(begin, end); - audio_streams_.front().erase(begin, end); - - boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1); - - return samples; - } - - void update_display_mode(const std::shared_ptr& frame, bool force_deinterlace) - { - std::wstring filter_str = filter_str_; - - display_mode_ = display_mode::simple; - if(auto_transcode_) - { - auto mode = get_mode(*frame); - auto fps = in_fps_; - - if(filter::is_deinterlacing(filter_str_)) - mode = core::field_mode::progressive; - - if(filter::is_double_rate(filter_str_)) - fps *= 2; - - display_mode_ = get_display_mode(mode, fps, format_desc_.field_mode, format_desc_.fps); - - if((frame->height != 480 || format_desc_.height != 486) && // don't deinterlace for NTSC DV - display_mode_ == display_mode::simple && mode != core::field_mode::progressive && format_desc_.field_mode != core::field_mode::progressive && - frame->height != static_cast(format_desc_.height)) - { - display_mode_ = display_mode::deinterlace_bob_reinterlace; // The frame will most likely be scaled, we need to deinterlace->reinterlace - } - - if(force_deinterlace && mode != core::field_mode::progressive && - display_mode_ != display_mode::deinterlace && - display_mode_ != display_mode::deinterlace_bob && - display_mode_ != display_mode::deinterlace_bob_reinterlace) - { - CASPAR_LOG(info) << L"[frame_muxer] Automatically started non bob-deinterlacing. Consider starting producer with bob-deinterlacing (FILTER DEINTERLACE_BOB) for smoothest playback."; - display_mode_ = display_mode::deinterlace; - } - - if(display_mode_ == display_mode::deinterlace) - filter_str = append_filter(filter_str, L"YADIF=0:-1"); - else if(display_mode_ == display_mode::deinterlace_bob || display_mode_ == display_mode::deinterlace_bob_reinterlace) - filter_str = append_filter(filter_str, L"YADIF=1:-1"); - } - - if(display_mode_ == display_mode::invalid) - { - CASPAR_LOG(warning) << L"[frame_muxer] Auto-transcode: Failed to detect display-mode."; - display_mode_ = display_mode::simple; - } - - if(!boost::iequals(filter_.filter_str(), filter_str)) - { - for(int n = 0; n < filter_.delay(); ++n) - { - filter_.push(frame); - auto av_frame = filter_.poll(); - if(av_frame) - video_streams_.back().push(make_write_frame(this, make_safe_ptr(av_frame), frame_factory_, 0)); - } - filter_ = filter(filter_str); - CASPAR_LOG(info) << L"[frame_muxer] " << display_mode_ << L" " << print_mode(frame->width, frame->height, in_fps_, frame->interlaced_frame > 0); - } - } - - uint32_t calc_nb_frames(uint32_t nb_frames) const - { - uint64_t nb_frames2 = nb_frames; - - if(filter_.is_double_rate()) // Take into account transformations in filter. - nb_frames2 *= 2; - - switch(display_mode_) // Take into account transformation in run. - { - case display_mode::deinterlace_bob_reinterlace: - case display_mode::interlace: - case display_mode::half: - nb_frames2 /= 2; - break; - case display_mode::duplicate: - nb_frames2 *= 2; - break; - } - - return static_cast(nb_frames2); - } -}; - -frame_muxer::frame_muxer(double in_fps, const safe_ptr& frame_factory, const std::wstring& filter) - : impl_(new impl(in_fps, frame_factory, filter)){} -void frame_muxer::push(const std::shared_ptr& video_frame, int flags){impl_->push(video_frame, flags);} -void frame_muxer::push(const std::shared_ptr& audio_samples){return impl_->push(audio_samples);} -std::shared_ptr frame_muxer::poll(){return impl_->poll();} -uint32_t frame_muxer::calc_nb_frames(uint32_t nb_frames) const {return impl_->calc_nb_frames(nb_frames);} -bool frame_muxer::video_ready() const{return impl_->video_ready();} -bool frame_muxer::audio_ready() const{return impl_->audio_ready();} - +/* +* Copyright (c) 2011 Sveriges Television AB +* +* This file is part of CasparCG (www.casparcg.com). +* +* CasparCG is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* CasparCG is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with CasparCG. If not, see . +* +* Author: Robert Nagy, ronag89@gmail.com +*/ + +#include "../../StdAfx.h" + +#include "frame_muxer.h" + +#include "../filter/filter.h" +#include "../util/util.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning (push) +#pragma warning (disable : 4244) +#endif +extern "C" +{ + #define __STDC_CONSTANT_MACROS + #define __STDC_LIMIT_MACROS + #include + #include +} +#if defined(_MSC_VER) +#pragma warning (pop) +#endif + +#include +#include +#include + +#include +#include +#include + +using namespace caspar::core; + +namespace caspar { namespace ffmpeg { + +bool is_frame_format_changed(const AVFrame& lhs, const AVFrame& rhs) +{ + if (lhs.format != rhs.format) + return true; + + for (int i = 0; i < AV_NUM_DATA_POINTERS; ++i) + { + if (lhs.linesize[i] != rhs.linesize[i]) + return true; + } + + return false; +} + +struct frame_muxer::impl : boost::noncopyable +{ + std::queue video_stream_; + core::mutable_audio_buffer audio_stream_; + std::queue frame_buffer_; + display_mode display_mode_ = display_mode::invalid; + const double in_fps_; + const video_format_desc format_desc_; + audio_channel_layout channel_layout_; + + std::vector audio_cadence_ = format_desc_.audio_cadence; + + spl::shared_ptr frame_factory_; + std::shared_ptr previous_frame_; + + std::unique_ptr filter_; + const std::wstring filter_str_; + bool force_deinterlacing_ = env::properties().get(L"configuration.force-deinterlace", true); + + impl( + double in_fps, + const spl::shared_ptr& frame_factory, + const core::video_format_desc& format_desc, + const core::audio_channel_layout& channel_layout, + const std::wstring& filter_str) + : in_fps_(in_fps) + , format_desc_(format_desc) + , channel_layout_(channel_layout) + , frame_factory_(frame_factory) + , filter_str_(filter_str) + { + // Note: Uses 1 step rotated cadence for 1001 modes (1602, 1602, 1601, 1602, 1601) + // This cadence fills the audio mixer most optimally. + boost::range::rotate(audio_cadence_, std::end(audio_cadence_)-1); + } + + void push_video(const std::shared_ptr& video) + { + if(!video) + return; + + if (previous_frame_ && video->data[0] && is_frame_format_changed(*previous_frame_, *video)) + { + // Fixes bug where avfilter crashes server on some DV files (starts in YUV420p but changes to YUV411p after the first frame). + CASPAR_LOG(info) << L"[frame_muxer] Frame format has changed. Resetting display mode."; + display_mode_ = display_mode::invalid; + } + + if(!video->data[0]) + { + auto empty_frame = frame_factory_->create_frame(this, core::pixel_format_desc(core::pixel_format::invalid), channel_layout_); + video_stream_.push(std::move(empty_frame)); + display_mode_ = display_mode::simple; + } + else + { + if(!filter_ || display_mode_ == display_mode::invalid) + update_display_mode(video); + + filter_->push(video); + previous_frame_ = video; + for (auto& av_frame : filter_->poll_all()) + video_stream_.push(make_frame(this, av_frame, format_desc_.fps, *frame_factory_, channel_layout_)); + } + + merge(); + } + + void push_audio(const std::shared_ptr& audio) + { + if(!audio) + return; + + if(!audio->data[0]) + { + if (channel_layout_ == core::audio_channel_layout::invalid()) + channel_layout_ = *core::audio_channel_layout_repository::get_default()->get_layout(L"stereo"); + + boost::range::push_back(audio_stream_, core::mutable_audio_buffer(audio_cadence_.front() * channel_layout_.num_channels, 0)); + } + else + { + auto ptr = reinterpret_cast(audio->data[0]); + audio_stream_.insert(audio_stream_.end(), ptr, ptr + audio->linesize[0]/sizeof(int32_t)); + } + + merge(); + } + + bool video_ready() const + { + switch(display_mode_) + { + case display_mode::deinterlace_bob_reinterlace: + case display_mode::interlace: + case display_mode::half: + return video_stream_.size() >= 2; + default: + return video_stream_.size() >= 1; + } + } + + bool audio_ready() const + { + switch(display_mode_) + { + case display_mode::duplicate: + return audio_stream_.size() >= static_cast(audio_cadence_[0] + audio_cadence_[1 % audio_cadence_.size()]) * channel_layout_.num_channels; + default: + return audio_stream_.size() >= static_cast(audio_cadence_.front()) * channel_layout_.num_channels; + } + } + + bool empty() const + { + return frame_buffer_.empty(); + } + + core::draw_frame front() const + { + return frame_buffer_.front(); + } + + void pop() + { + frame_buffer_.pop(); + } + + void merge() + { + while(video_ready() && audio_ready() && display_mode_ != display_mode::invalid) + { + auto frame1 = pop_video(); + frame1.audio_data() = pop_audio(); + + switch(display_mode_) + { + case display_mode::simple: + case display_mode::deinterlace_bob: + case display_mode::deinterlace: + { + frame_buffer_.push(core::draw_frame(std::move(frame1))); + break; + } + case display_mode::interlace: + case display_mode::deinterlace_bob_reinterlace: + { + auto frame2 = pop_video(); + + frame_buffer_.push(core::draw_frame::interlace( + core::draw_frame(std::move(frame1)), + core::draw_frame(std::move(frame2)), + format_desc_.field_mode)); + break; + } + case display_mode::duplicate: + { + //boost::range::push_back(frame1.audio_data(), pop_audio()); + + auto second_audio_frame = core::mutable_frame( + std::vector>(), + pop_audio(), + frame1.stream_tag(), + core::pixel_format_desc(), + channel_layout_); + auto first_frame = core::draw_frame(std::move(frame1)); + auto muted_first_frame = core::draw_frame(first_frame); + muted_first_frame.transform().audio_transform.volume = 0; + auto second_frame = core::draw_frame({ core::draw_frame(std::move(second_audio_frame)), muted_first_frame }); + + // Same video but different audio. + frame_buffer_.push(first_frame); + frame_buffer_.push(second_frame); + break; + } + case display_mode::half: + { + pop_video(); // Throw away + + frame_buffer_.push(core::draw_frame(std::move(frame1))); + break; + } + default: + CASPAR_THROW_EXCEPTION(invalid_operation()); + } + } + } + + core::mutable_frame pop_video() + { + auto frame = std::move(video_stream_.front()); + video_stream_.pop(); + return std::move(frame); + } + + core::mutable_audio_buffer pop_audio() + { + if (audio_stream_.size() < audio_cadence_.front() * channel_layout_.num_channels) + CASPAR_THROW_EXCEPTION(out_of_range()); + + auto begin = audio_stream_.begin(); + auto end = begin + audio_cadence_.front() * channel_layout_.num_channels; + + core::mutable_audio_buffer samples(begin, end); + audio_stream_.erase(begin, end); + + boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1); + + return samples; + } + + void update_display_mode(const std::shared_ptr& frame) + { + std::wstring filter_str = filter_str_; + + display_mode_ = display_mode::simple; + + auto mode = get_mode(*frame); + if(mode == core::field_mode::progressive && frame->height < 720 && in_fps_ < 50.0) // SD frames are interlaced. Probably incorrect meta-data. Fix it. + mode = core::field_mode::upper; + + auto fps = in_fps_; + + if(filter::is_deinterlacing(filter_str_)) + mode = core::field_mode::progressive; + + if(filter::is_double_rate(filter_str_)) + fps *= 2; + + display_mode_ = get_display_mode(mode, fps, format_desc_.field_mode, format_desc_.fps); + + if((frame->height != 480 || format_desc_.height != 486) && // don't deinterlace for NTSC DV + display_mode_ == display_mode::simple && mode != core::field_mode::progressive && format_desc_.field_mode != core::field_mode::progressive && + frame->height != format_desc_.height) + { + display_mode_ = display_mode::deinterlace_bob_reinterlace; // The frame will most likely be scaled, we need to deinterlace->reinterlace + } + + // ALWAYS de-interlace, until we have GPU de-interlacing. + if(force_deinterlacing_ && frame->interlaced_frame && display_mode_ != display_mode::deinterlace_bob && display_mode_ != display_mode::deinterlace) + display_mode_ = display_mode::deinterlace_bob_reinterlace; + + if(display_mode_ == display_mode::deinterlace) + filter_str = append_filter(filter_str, L"YADIF=0:-1"); + else if(display_mode_ == display_mode::deinterlace_bob || display_mode_ == display_mode::deinterlace_bob_reinterlace) + filter_str = append_filter(filter_str, L"YADIF=1:-1"); + + if(display_mode_ == display_mode::invalid) + { + CASPAR_LOG(warning) << L"[frame_muxer] Auto-transcode: Failed to detect display-mode."; + display_mode_ = display_mode::simple; + } + + if(frame->height == 480) // NTSC DV + { + auto pad_str = L"PAD=" + boost::lexical_cast(frame->width) + L":486:0:2:black"; + filter_str = append_filter(filter_str, pad_str); + } + + filter_.reset (new filter( + frame->width, + frame->height, + boost::rational(1000000, static_cast(in_fps_ * 1000000)), + boost::rational(static_cast(in_fps_ * 1000000), 1000000), + boost::rational(frame->sample_aspect_ratio.num, frame->sample_aspect_ratio.den), + static_cast(frame->format), + std::vector(), + u8(filter_str))); + + CASPAR_LOG(info) << L"[frame_muxer] " << display_mode_ << L" " << print_mode(frame->width, frame->height, in_fps_, frame->interlaced_frame > 0); + } + + uint32_t calc_nb_frames(uint32_t nb_frames) const + { + uint64_t nb_frames2 = nb_frames; + + if(filter_ && filter_->is_double_rate()) // Take into account transformations in filter. + nb_frames2 *= 2; + + switch(display_mode_) // Take into account transformation in run. + { + case display_mode::deinterlace_bob_reinterlace: + case display_mode::interlace: + case display_mode::half: + nb_frames2 /= 2; + break; + case display_mode::duplicate: + nb_frames2 *= 2; + break; + } + + return static_cast(nb_frames2); + } + + void clear() + { + while(!video_stream_.empty()) + video_stream_.pop(); + + audio_stream_.clear(); + + while(!frame_buffer_.empty()) + frame_buffer_.pop(); + + filter_.reset(); + } +}; + +frame_muxer::frame_muxer( + double in_fps, + const spl::shared_ptr& frame_factory, + const core::video_format_desc& format_desc, + const core::audio_channel_layout& channel_layout, + const std::wstring& filter) + : impl_(new impl(in_fps, frame_factory, format_desc, channel_layout, filter)){} +void frame_muxer::push_video(const std::shared_ptr& frame){impl_->push_video(frame);} +void frame_muxer::push_audio(const std::shared_ptr& frame){impl_->push_audio(frame);} +bool frame_muxer::empty() const{return impl_->empty();} +core::draw_frame frame_muxer::front() const{return impl_->front();} +void frame_muxer::pop(){return impl_->pop();} +void frame_muxer::clear(){impl_->clear();} +uint32_t frame_muxer::calc_nb_frames(uint32_t nb_frames) const {return impl_->calc_nb_frames(nb_frames);} +bool frame_muxer::video_ready() const{return impl_->video_ready();} +bool frame_muxer::audio_ready() const{return impl_->audio_ready();} + }} \ No newline at end of file