From 70235ae09df45c874e133fd4d3fcc1e6e34e993a Mon Sep 17 00:00:00 2001 From: Helge Norberg Date: Fri, 9 Sep 2016 17:04:51 +0200 Subject: [PATCH] [ffmpeg] Copied flush logic when seeking from 2.0, as well as current frame in clip counter. Fixes audio drifting out of sync when looping --- .../ffmpeg_pipeline_backend_internal.cpp | 30 ++++++++++++++++--- .../ffmpeg/producer/audio/audio_decoder.cpp | 12 ++++---- modules/ffmpeg/producer/input/input.cpp | 11 +++++-- modules/ffmpeg/producer/util/util.cpp | 10 +++++++ modules/ffmpeg/producer/util/util.h | 2 ++ .../ffmpeg/producer/video/video_decoder.cpp | 30 +++++++++---------- 6 files changed, 67 insertions(+), 28 deletions(-) diff --git a/modules/ffmpeg/ffmpeg_pipeline_backend_internal.cpp b/modules/ffmpeg/ffmpeg_pipeline_backend_internal.cpp index 663d7dee3..9feb2d734 100644 --- a/modules/ffmpeg/ffmpeg_pipeline_backend_internal.cpp +++ b/modules/ffmpeg/ffmpeg_pipeline_backend_internal.cpp @@ -357,7 +357,7 @@ public: { frame = (*a_decoder)(); - if (frame && frame->data[0]) + if (frame == flush() || (frame && frame->data[0])) break; else frame.reset(); @@ -376,7 +376,7 @@ public: { frame = (*v_decoder)(); - if (frame && frame->data[0]) + if (frame == flush() || (frame && frame->data[0])) return { frame }; } } @@ -604,11 +604,12 @@ struct sink virtual void framerate(boost::rational framerate) { CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info(print() + L" not an encoder.")); } virtual void start(bool has_audio, bool has_video) { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); } virtual void stop() { } + virtual void flush_all() { } virtual std::vector supported_sample_formats() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); } virtual std::vector supported_samplerates() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); } virtual std::vector supported_pixel_formats() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); } virtual int wanted_num_audio_streams() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); } - virtual boost::optional wanted_num_channels_per_stream() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); } + virtual boost::optional wanted_num_channels_per_stream() const { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); } virtual boost::optional try_push(AVMediaType type, int stream_index, spl::shared_ptr frame) { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); } virtual void eof() { CASPAR_THROW_EXCEPTION(not_implemented() << msg_info(print())); } }; @@ -738,6 +739,14 @@ public: return boost::none; } + void flush_all() override + { + audio_samples_.clear(); + + while (!video_frames_.empty()) + video_frames_.pop(); + } + boost::optional try_push(AVMediaType type, int stream_index, spl::shared_ptr av_frame) override { if (!has_audio_ && !has_video_) @@ -1110,8 +1119,21 @@ private: { auto needed = *result; auto input_frames_for_streams = source_->get_input_frames_for_streams(needed); + bool flush_all = !input_frames_for_streams.empty() && input_frames_for_streams.at(0) == flush(); + + if (flush_all) + { + sink_->flush_all(); + + if (source_->has_audio() && source_->has_video()) + result = needed == AVMediaType::AVMEDIA_TYPE_AUDIO ? AVMediaType::AVMEDIA_TYPE_VIDEO : AVMediaType::AVMEDIA_TYPE_AUDIO; + + continue; + } + + bool got_requested_media_type = !input_frames_for_streams.empty() && input_frames_for_streams.at(0); - if (!input_frames_for_streams.empty() && input_frames_for_streams.at(0)) + if (got_requested_media_type) { for (int input_stream_index = 0; input_stream_index < input_frames_for_streams.size(); ++input_stream_index) { diff --git a/modules/ffmpeg/producer/audio/audio_decoder.cpp b/modules/ffmpeg/producer/audio/audio_decoder.cpp index 2366c10d8..e92a7c181 100644 --- a/modules/ffmpeg/producer/audio/audio_decoder.cpp +++ b/modules/ffmpeg/producer/audio/audio_decoder.cpp @@ -108,17 +108,17 @@ public: std::shared_ptr audio; - if(!current_packet_) - { - avcodec_flush_buffers(codec_context_.get()); - } - else if(!current_packet_->data) + if (!current_packet_->data) { if(codec_context_->codec->capabilities & CODEC_CAP_DELAY) audio = decode(*current_packet_); - if(!audio) + if (!audio) + { + avcodec_flush_buffers(codec_context_.get()); current_packet_.reset(); + audio = flush(); + } } else { diff --git a/modules/ffmpeg/producer/input/input.cpp b/modules/ffmpeg/producer/input/input.cpp index 13ca48d5e..c51cf152d 100644 --- a/modules/ffmpeg/producer/input/input.cpp +++ b/modules/ffmpeg/producer/input/input.cpp @@ -281,7 +281,7 @@ private: auto stream = format_context_->streams[default_stream_index_]; auto fps = read_fps(*format_context_, 0.0); - auto target_timestamp = static_cast((target / fps * stream->time_base.den) / stream->time_base.num); + auto target_timestamp = static_cast((target / fps * stream->time_base.den) / stream->time_base.num); THROW_ON_ERROR2(avformat_seek_file( format_context_.get(), @@ -290,11 +290,16 @@ private: target_timestamp, std::numeric_limits::max(), 0), print()); + + auto flush_packet = create_packet(); + flush_packet->data = nullptr; + flush_packet->size = 0; + flush_packet->pos = target; - video_stream_.push(nullptr); + video_stream_.push(flush_packet); for (auto& audio_stream : audio_streams_) - audio_stream.push(nullptr); + audio_stream.push(flush_packet); } void tick() diff --git a/modules/ffmpeg/producer/util/util.cpp b/modules/ffmpeg/producer/util/util.cpp index 3546305f5..eb773c6c1 100644 --- a/modules/ffmpeg/producer/util/util.cpp +++ b/modules/ffmpeg/producer/util/util.cpp @@ -491,6 +491,16 @@ spl::shared_ptr create_frame() return frame; } +std::shared_ptr flush() +{ + static std::shared_ptr dummy(av_frame_alloc(), [](AVFrame* p) + { + av_frame_free(&p); + }); + + return dummy; +} + spl::shared_ptr open_codec(AVFormatContext& context, enum AVMediaType type, int& index, bool single_threaded) { AVCodec* decoder; diff --git a/modules/ffmpeg/producer/util/util.h b/modules/ffmpeg/producer/util/util.h index 626cccfb4..abd167a01 100644 --- a/modules/ffmpeg/producer/util/util.h +++ b/modules/ffmpeg/producer/util/util.h @@ -71,6 +71,8 @@ spl::shared_ptr open_input(const std::wstring& filename); bool is_sane_fps(AVRational time_base); AVRational fix_time_base(AVRational time_base); +std::shared_ptr flush(); + double read_fps(AVFormatContext& context, double fail_value); boost::rational read_framerate(AVFormatContext& context, const boost::rational& fail_value); diff --git a/modules/ffmpeg/producer/video/video_decoder.cpp b/modules/ffmpeg/producer/video/video_decoder.cpp index 87ee744dc..bc379a8c0 100644 --- a/modules/ffmpeg/producer/video/video_decoder.cpp +++ b/modules/ffmpeg/producer/video/video_decoder.cpp @@ -34,6 +34,8 @@ #include +#include + #include #if defined(_MSC_VER) @@ -65,8 +67,8 @@ struct video_decoder::impl : boost::noncopyable const int width_; const int height_; - bool is_progressive_; - uint32_t file_frame_number_; + tbb::atomic is_progressive_; + tbb::atomic file_frame_number_; boost::rational framerate_; std::shared_ptr current_packet_; @@ -79,9 +81,10 @@ public: , nb_frames_(static_cast(stream_->nb_frames)) , width_(codec_context_->width) , height_(codec_context_->height) - , file_frame_number_(0) , framerate_(read_framerate(input_->context(), 0)) { + is_progressive_ = false; + file_frame_number_ = 0; } std::shared_ptr poll() @@ -91,17 +94,18 @@ public: std::shared_ptr frame; - if(!current_packet_) - { - avcodec_flush_buffers(codec_context_.get()); - } - else if(!current_packet_->data) + if (!current_packet_->data) { if(codec_context_->codec->capabilities & CODEC_CAP_DELAY) frame = decode(*current_packet_); - if(!frame) + if (!frame) + { + file_frame_number_ = static_cast(current_packet_->pos); + avcodec_flush_buffers(codec_context_.get()); current_packet_.reset(); + frame = flush(); + } } else { @@ -133,11 +137,7 @@ public: if(got_frame == 0) return nullptr; - auto stream_time_base = stream_->time_base; - auto fps = static_cast(framerate_.numerator()) / static_cast(framerate_.denominator()); - auto packet_frame_number = static_cast((static_cast(pkt.pts * stream_time_base.num) / stream_time_base.den) * fps); - - file_frame_number_ = packet_frame_number; + ++file_frame_number_; is_progressive_ = !frame->interlaced_frame; @@ -154,7 +154,7 @@ public: uint32_t nb_frames() const { - return std::max(nb_frames_, file_frame_number_); + return std::max(nb_frames_, static_cast(file_frame_number_)); } std::wstring print() const -- 2.39.2