From: ronag Date: Tue, 24 May 2011 23:05:10 +0000 (+0000) Subject: 2.0.0.2: ffmpeg_producer: Fixed problem with audio and video out of sync which was... X-Git-Tag: 2.0.1~500 X-Git-Url: https://git.sesse.net/?a=commitdiff_plain;h=7f97f2ce5fd102061355b24a99f720173f57042a;p=casparcg 2.0.0.2: ffmpeg_producer: Fixed problem with audio and video out of sync which was caused by uneccessary audio samples at end of video. git-svn-id: https://casparcg.svn.sourceforge.net/svnroot/casparcg/server/branches/2.0.0.2@807 362d55ac-95cf-4e76-9f9a-cbaa9c17b72d --- diff --git a/modules/ffmpeg/producer/audio/audio_decoder.cpp b/modules/ffmpeg/producer/audio/audio_decoder.cpp index 20fb263fc..bc8e2b76b 100644 --- a/modules/ffmpeg/producer/audio/audio_decoder.cpp +++ b/modules/ffmpeg/producer/audio/audio_decoder.cpp @@ -40,18 +40,16 @@ namespace caspar { struct audio_decoder::implementation : boost::noncopyable { - AVCodecContext& codec_context_; - - const core::video_format_desc format_desc_; - - std::vector current_chunk_; - - std::vector> chunks_; - + AVCodecContext& codec_context_; + const core::video_format_desc format_desc_; + std::vector current_chunk_; + std::vector> chunks_; + size_t frame_number_; public: explicit implementation(AVCodecContext& codec_context, const core::video_format_desc& format_desc) : codec_context_(codec_context) - , format_desc_(format_desc) + , format_desc_(format_desc) + , frame_number_(0) { if(codec_context_.sample_rate != static_cast(format_desc_.audio_sample_rate) || codec_context_.channels != static_cast(format_desc_.audio_channels)) @@ -63,20 +61,19 @@ public: arg_name_info("codec_context")); } } - - void push(std::shared_ptr&& audio_packet) - { + + void push(const std::shared_ptr& audio_packet) + { if(!audio_packet) - return; - - if(audio_packet->size == 0) - { + { avcodec_flush_buffers(&codec_context_); + current_chunk_.clear(); + frame_number_ = 0; return; } - + auto s = current_chunk_.size(); - current_chunk_.resize(s + 4*format_desc_.audio_sample_rate*2+FF_INPUT_BUFFER_PADDING_SIZE/2); + current_chunk_.resize(s + 4*format_desc_.audio_sample_rate*2+FF_INPUT_BUFFER_PADDING_SIZE/2, 0); int written_bytes = (current_chunk_.size() - s)*2 - FF_INPUT_BUFFER_PADDING_SIZE; const int errn = avcodec_decode_audio3(&codec_context_, ¤t_chunk_[s], &written_bytes, audio_packet.get()); @@ -110,13 +107,15 @@ public: void pop() { + ++frame_number_; chunks_.pop_back(); } }; audio_decoder::audio_decoder(AVCodecContext& codec_context, const core::video_format_desc& format_desc) : impl_(new implementation(codec_context, format_desc)){} -void audio_decoder::push(std::shared_ptr&& audio_packet){impl_->push(std::move(audio_packet));} +void audio_decoder::push(const std::shared_ptr& audio_packet){impl_->push(std::move(audio_packet));} bool audio_decoder::empty() const {return impl_->empty();} std::vector audio_decoder::front() {return impl_->front();} void audio_decoder::pop(){impl_->pop();} +size_t audio_decoder::frame_number() const{return impl_->frame_number_;} } \ No newline at end of file diff --git a/modules/ffmpeg/producer/audio/audio_decoder.h b/modules/ffmpeg/producer/audio/audio_decoder.h index 5291526ef..5d87dac72 100644 --- a/modules/ffmpeg/producer/audio/audio_decoder.h +++ b/modules/ffmpeg/producer/audio/audio_decoder.h @@ -36,11 +36,13 @@ class audio_decoder : boost::noncopyable { public: explicit audio_decoder(AVCodecContext& codec_context, const core::video_format_desc& format_desc); - void push(std::shared_ptr&& audio_packet); - - bool empty() const; + + void push(const std::shared_ptr& audio_packet); std::vector front(); - void pop(); + void pop(); + bool empty() const; + + size_t frame_number() const; private: struct implementation; std::shared_ptr impl_; diff --git a/modules/ffmpeg/producer/ffmpeg_producer.cpp b/modules/ffmpeg/producer/ffmpeg_producer.cpp index 51ff7dfc5..73a3d763c 100644 --- a/modules/ffmpeg/producer/ffmpeg_producer.cpp +++ b/modules/ffmpeg/producer/ffmpeg_producer.cpp @@ -25,6 +25,7 @@ #include "audio/audio_decoder.h" #include "video/video_decoder.h" +#include #include #include @@ -62,7 +63,7 @@ public: : filename_(filename) , graph_(diagnostics::create_graph(narrow(print()))) , frame_factory_(frame_factory) - , input_(safe_ptr(graph_), filename_, loop, start) + , input_(safe_ptr(graph_), filename_, loop, start, length) { graph_->add_guide("frame-time", 0.5); graph_->set_color("frame-time", diagnostics::color(1.0f, 0.0f, 0.0f)); @@ -105,17 +106,38 @@ public: ( [&] { + if(!video_decoder_) + return; + std::shared_ptr pkt; - for(int n = 0; n < 8 && video_decoder_ && video_decoder_->empty() && input_.try_pop_video_packet(pkt); ++n) - video_decoder_->push(std::move(pkt)); + for(int n = 0; n < 16 && video_decoder_->empty() && input_.try_pop_video_packet(pkt); ++n) + video_decoder_->push(pkt); }, [&] { - std::shared_ptr pkt; - for(int n = 0; n < 8 && audio_decoder_ && audio_decoder_->empty() && input_.try_pop_audio_packet(pkt); ++n) - audio_decoder_->push(std::move(pkt)); + if(!audio_decoder_) + return; + + std::shared_ptr pkt; + for(int n = 0; n < 16 && audio_decoder_->empty() && input_.try_pop_audio_packet(pkt); ++n) + audio_decoder_->push(pkt); + } + ); + + // Sync up audio with video on start. Last frame sometimes have more samples than required. Remove those. + if(audio_decoder_ && video_decoder_ && video_decoder_->frame_number() == 0) + { + std::shared_ptr pkt; + while(audio_decoder_->frame_number() > 0 && input_.try_pop_audio_packet(pkt)) + { + audio_decoder_->push(pkt); + if(audio_decoder_->frame_number() > 0) + audio_decoder_->pop(); } - ); + + for(int n = 0; n < 16 && audio_decoder_->empty() && input_.try_pop_audio_packet(pkt); ++n) + audio_decoder_->push(pkt); + } } safe_ptr decode_frame() @@ -129,7 +151,7 @@ public: frame->audio_data() = std::move(audio_decoder_->front()); audio_decoder_->pop(); - + return frame; } else if(video_decoder_ && !video_decoder_->empty() && !audio_decoder_) @@ -137,7 +159,7 @@ public: auto frame = std::move(video_decoder_->front()); video_decoder_->pop(); frame->get_audio_transform().set_has_audio(false); - + return frame; } else if(audio_decoder_ && !audio_decoder_->empty() && !video_decoder_) diff --git a/modules/ffmpeg/producer/input.cpp b/modules/ffmpeg/producer/input.cpp index 736e1cd1b..ce4b1b47f 100644 --- a/modules/ffmpeg/producer/input.cpp +++ b/modules/ffmpeg/producer/input.cpp @@ -57,12 +57,16 @@ class stream public: - stream() - : index_(-1) + stream() : index_(-1) { buffer_.set_capacity(PACKET_BUFFER_COUNT); } + ~stream() + { + CASPAR_LOG(trace) << "##: " << size(); + } + int open(std::shared_ptr& fctx, AVMediaType media_type) { const auto streams = boost::iterator_range(fctx->streams, fctx->streams+fctx->nb_streams); @@ -99,23 +103,17 @@ public: void push(const std::shared_ptr& pkt) { - if(pkt->stream_index != index_) + if(pkt && pkt->stream_index != index_) return; - av_dup_packet(pkt.get()); - buffer_.push(pkt); - } - - void flush() - { - if(index_ == -1) - return; + if(pkt) + av_dup_packet(pkt.get()); - std::shared_ptr flsh_pkt(new AVPacket); - flsh_pkt->size = 0; - buffer_.push(flsh_pkt); + buffer_.push(pkt); } + int index() const {return index_;} + const std::shared_ptr& ctx() { return ctx_; } operator bool(){return ctx_ != nullptr;} @@ -150,7 +148,8 @@ public: , executor_(print()) , start_(std::max(start, 0)) { - graph_->set_color("input-buffer", diagnostics::color(1.0f, 1.0f, 0.0f)); + graph_->set_color("audio-input-buffer", diagnostics::color(0.5f, 1.0f, 0.2f)); + graph_->set_color("video-input-buffer", diagnostics::color(0.2f, 0.5f, 1.0f)); graph_->set_color("seek", diagnostics::color(0.5f, 1.0f, 0.5f)); int errn; @@ -215,14 +214,13 @@ public: stop(); } - bool try_pop_video_packet(std::shared_ptr& packet) { return video_stream_.try_pop(packet); } bool try_pop_audio_packet(std::shared_ptr& packet) - { + { return audio_stream_.try_pop(packet); } @@ -295,7 +293,8 @@ private: audio_stream_.push(read_packet); } - graph_->update_value("input-buffer", static_cast(std::max(video_stream_.size(), audio_stream_.size()))/static_cast(PACKET_BUFFER_COUNT)); + graph_->update_value("video-input-buffer", static_cast(video_stream_.size())/static_cast(PACKET_BUFFER_COUNT)); + graph_->update_value("audio-input-buffer", static_cast(audio_stream_.size())/static_cast(PACKET_BUFFER_COUNT)); } catch(...) { @@ -307,10 +306,17 @@ private: void seek_frame(int64_t frame, int flags = 0) { + static const AVRational base_q = {1, AV_TIME_BASE}; + // Convert from frames into seconds. - const auto ts = frame*static_cast(AV_TIME_BASE/fps_); + auto seek_target = frame*static_cast(AV_TIME_BASE/fps_); + + int stream_index = video_stream_.index() >= 0 ? video_stream_.index() : audio_stream_.index(); + + if(stream_index >= 0) + seek_target = av_rescale_q(seek_target, base_q, format_context_->streams[stream_index]->time_base); - const int errn = av_seek_frame(format_context_.get(), -1, ts, flags | AVSEEK_FLAG_FRAME); + const int errn = av_seek_frame(format_context_.get(), stream_index, seek_target, flags); if(errn < 0) { BOOST_THROW_EXCEPTION( @@ -321,8 +327,8 @@ private: boost::errinfo_errno(AVUNERROR(errn))); } - video_stream_.flush(); - audio_stream_.flush(); + video_stream_.push(nullptr); + audio_stream_.push(nullptr); } bool is_eof(int errn) @@ -339,7 +345,7 @@ private: } }; -input::input(const safe_ptr& graph, const std::wstring& filename, bool loop, int start) +input::input(const safe_ptr& graph, const std::wstring& filename, bool loop, int start, int length) : impl_(new implementation(graph, filename, loop, start)){} const std::shared_ptr& input::get_video_codec_context() const{return impl_->video_stream_.ctx();} const std::shared_ptr& input::get_audio_codec_context() const{return impl_->audio_stream_.ctx();} diff --git a/modules/ffmpeg/producer/input.h b/modules/ffmpeg/producer/input.h index 587579fbd..a6fd00a4d 100644 --- a/modules/ffmpeg/producer/input.h +++ b/modules/ffmpeg/producer/input.h @@ -24,6 +24,8 @@ #include #include +#include + struct AVCodecContext; namespace caspar { @@ -31,21 +33,49 @@ namespace caspar { class input : boost::noncopyable { public: - explicit input(const safe_ptr& graph, const std::wstring& filename, bool loop, int start); + explicit input(const safe_ptr& graph, const std::wstring& filename, bool loop, int start, int length); const std::shared_ptr& get_video_codec_context() const; const std::shared_ptr& get_audio_codec_context() const; bool try_pop_video_packet(std::shared_ptr& packet); bool try_pop_audio_packet(std::shared_ptr& packet); - bool has_video_packet() const; - bool has_audio_packet() const; bool is_running() const; double fps() const; private: struct implementation; std::shared_ptr impl_; }; +// +//class input_video_iterator : public boost::iterator_facade, boost::forward_traversal_tag> +//{ +// std::shared_ptr pkt_; +// input* input_; +//public: +// input_video_iterator() : input_(nullptr){} +// +// input_video_iterator(input& input) +// : input_(&input) {} +// +// input_video_iterator(const input_video_iterator& other) +// : input_(other.input_) {} +// +// private: +// friend class boost::iterator_core_access; +// +// void increment() +// { +// if(input_ && !input_->try_pop_video_packet(pkt_)) +// input_ = nullptr; +// } +// +// bool equal(input_video_iterator const& other) const +// { +// return input_ == other.input_; +// } +// +// std::shared_ptr const& dereference() const { return pkt_; } +//}; } diff --git a/modules/ffmpeg/producer/video/video_decoder.cpp b/modules/ffmpeg/producer/video/video_decoder.cpp index 7abedb6ec..b5980d8e4 100644 --- a/modules/ffmpeg/producer/video/video_decoder.cpp +++ b/modules/ffmpeg/producer/video/video_decoder.cpp @@ -125,6 +125,7 @@ struct video_decoder::implementation : boost::noncopyable const PixelFormat pix_fmt_; core::pixel_format_desc desc_; std::vector> frames_; + size_t frame_number_; public: explicit implementation(AVCodecContext& codec_context, const safe_ptr& frame_factory) @@ -134,6 +135,7 @@ public: , height_(codec_context_.height) , pix_fmt_(codec_context_.pix_fmt) , desc_(get_pixel_format_desc(pix_fmt_, width_, height_)) + , frame_number_(0) { if(desc_.pix_fmt == core::pixel_format::invalid) { @@ -148,18 +150,16 @@ public: boost::errinfo_api_function("sws_getContext")); } } - - void push(std::shared_ptr&& video_packet) + + void push(const std::shared_ptr& video_packet) { if(!video_packet) - return; - - if(video_packet->size == 0) - { + { avcodec_flush_buffers(&codec_context_); + frame_number_ = 0; return; } - + safe_ptr decoded_frame(avcodec_alloc_frame(), av_free); int frame_finished = 0; @@ -227,14 +227,16 @@ public: void pop() { + ++frame_number_; frames_.pop_back(); } }; video_decoder::video_decoder(AVCodecContext& codec_context, const safe_ptr& frame_factory) : impl_(new implementation(codec_context, frame_factory)){} -void video_decoder::push(std::shared_ptr&& video_packet){impl_->push(std::move(video_packet));} +void video_decoder::push(const std::shared_ptr& video_packet){impl_->push(std::move(video_packet));} bool video_decoder::empty() const {return impl_->empty();} safe_ptr video_decoder::front() {return impl_->front();} void video_decoder::pop(){impl_->pop();} +size_t video_decoder::frame_number() const{return impl_->frame_number_;} } \ No newline at end of file diff --git a/modules/ffmpeg/producer/video/video_decoder.h b/modules/ffmpeg/producer/video/video_decoder.h index ddb9c94cf..e7b1ad5b3 100644 --- a/modules/ffmpeg/producer/video/video_decoder.h +++ b/modules/ffmpeg/producer/video/video_decoder.h @@ -34,11 +34,14 @@ class video_decoder : boost::noncopyable { public: explicit video_decoder(AVCodecContext& codec_context, const safe_ptr& frame_factory); - void push(std::shared_ptr&& video_packet); + void push(const std::shared_ptr& video_packet); bool empty() const; safe_ptr front(); void pop(); + + size_t frame_number() const; + private: struct implementation; safe_ptr impl_; diff --git a/modules/ffmpeg/tbb_avcodec.cpp b/modules/ffmpeg/tbb_avcodec.cpp index eb2841f85..203890da2 100644 --- a/modules/ffmpeg/tbb_avcodec.cpp +++ b/modules/ffmpeg/tbb_avcodec.cpp @@ -39,25 +39,23 @@ int thread_execute(AVCodecContext* s, int (*func)(AVCodecContext *c2, void *arg2 int thread_execute2(AVCodecContext* s, int (*func)(AVCodecContext* c2, void* arg2, int, int), void* arg, int* ret, int count) { - tbb::atomic counter; - counter = 0; - - // Execute s->thread_count number of tasks in parallel. - tbb::parallel_for(0, s->thread_count, 1, [&](int threadnr) - { - while(true) - { - int jobnr = counter++; - if(jobnr >= count) - break; - - int r = func(s, arg, jobnr, threadnr); - if (ret) - ret[jobnr] = r; - } - }); - - return 0; + tbb::atomic counter; + counter = 0; + + // Note: this will probably only work when tbb::task_scheduler_init::num_threads() < 16. + tbb::parallel_for(tbb::blocked_range(0, count, 2), [&](const tbb::blocked_range &r) + { + int threadnr = counter++; + for(int jobnr = r.begin(); jobnr != r.end(); ++jobnr) + { + int r = func(s, arg, jobnr, threadnr); + if (ret) + ret[jobnr] = r; + } + --counter; + }); + + return 0; } void thread_init(AVCodecContext* s) diff --git a/shell/main.cpp b/shell/main.cpp index dab8ac3a7..757e8f49e 100644 --- a/shell/main.cpp +++ b/shell/main.cpp @@ -81,7 +81,7 @@ void setup_console_window() SMALL_RECT DisplayArea = {0, 0, 0, 0}; DisplayArea.Right = coord.X-1; - DisplayArea.Bottom = coord.Y-1; + DisplayArea.Bottom = (coord.Y-1)/2; SetConsoleWindowInfo(hOut, TRUE, &DisplayArea); // Set console title.