2.0.0.2: ffmpeg_producer: Fixed problem with audio and video out of sync which was...

git-svn-id: https://casparcg.svn.sourceforge.net/svnroot/casparcg/server/branches/2.0.0.2@807 362d55ac-95cf-4e76-9f9a-cbaa9c17b72d

diff --git a/modules/ffmpeg/producer/audio/audio_decoder.cpp b/modules/ffmpeg/producer/audio/audio_decoder.cpp
index 20fb263fc0d2702741798d404b27569398c47ad9..bc8e2b76bdf0114df340788742f08103c187b8f3 100644 (file)
--- a/modules/ffmpeg/producer/audio/audio_decoder.cpp
+++ b/modules/ffmpeg/producer/audio/audio_decoder.cpp
@@ -40,18 +40,16 @@ namespace caspar {
 \r
 struct audio_decoder::implementation : boost::noncopyable\r
 {      \r
-       AVCodecContext& codec_context_;\r
-               \r
-       const core::video_format_desc format_desc_;\r
-\r
-       std::vector<short> current_chunk_;\r
-\r
-       std::vector<std::vector<short>> chunks_;\r
-\r
+       AVCodecContext&                                 codec_context_;         \r
+       const core::video_format_desc   format_desc_;\r
+       std::vector<short>                              current_chunk_;\r
+       std::vector<std::vector<short>> chunks_;                \r
+       size_t                                                  frame_number_;\r
 public:\r
        explicit implementation(AVCodecContext& codec_context, const core::video_format_desc& format_desc) \r
                : codec_context_(codec_context)\r
-               , format_desc_(format_desc)             \r
+               , format_desc_(format_desc)     \r
+               , frame_number_(0)\r
        {\r
                if(codec_context_.sample_rate != static_cast<int>(format_desc_.audio_sample_rate) || \r
                   codec_context_.channels != static_cast<int>(format_desc_.audio_channels))\r
@@ -63,20 +61,19 @@ public:
                                arg_name_info("codec_context"));\r
                }\r
        }\r
-               \r
-       void push(std::shared_ptr<AVPacket>&& audio_packet)\r
-       {       \r
+                       \r
+       void push(const std::shared_ptr<AVPacket>& audio_packet)\r
+       {                       \r
                if(!audio_packet)\r
-                       return;\r
-               \r
-               if(audio_packet->size == 0)\r
-               {\r
+               {       \r
                        avcodec_flush_buffers(&codec_context_);\r
+                       current_chunk_.clear();\r
+                       frame_number_ = 0;\r
                        return;\r
                }\r
-\r
+                               \r
                auto s = current_chunk_.size();\r
-               current_chunk_.resize(s + 4*format_desc_.audio_sample_rate*2+FF_INPUT_BUFFER_PADDING_SIZE/2);\r
+               current_chunk_.resize(s + 4*format_desc_.audio_sample_rate*2+FF_INPUT_BUFFER_PADDING_SIZE/2, 0);\r
                \r
                int written_bytes = (current_chunk_.size() - s)*2 - FF_INPUT_BUFFER_PADDING_SIZE;\r
                const int errn = avcodec_decode_audio3(&codec_context_, &current_chunk_[s], &written_bytes, audio_packet.get());\r
@@ -110,13 +107,15 @@ public:
 \r
        void pop()\r
        {\r
+               ++frame_number_;\r
                chunks_.pop_back();\r
        }\r
 };\r
 \r
 audio_decoder::audio_decoder(AVCodecContext& codec_context, const core::video_format_desc& format_desc) : impl_(new implementation(codec_context, format_desc)){}\r
-void audio_decoder::push(std::shared_ptr<AVPacket>&& audio_packet){impl_->push(std::move(audio_packet));}\r
+void audio_decoder::push(const std::shared_ptr<AVPacket>& audio_packet){impl_->push(std::move(audio_packet));}\r
 bool audio_decoder::empty() const {return impl_->empty();}\r
 std::vector<short> audio_decoder::front() {return impl_->front();}\r
 void audio_decoder::pop(){impl_->pop();}\r
+size_t audio_decoder::frame_number() const{return impl_->frame_number_;}\r
 }
\ No newline at end of file

diff --git a/modules/ffmpeg/producer/audio/audio_decoder.h b/modules/ffmpeg/producer/audio/audio_decoder.h
index 5291526ef04f68a6102d57f7529fd54f222ae09a..5d87dac72902cc63e93e179d7d58016128247d73 100644 (file)
--- a/modules/ffmpeg/producer/audio/audio_decoder.h
+++ b/modules/ffmpeg/producer/audio/audio_decoder.h
@@ -36,11 +36,13 @@ class audio_decoder : boost::noncopyable
 {\r
 public:\r
        explicit audio_decoder(AVCodecContext& codec_context, const core::video_format_desc& format_desc);\r
-       void push(std::shared_ptr<AVPacket>&& audio_packet);\r
-       \r
-       bool empty() const;\r
+\r
+       void push(const std::shared_ptr<AVPacket>& audio_packet);\r
        std::vector<short> front();\r
-       void pop();\r
+       void pop();     \r
+       bool empty() const;\r
+\r
+       size_t frame_number() const;\r
 private:\r
        struct implementation;\r
        std::shared_ptr<implementation> impl_;\r

diff --git a/modules/ffmpeg/producer/ffmpeg_producer.cpp b/modules/ffmpeg/producer/ffmpeg_producer.cpp
index 51ff7dfc568609c13df7ae69401915cba04e3c38..73a3d763c984463c1d9306fe4436d5924e01a632 100644 (file)
--- a/modules/ffmpeg/producer/ffmpeg_producer.cpp
+++ b/modules/ffmpeg/producer/ffmpeg_producer.cpp
@@ -25,6 +25,7 @@
 #include "audio/audio_decoder.h"\r
 #include "video/video_decoder.h"\r
 \r
+#include <common/utility/assert.h>\r
 #include <common/utility/timer.h>\r
 #include <common/diagnostics/graph.h>\r
 \r
@@ -62,7 +63,7 @@ public:
                : filename_(filename)\r
                , graph_(diagnostics::create_graph(narrow(print())))\r
                , frame_factory_(frame_factory)         \r
-               , input_(safe_ptr<diagnostics::graph>(graph_), filename_, loop, start)\r
+               , input_(safe_ptr<diagnostics::graph>(graph_), filename_, loop, start, length)\r
        {\r
                graph_->add_guide("frame-time", 0.5);\r
                graph_->set_color("frame-time",  diagnostics::color(1.0f, 0.0f, 0.0f));\r
@@ -105,17 +106,38 @@ public:
                (\r
                        [&]\r
                        {\r
+                               if(!video_decoder_)\r
+                                       return;\r
+\r
                                std::shared_ptr<AVPacket> pkt;\r
-                               for(int n = 0; n < 8 && video_decoder_ && video_decoder_->empty() && input_.try_pop_video_packet(pkt); ++n)\r
-                                       video_decoder_->push(std::move(pkt));\r
+                               for(int n = 0; n < 16 && video_decoder_->empty() && input_.try_pop_video_packet(pkt); ++n)                              \r
+                                       video_decoder_->push(pkt);                              \r
                        }, \r
                        [&]\r
                        {\r
-                               std::shared_ptr<AVPacket> pkt;\r
-                               for(int n = 0; n < 8 && audio_decoder_ && audio_decoder_->empty() && input_.try_pop_audio_packet(pkt); ++n)\r
-                                       audio_decoder_->push(std::move(pkt));\r
+                               if(!audio_decoder_)\r
+                                       return;\r
+                               \r
+                               std::shared_ptr<AVPacket> pkt;  \r
+                               for(int n = 0; n < 16 && audio_decoder_->empty() && input_.try_pop_audio_packet(pkt); ++n)                              \r
+                                       audio_decoder_->push(pkt);                              \r
+                       }\r
+               );\r
+               \r
+               // Sync up audio with video on start. Last frame sometimes have more samples than required. Remove those.\r
+               if(audio_decoder_ && video_decoder_ && video_decoder_->frame_number() == 0)\r
+               {\r
+                       std::shared_ptr<AVPacket> pkt;\r
+                       while(audio_decoder_->frame_number() > 0 && input_.try_pop_audio_packet(pkt))\r
+                       {\r
+                               audio_decoder_->push(pkt);\r
+                               if(audio_decoder_->frame_number() > 0)\r
+                                       audio_decoder_->pop();\r
                        }\r
-               );      \r
+                       \r
+                       for(int n = 0; n < 16 && audio_decoder_->empty() && input_.try_pop_audio_packet(pkt); ++n)                              \r
+                               audio_decoder_->push(pkt);              \r
+               }\r
        }\r
 \r
        safe_ptr<core::basic_frame> decode_frame()\r
@@ -129,7 +151,7 @@ public:
                                \r
                        frame->audio_data() = std::move(audio_decoder_->front());\r
                        audio_decoder_->pop();\r
-\r
+                       \r
                        return frame;\r
                }\r
                else if(video_decoder_ && !video_decoder_->empty() && !audio_decoder_)\r
@@ -137,7 +159,7 @@ public:
                        auto frame = std::move(video_decoder_->front());                                \r
                        video_decoder_->pop();\r
                        frame->get_audio_transform().set_has_audio(false);      \r
-\r
+                       \r
                        return frame;\r
                }\r
                else if(audio_decoder_ && !audio_decoder_->empty() && !video_decoder_)\r

diff --git a/modules/ffmpeg/producer/input.cpp b/modules/ffmpeg/producer/input.cpp
index 736e1cd1bd492e5a8419764d5019a882aaffa8d9..ce4b1b47ff42329948e57f6036972f3ae139a2be 100644 (file)
--- a/modules/ffmpeg/producer/input.cpp
+++ b/modules/ffmpeg/producer/input.cpp
@@ -57,12 +57,16 @@ class stream
 \r
 public:\r
 \r
-       stream()\r
-               : index_(-1)\r
+       stream() : index_(-1)\r
        {\r
                buffer_.set_capacity(PACKET_BUFFER_COUNT);\r
        }\r
 \r
+       ~stream()\r
+       {\r
+               CASPAR_LOG(trace) << "##: " << size();\r
+       }\r
+\r
        int open(std::shared_ptr<AVFormatContext>& fctx, AVMediaType media_type)\r
        {               \r
                const auto streams = boost::iterator_range<AVStream**>(fctx->streams, fctx->streams+fctx->nb_streams);\r
@@ -99,23 +103,17 @@ public:
 \r
        void push(const std::shared_ptr<AVPacket>& pkt)\r
        {\r
-               if(pkt->stream_index != index_)\r
+               if(pkt && pkt->stream_index != index_)\r
                        return;\r
 \r
-               av_dup_packet(pkt.get());\r
-               buffer_.push(pkt);      \r
-       }\r
-\r
-       void flush()\r
-       {\r
-               if(index_ == -1)\r
-                       return;\r
+               if(pkt)\r
+                       av_dup_packet(pkt.get());\r
 \r
-               std::shared_ptr<AVPacket> flsh_pkt(new AVPacket);\r
-               flsh_pkt->size = 0;\r
-               buffer_.push(flsh_pkt); \r
+               buffer_.push(pkt);      \r
        }\r
 \r
+       int index() const {return index_;}\r
+       \r
        const std::shared_ptr<AVCodecContext>& ctx() { return ctx_; }\r
 \r
        operator bool(){return ctx_ != nullptr;}\r
@@ -150,7 +148,8 @@ public:
                , executor_(print())\r
                , start_(std::max(start, 0))\r
        {               \r
-               graph_->set_color("input-buffer", diagnostics::color(1.0f, 1.0f, 0.0f));\r
+               graph_->set_color("audio-input-buffer", diagnostics::color(0.5f, 1.0f, 0.2f));\r
+               graph_->set_color("video-input-buffer", diagnostics::color(0.2f, 0.5f, 1.0f));\r
                graph_->set_color("seek", diagnostics::color(0.5f, 1.0f, 0.5f));        \r
                \r
                int errn;\r
@@ -215,14 +214,13 @@ public:
                stop();\r
        }\r
                \r
-\r
        bool try_pop_video_packet(std::shared_ptr<AVPacket>& packet)\r
        {\r
                return video_stream_.try_pop(packet);\r
        }\r
 \r
        bool try_pop_audio_packet(std::shared_ptr<AVPacket>& packet)\r
-       {\r
+       {       \r
                return audio_stream_.try_pop(packet);\r
        }\r
 \r
@@ -295,7 +293,8 @@ private:
                                audio_stream_.push(read_packet);\r
                        }\r
                                                \r
-                       graph_->update_value("input-buffer", static_cast<float>(std::max(video_stream_.size(), audio_stream_.size()))/static_cast<float>(PACKET_BUFFER_COUNT));         \r
+                       graph_->update_value("video-input-buffer", static_cast<float>(video_stream_.size())/static_cast<float>(PACKET_BUFFER_COUNT));           \r
+                       graph_->update_value("audio-input-buffer", static_cast<float>(audio_stream_.size())/static_cast<float>(PACKET_BUFFER_COUNT));           \r
                }\r
                catch(...)\r
                {\r
@@ -307,10 +306,17 @@ private:
 \r
        void seek_frame(int64_t frame, int flags = 0)\r
        {       \r
+               static const AVRational base_q = {1, AV_TIME_BASE};\r
+\r
                // Convert from frames into seconds.\r
-               const auto ts = frame*static_cast<int64_t>(AV_TIME_BASE/fps_);\r
+               auto seek_target = frame*static_cast<int64_t>(AV_TIME_BASE/fps_);\r
+\r
+               int stream_index = video_stream_.index() >= 0 ? video_stream_.index() : audio_stream_.index();\r
+\r
+               if(stream_index >= 0)           \r
+                       seek_target = av_rescale_q(seek_target, base_q, format_context_->streams[stream_index]->time_base);\r
 \r
-               const int errn = av_seek_frame(format_context_.get(), -1, ts, flags | AVSEEK_FLAG_FRAME);\r
+               const int errn = av_seek_frame(format_context_.get(), stream_index, seek_target, flags);\r
                if(errn < 0)\r
                {       \r
                        BOOST_THROW_EXCEPTION(\r
@@ -321,8 +327,8 @@ private:
                                boost::errinfo_errno(AVUNERROR(errn)));\r
                }\r
 \r
-               video_stream_.flush();\r
-               audio_stream_.flush();\r
+               video_stream_.push(nullptr);\r
+               audio_stream_.push(nullptr);\r
        }               \r
 \r
        bool is_eof(int errn)\r
@@ -339,7 +345,7 @@ private:
        }\r
 };\r
 \r
-input::input(const safe_ptr<diagnostics::graph>& graph, const std::wstring& filename, bool loop, int start) \r
+input::input(const safe_ptr<diagnostics::graph>& graph, const std::wstring& filename, bool loop, int start, int length) \r
        : impl_(new implementation(graph, filename, loop, start)){}\r
 const std::shared_ptr<AVCodecContext>& input::get_video_codec_context() const{return impl_->video_stream_.ctx();}\r
 const std::shared_ptr<AVCodecContext>& input::get_audio_codec_context() const{return impl_->audio_stream_.ctx();}\r

diff --git a/modules/ffmpeg/producer/input.h b/modules/ffmpeg/producer/input.h
index 587579fbd526e1f7b00624ad42c3e5178dea28ec..a6fd00a4d74a8583ae3fac1de2a5aebfb0f19ca1 100644 (file)
--- a/modules/ffmpeg/producer/input.h
+++ b/modules/ffmpeg/producer/input.h
@@ -24,6 +24,8 @@
 #include <memory>\r
 #include <string>\r
 \r
+#include <boost/iterator/iterator_facade.hpp>\r
+\r
 struct AVCodecContext;\r
 \r
 namespace caspar {\r
@@ -31,21 +33,49 @@ namespace caspar {
 class input : boost::noncopyable\r
 {\r
 public:\r
-       explicit input(const safe_ptr<diagnostics::graph>& graph, const std::wstring& filename, bool loop, int start);\r
+       explicit input(const safe_ptr<diagnostics::graph>& graph, const std::wstring& filename, bool loop, int start, int length);\r
        const std::shared_ptr<AVCodecContext>& get_video_codec_context() const;\r
        const std::shared_ptr<AVCodecContext>& get_audio_codec_context() const;\r
 \r
        bool try_pop_video_packet(std::shared_ptr<AVPacket>& packet);\r
        bool try_pop_audio_packet(std::shared_ptr<AVPacket>& packet);\r
 \r
-       bool has_video_packet() const;\r
-       bool has_audio_packet() const;\r
        bool is_running() const;\r
        double fps() const;\r
 private:\r
        struct implementation;\r
        std::shared_ptr<implementation> impl_;\r
 };\r
+//\r
+//class input_video_iterator : public boost::iterator_facade<input_video_iterator, std::shared_ptr<AVPacket>, boost::forward_traversal_tag>\r
+//{\r
+//     std::shared_ptr<AVPacket> pkt_;\r
+//     input* input_;\r
+//public:\r
+//     input_video_iterator() : input_(nullptr){}\r
+//\r
+//    input_video_iterator(input& input)\r
+//      : input_(&input) {}\r
+//\r
+//    input_video_iterator(const input_video_iterator& other)\r
+//      : input_(other.input_) {}\r
+//\r
+// private:\r
+//    friend class boost::iterator_core_access;\r
+//\r
+//    void increment() \r
+//     {\r
+//             if(input_ && !input_->try_pop_video_packet(pkt_))\r
+//                     input_ = nullptr;\r
+//     }\r
+//\r
+//    bool equal(input_video_iterator const& other) const\r
+//    {\r
+//        return input_ == other.input_;\r
+//    }\r
+//\r
+//    std::shared_ptr<AVPacket> const& dereference() const { return pkt_; }\r
+//};\r
 \r
        \r
 }\r

diff --git a/modules/ffmpeg/producer/video/video_decoder.cpp b/modules/ffmpeg/producer/video/video_decoder.cpp
index 7abedb6ec52ca80a8aa0b3d7c202da423a5342fb..b5980d8e4bfad7cb5702173186b0bceead115aee 100644 (file)
--- a/modules/ffmpeg/producer/video/video_decoder.cpp
+++ b/modules/ffmpeg/producer/video/video_decoder.cpp
@@ -125,6 +125,7 @@ struct video_decoder::implementation : boost::noncopyable
        const PixelFormat                                                       pix_fmt_;\r
        core::pixel_format_desc                                         desc_;\r
        std::vector<safe_ptr<core::write_frame>>        frames_;\r
+       size_t                                                                          frame_number_;\r
 \r
 public:\r
        explicit implementation(AVCodecContext& codec_context, const safe_ptr<core::frame_factory>& frame_factory) \r
@@ -134,6 +135,7 @@ public:
                , height_(codec_context_.height)\r
                , pix_fmt_(codec_context_.pix_fmt)\r
                , desc_(get_pixel_format_desc(pix_fmt_, width_, height_))\r
+               , frame_number_(0)\r
        {\r
                if(desc_.pix_fmt == core::pixel_format::invalid)\r
                {\r
@@ -148,18 +150,16 @@ public:
                                                                          boost::errinfo_api_function("sws_getContext"));\r
                }\r
        }\r
-       \r
-       void push(std::shared_ptr<AVPacket>&& video_packet)\r
+\r
+       void push(const std::shared_ptr<AVPacket>& video_packet)\r
        {                               \r
                if(!video_packet)\r
-                       return;\r
-\r
-               if(video_packet->size == 0)\r
-               {\r
+               {       \r
                        avcodec_flush_buffers(&codec_context_);\r
+                       frame_number_ = 0;\r
                        return;\r
                }\r
-       \r
+\r
                safe_ptr<AVFrame> decoded_frame(avcodec_alloc_frame(), av_free);\r
 \r
                int frame_finished = 0;\r
@@ -227,14 +227,16 @@ public:
 \r
        void pop()\r
        {\r
+               ++frame_number_;\r
                frames_.pop_back();\r
        }\r
 };\r
 \r
 video_decoder::video_decoder(AVCodecContext& codec_context, const safe_ptr<core::frame_factory>& frame_factory) : impl_(new implementation(codec_context, frame_factory)){}\r
-void video_decoder::push(std::shared_ptr<AVPacket>&& video_packet){impl_->push(std::move(video_packet));}\r
+void video_decoder::push(const std::shared_ptr<AVPacket>& video_packet){impl_->push(std::move(video_packet));}\r
 bool video_decoder::empty() const {return impl_->empty();}\r
 safe_ptr<core::write_frame> video_decoder::front() {return impl_->front();}\r
 void video_decoder::pop(){impl_->pop();}\r
+size_t video_decoder::frame_number() const{return impl_->frame_number_;}\r
 \r
 }
\ No newline at end of file

diff --git a/modules/ffmpeg/producer/video/video_decoder.h b/modules/ffmpeg/producer/video/video_decoder.h
index ddb9c94cf48f75bf8aed036eda368cc71c04e608..e7b1ad5b3f3f80f86734e3d0940e44befd895762 100644 (file)
--- a/modules/ffmpeg/producer/video/video_decoder.h
+++ b/modules/ffmpeg/producer/video/video_decoder.h
@@ -34,11 +34,14 @@ class video_decoder : boost::noncopyable
 {\r
 public:\r
        explicit video_decoder(AVCodecContext& codec_context, const safe_ptr<core::frame_factory>& frame_factory);\r
-       void push(std::shared_ptr<AVPacket>&& video_packet);    \r
+       void push(const std::shared_ptr<AVPacket>& video_packet);       \r
 \r
        bool empty() const;\r
        safe_ptr<core::write_frame> front();\r
        void pop();\r
+       \r
+       size_t frame_number() const;\r
+\r
 private:\r
        struct implementation;\r
        safe_ptr<implementation> impl_;\r

diff --git a/modules/ffmpeg/tbb_avcodec.cpp b/modules/ffmpeg/tbb_avcodec.cpp
index eb2841f8543ccd4c714d6054fde5701ae3d53c4a..203890da27663e554a945107ccfb66ee37b0531a 100644 (file)
--- a/modules/ffmpeg/tbb_avcodec.cpp
+++ b/modules/ffmpeg/tbb_avcodec.cpp
@@ -39,25 +39,23 @@ int thread_execute(AVCodecContext* s, int (*func)(AVCodecContext *c2, void *arg2
 \r
 int thread_execute2(AVCodecContext* s, int (*func)(AVCodecContext* c2, void* arg2, int, int), void* arg, int* ret, int count)\r
 {      \r
-    tbb::atomic<int> counter;\r
-       counter = 0;\r
-               \r
-       // Execute s->thread_count number of tasks in parallel.\r
-       tbb::parallel_for(0, s->thread_count, 1, [&](int threadnr) \r
-       {\r
-               while(true)\r
-               {\r
-                       int jobnr = counter++;\r
-                       if(jobnr >= count)\r
-                               break;\r
-\r
-                       int r = func(s, arg, jobnr, threadnr);\r
-                       if (ret)\r
-                               ret[jobnr] = r;\r
-               }\r
-       });\r
-\r
-       return 0;\r
+       tbb::atomic<int> counter;   \r
+    counter = 0;   \r
+\r
+       // Note: this will probably only work when tbb::task_scheduler_init::num_threads() < 16.\r
+    tbb::parallel_for(tbb::blocked_range<int>(0, count, 2), [&](const tbb::blocked_range<int> &r)    \r
+    {   \r
+        int threadnr = counter++;   \r
+        for(int jobnr = r.begin(); jobnr != r.end(); ++jobnr)\r
+        {   \r
+            int r = func(s, arg, jobnr, threadnr);   \r
+            if (ret)   \r
+                ret[jobnr] = r;   \r
+        }\r
+        --counter;\r
+    });   \r
+\r
+    return 0;  \r
 }\r
 \r
 void thread_init(AVCodecContext* s)\r

diff --git a/shell/main.cpp b/shell/main.cpp
index dab8ac3a78a7474b192dac9af056b6843f5508b5..757e8f49e64838a8e53933b7aff99be4cc42419d 100644 (file)
--- a/shell/main.cpp
+++ b/shell/main.cpp
@@ -81,7 +81,7 @@ void setup_console_window()
 \r
        SMALL_RECT DisplayArea = {0, 0, 0, 0};\r
        DisplayArea.Right = coord.X-1;\r
-       DisplayArea.Bottom = coord.Y-1;\r
+       DisplayArea.Bottom = (coord.Y-1)/2;\r
        SetConsoleWindowInfo(hOut, TRUE, &DisplayArea);\r
                \r
        // Set console title.\r