Implemented support for multiple audio streams in input and audio_decoder.

diff --git a/modules/ffmpeg/producer/audio/audio_decoder.cpp b/modules/ffmpeg/producer/audio/audio_decoder.cpp
index 64930e77dc633de2d7262ad66da0cf64b69d16eb..7f6dc59f6899821b853815dacd6a082841801be0 100644 (file)
--- a/modules/ffmpeg/producer/audio/audio_decoder.cpp
+++ b/modules/ffmpeg/producer/audio/audio_decoder.cpp
@@ -62,8 +62,9 @@ struct audio_decoder::impl : boost::noncopyable
        core::monitor::subject                                                                          monitor_subject_;
        input&                                                                                                          input_;
        int                                                                                                                     index_;
+       int                                                                                                                     actual_index_;
        const core::video_format_desc                                                           format_desc_;
-       const spl::shared_ptr<AVCodecContext>                                           codec_context_          = open_codec(input_.context(), AVMEDIA_TYPE_AUDIO, index_, false);
+       const spl::shared_ptr<AVCodecContext>                                           codec_context_          = open_codec(input_.context(), AVMEDIA_TYPE_AUDIO, actual_index_, false);
 
        std::shared_ptr<SwrContext>                                                                     swr_                            {
                                                                                                                                                                                swr_alloc_set_opts(
@@ -85,8 +86,14 @@ struct audio_decoder::impl : boost::noncopyable
        std::shared_ptr<AVPacket>                                                                       current_packet_;
        
 public:
-       explicit impl(input& in, const core::video_format_desc& format_desc, const std::wstring& channel_layout_spec) 
+       explicit impl(
+                       input& in,
+                       const core::video_format_desc& format_desc,
+                       const std::wstring& channel_layout_spec,
+                       int audio_stream_index)
                : input_(in)
+               , index_(audio_stream_index)
+               , actual_index_(input_.get_actual_audio_stream_index(index_))
                , format_desc_(format_desc)
                , buffer_(480000 * 4)
                , channel_layout_(get_audio_channel_layout(
@@ -102,7 +109,7 @@ public:
                
        std::shared_ptr<AVFrame> poll()
        {               
-               if(!current_packet_ && !input_.try_pop_audio(current_packet_))
+               if(!current_packet_ && !input_.try_pop_audio(current_packet_, index_))
                        return nullptr;
                
                std::shared_ptr<AVFrame> audio;
@@ -180,7 +187,7 @@ public:
        }
 };
 
-audio_decoder::audio_decoder(input& input, const core::video_format_desc& format_desc, const std::wstring& channel_layout_spec) : impl_(new impl(input, format_desc, channel_layout_spec)){}
+audio_decoder::audio_decoder(input& input, const core::video_format_desc& format_desc, const std::wstring& channel_layout_spec, int audio_stream_index) : impl_(new impl(input, format_desc, channel_layout_spec, audio_stream_index)){}
 audio_decoder::audio_decoder(audio_decoder&& other) : impl_(std::move(other.impl_)){}
 audio_decoder& audio_decoder::operator=(audio_decoder&& other){impl_ = std::move(other.impl_); return *this;}
 std::shared_ptr<AVFrame> audio_decoder::operator()(){return impl_->poll();}

diff --git a/modules/ffmpeg/producer/audio/audio_decoder.h b/modules/ffmpeg/producer/audio/audio_decoder.h
index 31eb0e5157eea7cb8a9c69dde7c8a006dab8d7de..a1ed375067d2977021eb288b6854232250e321db 100644 (file)
--- a/modules/ffmpeg/producer/audio/audio_decoder.h
+++ b/modules/ffmpeg/producer/audio/audio_decoder.h
@@ -38,7 +38,7 @@ namespace caspar { namespace ffmpeg {
 class audio_decoder : public boost::noncopyable
 {
 public:
-       explicit audio_decoder(class input& input, const core::video_format_desc& format_desc, const std::wstring& channel_layout_spec);
+       explicit audio_decoder(class input& input, const core::video_format_desc& format_desc, const std::wstring& channel_layout_spec, int audio_stream_index = 0);
        
        audio_decoder(audio_decoder&& other);
        audio_decoder& operator=(audio_decoder&& other);

diff --git a/modules/ffmpeg/producer/input/input.cpp b/modules/ffmpeg/producer/input/input.cpp
index d196752ff3a16ac7914bd379855a72a374a0d4c1..74b122a93431eff0956f7c48866e3c200c417b93 100644 (file)
--- a/modules/ffmpeg/producer/input/input.cpp
+++ b/modules/ffmpeg/producer/input/input.cpp
@@ -60,6 +60,7 @@ extern "C"
 
 namespace caspar { namespace ffmpeg {
 
+static const int MAX_PUSH_WITHOUT_POP = 200;
 static const int MIN_FRAMES = 25;
 
 class stream
@@ -69,36 +70,53 @@ class stream
 
        typedef tbb::concurrent_bounded_queue<std::shared_ptr<AVPacket>>::size_type size_type;
 
-       int                                                                                                              index_;
-       tbb::concurrent_bounded_queue<std::shared_ptr<AVPacket>> packets_;
+       int                                                                                                                     index_;
+       tbb::concurrent_bounded_queue<std::shared_ptr<AVPacket>>        packets_;
+       tbb::atomic<int>                                                                                        push_since_pop_;
 public:
 
        stream(int index) 
                : index_(index)
        {
+               push_since_pop_ = 0;
        }
 
+       stream(stream&&) = default;
+
        bool is_available() const
        {
                return index_ >= 0;
        }
+
+       int index() const
+       {
+               return index_;
+       }
        
        void push(const std::shared_ptr<AVPacket>& packet)
        {
                if(packet && packet->data && packet->stream_index != index_)
                        return;
 
+               if (++push_since_pop_ > MAX_PUSH_WITHOUT_POP) // Out of memory protection for streams never being used.
+               {
+                       return;
+               }
+
                packets_.push(packet);
        }
 
        bool try_pop(std::shared_ptr<AVPacket>& packet)
        {
+               push_since_pop_ = 0;
+
                return packets_.try_pop(packet);
        }
 
        void clear()
        {
                std::shared_ptr<AVPacket> packet;
+               push_since_pop_ = 0;
                while(packets_.try_pop(packet));
        }
                
@@ -125,7 +143,7 @@ struct input::impl : boost::noncopyable
        uint32_t                                                                        frame_number_                   = 0;
 
        stream                                                                          video_stream_                   {                                                       av_find_best_stream(format_context_.get(), AVMEDIA_TYPE_VIDEO, -1, -1, 0, 0) };
-       stream                                                                          audio_stream_                   { thumbnail_mode_ ? -1 :        av_find_best_stream(format_context_.get(), AVMEDIA_TYPE_AUDIO, -1, -1, 0, 0) };
+       std::vector<stream>                                                     audio_streams_;
 
        boost::optional<uint32_t>                                       seek_target_;
 
@@ -156,8 +174,13 @@ struct input::impl : boost::noncopyable
                                                                                                                
                graph_->set_color("seek", diagnostics::color(1.0f, 0.5f, 0.0f));
 
-               if (audio_stream_.is_available())
-                       graph_->set_color("audio-buffer", diagnostics::color(0.7f, 0.4f, 0.4f));
+               if (!thumbnail_mode_)
+                       for (unsigned i = 0; i < format_context_->nb_streams; ++i)
+                               if (format_context_->streams[i]->codec->codec_type == AVMediaType::AVMEDIA_TYPE_AUDIO)
+                                       audio_streams_.emplace_back(i);
+
+               for (int i = 0; i < audio_streams_.size(); ++i)
+                       graph_->set_color("audio-buffer" + boost::lexical_cast<std::string>(i + 1), diagnostics::color(0.7f, 0.4f, 0.4f));
 
                if (video_stream_.is_available())
                        graph_->set_color("video-buffer", diagnostics::color(1.0f, 1.0f, 0.0f));
@@ -209,16 +232,18 @@ struct input::impl : boost::noncopyable
                return result;
        }
        
-       bool try_pop_audio(std::shared_ptr<AVPacket>& packet)
+       bool try_pop_audio(std::shared_ptr<AVPacket>& packet, int audio_stream_index)
        {
-               if (!audio_stream_.is_available())
+               if (audio_streams_.size() < audio_stream_index + 1)
                        return false;
 
-               bool result = audio_stream_.try_pop(packet);
+               auto& audio_stream = audio_streams_.at(audio_stream_index);
+               bool result = audio_stream.try_pop(packet);
                if(result)
                        cond_.notify_one();
-                               
-               graph_->set_value("audio-buffer", std::min(1.0, static_cast<double>(audio_stream_.size())/MIN_FRAMES));
+
+               auto buffer_nr = boost::lexical_cast<std::string>(audio_stream_index + 1);
+               graph_->set_value("audio-buffer" + buffer_nr, std::min(1.0, static_cast<double>(audio_stream.size())/MIN_FRAMES));
 
                return result;
        }
@@ -230,11 +255,21 @@ struct input::impl : boost::noncopyable
 
                        seek_target_ = target;
                        video_stream_.clear();
-                       audio_stream_.clear();
+
+                       for (auto& audio_stream : audio_streams_)
+                               audio_stream.clear();
                }
 
                cond_.notify_one();
        }
+
+       int get_actual_audio_stream_index(int audio_stream_index) const
+       {
+               if (audio_stream_index + 1 > audio_streams_.size())
+                       CASPAR_THROW_EXCEPTION(averror_stream_not_found());
+
+               return audio_streams_.at(audio_stream_index).index();
+       }
                
        std::wstring print() const
        {
@@ -278,7 +313,9 @@ private:
                                0), print());
                
                video_stream_.push(nullptr);
-               audio_stream_.push(nullptr);
+
+               for (auto& audio_stream : audio_streams_)
+                       audio_stream.push(nullptr);
        }
 
        void tick()
@@ -327,20 +364,33 @@ private:
                        if(packet_frame_number >= start_ && packet_frame_number < length_)
                        {
                                video_stream_.push(packet);
-                               audio_stream_.push(packet);
+
+                               for (auto& audio_stream : audio_streams_)
+                                       audio_stream.push(packet);
                        }
                }       
 
                if (video_stream_.is_available())
                        graph_->set_value("video-buffer", std::min(1.0, static_cast<double>(video_stream_.size())/MIN_FRAMES));
 
-               if (audio_stream_.is_available())
-                       graph_->set_value("audio-buffer", std::min(1.0, static_cast<double>(audio_stream_.size())/MIN_FRAMES));
+               for (int i = 0; i < audio_streams_.size(); ++i)
+                       graph_->set_value(
+                                       "audio-buffer" + boost::lexical_cast<std::string>(i + 1),
+                                       std::min(1.0, static_cast<double>(audio_streams_[i].size())/MIN_FRAMES));
        }
                        
        bool full() const
        {
-               return video_stream_.size() >= MIN_FRAMES && audio_stream_.size() >= MIN_FRAMES;
+               bool video_full = video_stream_.size() >= MIN_FRAMES;
+
+               if (!video_full)
+                       return false;
+
+               for (auto& audio_stream : audio_streams_)
+                       if (audio_stream.size() < MIN_FRAMES)
+                               return false;
+
+               return true;
        }
 
        void run()
@@ -378,8 +428,10 @@ private:
 
 input::input(const spl::shared_ptr<diagnostics::graph>& graph, const std::wstring& filename, bool loop, uint32_t start, uint32_t length, bool thumbnail_mode)
        : impl_(new impl(graph, filename, loop, start, length, thumbnail_mode)){}
+int input::get_actual_audio_stream_index(int audio_stream_index) const { return impl_->get_actual_audio_stream_index(audio_stream_index); };
+int input::num_audio_streams() const { return static_cast<int>(impl_->audio_streams_.size()); }
 bool input::try_pop_video(std::shared_ptr<AVPacket>& packet){return impl_->try_pop_video(packet);}
-bool input::try_pop_audio(std::shared_ptr<AVPacket>& packet){return impl_->try_pop_audio(packet);}
+bool input::try_pop_audio(std::shared_ptr<AVPacket>& packet, int audio_stream_index){return impl_->try_pop_audio(packet, audio_stream_index);}
 AVFormatContext& input::context(){return *impl_->format_context_;}
 void input::loop(bool value){impl_->loop_ = value;}
 bool input::loop() const{return impl_->loop_;}

diff --git a/modules/ffmpeg/producer/input/input.h b/modules/ffmpeg/producer/input/input.h
index a4a7dbb23a732526a976e34b1b5b7e2094dc0067..5f843b9d11770c8973a151cf7a695c6baa52dd2b 100644 (file)
--- a/modules/ffmpeg/producer/input/input.h
+++ b/modules/ffmpeg/producer/input/input.h
@@ -52,8 +52,11 @@ public:
                        uint32_t length,
                        bool thumbnail_mode);
 
+       int                     num_audio_streams() const;
+       int                     get_actual_audio_stream_index(int audio_stream_index) const;
+
        bool            try_pop_video(std::shared_ptr<AVPacket>& packet);
-       bool            try_pop_audio(std::shared_ptr<AVPacket>& packet);
+       bool            try_pop_audio(std::shared_ptr<AVPacket>& packet, int audio_stream_index);
 
        void            loop(bool value);
        bool            loop() const;

diff --git a/modules/ffmpeg/producer/util/util.cpp b/modules/ffmpeg/producer/util/util.cpp
index ce1994b8854f9536729dc7feaf7a3913b3b5594f..652a9a7ccc9b4107154d9e436d4ac0d4e5644080 100644 (file)
--- a/modules/ffmpeg/producer/util/util.cpp
+++ b/modules/ffmpeg/producer/util/util.cpp
@@ -494,7 +494,7 @@ spl::shared_ptr<AVFrame> create_frame()
 spl::shared_ptr<AVCodecContext> open_codec(AVFormatContext& context, enum AVMediaType type, int& index, bool single_threaded)
 {      
        AVCodec* decoder;
-       index = THROW_ON_ERROR2(av_find_best_stream(&context, type, -1, -1, &decoder, 0), "");
+       index = THROW_ON_ERROR2(av_find_best_stream(&context, type, index, -1, &decoder, 0), "");
        //if(strcmp(decoder->name, "prores") == 0 && decoder->next && strcmp(decoder->next->name, "prores_lgpl") == 0)
        //      decoder = decoder->next;
 

diff --git a/modules/ffmpeg/producer/video/video_decoder.cpp b/modules/ffmpeg/producer/video/video_decoder.cpp
index 429f78ff31c3c91ca94cb796fb856be0bc59f0e4..87ee744dc8a56133de3d5c9808f72ae30875c147 100644 (file)
--- a/modules/ffmpeg/producer/video/video_decoder.cpp
+++ b/modules/ffmpeg/producer/video/video_decoder.cpp
@@ -55,7 +55,7 @@ struct video_decoder::impl : boost::noncopyable
 {
        core::monitor::subject                                  monitor_subject_;
        input*                                                                  input_;
-       int                                                                             index_;
+       int                                                                             index_                          = -1;
        const spl::shared_ptr<AVCodecContext>   codec_context_;
 
        std::queue<spl::shared_ptr<AVPacket>>   packets_;