2.0. audio:

        - Fixed bug where channel samples were crossfaded incorrectly.
        - SSE optimized audio mixing.
     decklink_producer: Added 32 bit audio support.

git-svn-id: https://casparcg.svn.sourceforge.net/svnroot/casparcg/server/branches/2.0.0.2@1328 362d55ac-95cf-4e76-9f9a-cbaa9c17b72d

diff --git a/core/mixer/audio/audio_mixer.cpp b/core/mixer/audio/audio_mixer.cpp
index e356844d1b4962afcc7a6fd2cb57d5ec446ce13b..dafa6e4dd0991c81a3310d407d4c61daceca954a 100644 (file)
--- a/core/mixer/audio/audio_mixer.cpp
+++ b/core/mixer/audio/audio_mixer.cpp
@@ -90,9 +90,9 @@ public:
                transform_stack_.pop();\r
        }\r
        \r
-       std::vector<int32_t> mix()\r
+       audio_buffer mix()\r
        {\r
-               auto result = std::vector<int32_t>(format_desc_.audio_samples_per_frame, 0);\r
+               auto result = audio_buffer(format_desc_.audio_samples_per_frame, 0);\r
 \r
                std::map<const void*, core::frame_transform> next_frame_transforms;\r
 \r
@@ -109,30 +109,38 @@ public:
                                \r
                        if(next.volume < 0.001 && prev.volume < 0.001)\r
                                continue;\r
-               \r
-                       static const int BASE = 1<<31;\r
-\r
-                       const auto next_volume = static_cast<int64_t>(next.volume*BASE);\r
-                       const auto prev_volume = static_cast<int64_t>(prev.volume*BASE);\r
-               \r
-                       const int n_samples = result.size();\r
-               \r
-                       const auto in_size = static_cast<size_t>(item.audio_data.size());\r
-                       CASPAR_VERIFY(in_size == 0 || in_size == result.size());\r
-\r
-                       if(in_size > result.size())\r
+                                                                       \r
+                       if(static_cast<size_t>(item.audio_data.size()) != format_desc_.audio_samples_per_frame)\r
                                continue;\r
 \r
+                       CASPAR_ASSERT(format_desc_.audio_channels == 2);\r
+                                               \r
+                       const float prev_volume = static_cast<float>(prev.volume);\r
+                       const float next_volume = static_cast<float>(next.volume);\r
+                       const float delta               = 1.0f/static_cast<float>(format_desc_.audio_samples_per_frame/2);\r
+                       \r
                        tbb::parallel_for\r
                        (\r
-                               tbb::blocked_range<size_t>(0, item.audio_data.size()),\r
+                               tbb::blocked_range<size_t>(0, format_desc_.audio_samples_per_frame/2),\r
                                [&](const tbb::blocked_range<size_t>& r)\r
                                {\r
                                        for(size_t n = r.begin(); n < r.end(); ++n)\r
                                        {\r
-                                               const auto sample_volume = (prev_volume - (prev_volume * n)/n_samples) + (next_volume * n)/n_samples;\r
-                                               const auto sample = static_cast<int32_t>((static_cast<int64_t>(item.audio_data[n])*sample_volume)/BASE);\r
-                                               result[n] = result[n] + sample;\r
+                                               const float alpha               = n * delta;\r
+                                               const float volume              = prev_volume * (1.0f - alpha) + next_volume * alpha;\r
+\r
+                                               auto sample_epi32               = _mm_loadl_epi64(reinterpret_cast<__m128i*>(&item.audio_data[n*2]));\r
+\r
+                                               auto sample_ps                  = _mm_cvtepi32_ps(sample_epi32);                                                                                                \r
+                                               sample_ps                               = _mm_mul_ps(sample_ps, _mm_set1_ps(volume));   \r
+\r
+                                               auto res_sample_epi32   = _mm_loadl_epi64(reinterpret_cast<__m128i*>(&result[n*2]));\r
+                                               auto res_sample_ps              = _mm_cvtepi32_ps(res_sample_epi32);    \r
+\r
+                                               res_sample_ps                   = _mm_add_ps(sample_ps, res_sample_ps);\r
+                                               res_sample_epi32                = _mm_cvtps_epi32(res_sample_ps);\r
+                                               \r
+                                               _mm_storel_epi64(reinterpret_cast<__m128i*>(&result[n*2]), res_sample_epi32);\r
                                        }\r
                                }\r
                        );\r
@@ -141,6 +149,7 @@ public:
                items.clear();\r
                prev_frame_transforms_ = std::move(next_frame_transforms);      \r
 \r
+               result.resize(format_desc_.audio_samples_per_frame);\r
                return std::move(result);\r
        }\r
 };\r
@@ -149,7 +158,7 @@ audio_mixer::audio_mixer(const core::video_format_desc& format_desc) : impl_(new
 void audio_mixer::begin(core::basic_frame& frame){impl_->begin(frame);}\r
 void audio_mixer::visit(core::write_frame& frame){impl_->visit(frame);}\r
 void audio_mixer::end(){impl_->end();}\r
-std::vector<int32_t> audio_mixer::mix(){return impl_->mix();}\r
+audio_buffer audio_mixer::mix(){return impl_->mix();}\r
 audio_mixer& audio_mixer::operator=(audio_mixer&& other)\r
 {\r
        impl_ = std::move(other.impl_);\r

diff --git a/core/mixer/audio/audio_mixer.h b/core/mixer/audio/audio_mixer.h
index 3a5648230c93d15a4de7c03c34a8043fa3bb8914..6523ef253992f9c848474b6d86c32544dd9536ed 100644 (file)
--- a/core/mixer/audio/audio_mixer.h
+++ b/core/mixer/audio/audio_mixer.h
@@ -25,12 +25,16 @@
 \r
 #include <boost/noncopyable.hpp>\r
 \r
+#include <tbb/cache_aligned_allocator.h>\r
+\r
 #include <vector>\r
 \r
 namespace caspar { namespace core {\r
 \r
 struct video_format_desc;\r
        \r
+typedef std::vector<int32_t, tbb::cache_aligned_allocator<int32_t>> audio_buffer;\r
+\r
 class audio_mixer : public core::frame_visitor, boost::noncopyable\r
 {\r
 public:\r
@@ -40,7 +44,7 @@ public:
        virtual void visit(core::write_frame& frame);\r
        virtual void end();\r
 \r
-       std::vector<int32_t> mix();\r
+       audio_buffer mix();\r
        \r
        audio_mixer& operator=(audio_mixer&& other);\r
 private:\r

diff --git a/core/mixer/mixer.cpp b/core/mixer/mixer.cpp
index 95721d04400ab65464d8b8c6d86bd3e61ca9e552..8353d4170769bd351b345e049b7a00899be812c4 100644 (file)
--- a/core/mixer/mixer.cpp
+++ b/core/mixer/mixer.cpp
@@ -96,7 +96,7 @@ struct mixer::implementation : boost::noncopyable
        std::unordered_map<int, tweened_transform<core::frame_transform>> transforms_;  \r
        std::unordered_map<int, blend_mode::type> blend_modes_;\r
 \r
-       std::queue<std::pair<boost::unique_future<safe_ptr<host_buffer>>, std::vector<int32_t>>> buffer_;\r
+       std::queue<std::pair<boost::unique_future<safe_ptr<host_buffer>>, core::audio_buffer>> buffer_;\r
        \r
        const size_t buffer_size_;\r
 \r

diff --git a/core/mixer/read_frame.cpp b/core/mixer/read_frame.cpp
index 77ab0c3b666f85d3088d1950be01deae97491eee..13a97b10bb461b79f8547eaaf49743cc12d0dd06 100644 (file)
--- a/core/mixer/read_frame.cpp
+++ b/core/mixer/read_frame.cpp
@@ -35,10 +35,10 @@ struct read_frame::implementation : boost::noncopyable
        size_t                                          size_;\r
        safe_ptr<host_buffer>           image_data_;\r
        tbb::mutex                                      mutex_;\r
-       std::vector<int32_t>            audio_data_;\r
+       audio_buffer                            audio_data_;\r
 \r
 public:\r
-       implementation(ogl_device& ogl, size_t size, safe_ptr<host_buffer>&& image_data, std::vector<int32_t>&& audio_data) \r
+       implementation(ogl_device& ogl, size_t size, safe_ptr<host_buffer>&& image_data, audio_buffer&& audio_data) \r
                : ogl_(ogl)\r
                , size_(size)\r
                , image_data_(std::move(image_data))\r
@@ -65,7 +65,7 @@ public:
        }\r
 };\r
 \r
-read_frame::read_frame(ogl_device& ogl, size_t size, safe_ptr<host_buffer>&& image_data, std::vector<int32_t>&& audio_data) \r
+read_frame::read_frame(ogl_device& ogl, size_t size, safe_ptr<host_buffer>&& image_data, audio_buffer&& audio_data) \r
        : impl_(new implementation(ogl, size, std::move(image_data), std::move(audio_data))){}\r
 read_frame::read_frame(){}\r
 const boost::iterator_range<const uint8_t*> read_frame::image_data()\r

diff --git a/core/mixer/read_frame.h b/core/mixer/read_frame.h
index a3afc36cd14b22d255008618067635d818aa2b17..976ca39f90d42517d52a2af045cc87470fe21267 100644 (file)
--- a/core/mixer/read_frame.h
+++ b/core/mixer/read_frame.h
@@ -21,6 +21,8 @@
 \r
 #include <common/memory/safe_ptr.h>\r
 \r
+#include <core/mixer/audio/audio_mixer.h>\r
+\r
 #include <boost/noncopyable.hpp>\r
 #include <boost/range/iterator_range.hpp>\r
 \r
@@ -37,7 +39,7 @@ class read_frame : boost::noncopyable
 {\r
 public:\r
        read_frame();\r
-       read_frame(ogl_device& ogl, size_t size, safe_ptr<host_buffer>&& image_data, std::vector<int32_t>&& audio_data);\r
+       read_frame(ogl_device& ogl, size_t size, safe_ptr<host_buffer>&& image_data, audio_buffer&& audio_data);\r
 \r
        virtual const boost::iterator_range<const uint8_t*> image_data();\r
        virtual const boost::iterator_range<const int32_t*> audio_data();\r

diff --git a/core/mixer/write_frame.cpp b/core/mixer/write_frame.cpp
index 1fc4553497349a876efcb6242d1dd830b8567c2e..f7b25f50a08c56660f7fa8c17886fe9c942ce677 100644 (file)
--- a/core/mixer/write_frame.cpp
+++ b/core/mixer/write_frame.cpp
@@ -37,7 +37,7 @@ struct write_frame::implementation
        ogl_device*                                                                     ogl_;\r
        std::vector<std::shared_ptr<host_buffer>>       buffers_;\r
        std::vector<safe_ptr<device_buffer>>            textures_;\r
-       std::vector<int32_t>                                            audio_data_;\r
+       audio_buffer                                                            audio_data_;\r
        const core::pixel_format_desc                           desc_;\r
        const void*                                                                     tag_;\r
        core::field_mode::type                                          mode_;\r
@@ -142,7 +142,7 @@ write_frame& write_frame::operator=(write_frame&& other)
 void write_frame::swap(write_frame& other){impl_.swap(other.impl_);}\r
 \r
 boost::iterator_range<uint8_t*> write_frame::image_data(size_t index){return impl_->image_data(index);}\r
-std::vector<int32_t>& write_frame::audio_data() { return impl_->audio_data_; }\r
+audio_buffer& write_frame::audio_data() { return impl_->audio_data_; }\r
 const boost::iterator_range<const uint8_t*> write_frame::image_data(size_t index) const\r
 {\r
        return boost::iterator_range<const uint8_t*>(impl_->image_data(index).begin(), impl_->image_data(index).end());\r

diff --git a/core/mixer/write_frame.h b/core/mixer/write_frame.h
index 9f295f819fc4f7c3f84d5b4f4bba165455bf2e8d..cf95d1728c5a05c2c9c269dabb53ea718a7c6381 100644 (file)
--- a/core/mixer/write_frame.h
+++ b/core/mixer/write_frame.h
@@ -23,6 +23,7 @@
 \r
 #include <core/producer/frame/basic_frame.h>\r
 #include <core/video_format.h>\r
+#include <core/mixer/audio/audio_mixer.h>\r
 \r
 #include <boost/noncopyable.hpp>\r
 #include <boost/range/iterator_range.hpp>\r
@@ -54,7 +55,7 @@ public:
        boost::iterator_range<uint8_t*> image_data(size_t plane_index = 0);     \r
        const boost::iterator_range<const uint8_t*> image_data(size_t plane_index = 0) const;\r
 \r
-       std::vector<int32_t>& audio_data();\r
+       audio_buffer& audio_data();\r
        const boost::iterator_range<const int32_t*> audio_data() const;\r
        \r
        void commit(uint32_t plane_index);\r

diff --git a/modules/decklink/producer/decklink_producer.cpp b/modules/decklink/producer/decklink_producer.cpp
index 3ab2706499a86fd5b4aac598139009f8271334e0..f5734796641bd2b7354fadf797abd3e590590c6b 100644 (file)
--- a/modules/decklink/producer/decklink_producer.cpp
+++ b/modules/decklink/producer/decklink_producer.cpp
@@ -124,7 +124,7 @@ public:
                                                                        << msg_info(narrow(print()) + " Could not enable video input.")\r
                                                                        << boost::errinfo_api_function("EnableVideoInput"));\r
 \r
-               if(FAILED(input_->EnableAudioInput(bmdAudioSampleRate48kHz, bmdAudioSampleType16bitInteger, 2))) \r
+               if(FAILED(input_->EnableAudioInput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2))) \r
                        BOOST_THROW_EXCEPTION(caspar_exception() \r
                                                                        << msg_info(narrow(print()) + " Could not enable audio input.")\r
                                                                        << boost::errinfo_api_function("EnableAudioInput"));\r
@@ -194,11 +194,11 @@ public:
                        if(audio && SUCCEEDED(audio->GetBytes(&bytes)))\r
                        {\r
                                auto sample_frame_count = audio->GetSampleFrameCount();\r
-                               auto audio_data = reinterpret_cast<short*>(bytes);\r
-                               muxer_.push(std::make_shared<std::vector<int32_t>>(audio_data, audio_data + sample_frame_count*2));\r
+                               auto audio_data = reinterpret_cast<int32_t*>(bytes);\r
+                               muxer_.push(std::make_shared<core::audio_buffer>(audio_data, audio_data + sample_frame_count*sizeof(int32_t)));\r
                        }\r
                        else\r
-                               muxer_.push(std::make_shared<std::vector<int32_t>>(frame_factory_->get_video_format_desc().audio_samples_per_frame, 0));\r
+                               muxer_.push(std::make_shared<core::audio_buffer>(frame_factory_->get_video_format_desc().audio_samples_per_frame, 0));\r
                                        \r
                        muxer_.commit();\r
 \r

diff --git a/modules/ffmpeg/producer/audio/audio_decoder.cpp b/modules/ffmpeg/producer/audio/audio_decoder.cpp
index dd0a9e58675e55e488a5775fa716183d137fbed0..a71c672c25271f23c474f59fb00aecefbc2e535e 100644 (file)
--- a/modules/ffmpeg/producer/audio/audio_decoder.cpp
+++ b/modules/ffmpeg/producer/audio/audio_decoder.cpp
@@ -95,9 +95,9 @@ public:
                packets_.push(packet);\r
        }       \r
        \r
-       std::vector<std::shared_ptr<std::vector<int32_t>>> poll()\r
+       std::vector<std::shared_ptr<core::audio_buffer>> poll()\r
        {\r
-               std::vector<std::shared_ptr<std::vector<int32_t>>> result;\r
+               std::vector<std::shared_ptr<core::audio_buffer>> result;\r
 \r
                if(packets_.empty())\r
                        return result;\r
@@ -123,7 +123,7 @@ public:
                return result;\r
        }\r
 \r
-       std::vector<std::shared_ptr<std::vector<int32_t>>> empty_poll()\r
+       std::vector<std::shared_ptr<core::audio_buffer>> empty_poll()\r
        {\r
                auto packet = packets_.front();\r
                packets_.pop();\r
@@ -131,10 +131,10 @@ public:
                if(!packet)                     \r
                        return boost::assign::list_of(nullptr);\r
                \r
-               return boost::assign::list_of(std::make_shared<std::vector<int32_t>>(format_desc_.audio_samples_per_frame, 0)); \r
+               return boost::assign::list_of(std::make_shared<core::audio_buffer>(format_desc_.audio_samples_per_frame, 0));   \r
        }\r
 \r
-       std::shared_ptr<std::vector<int32_t>> decode(AVPacket& pkt)\r
+       std::shared_ptr<core::audio_buffer> decode(AVPacket& pkt)\r
        {               \r
                buffer1_.resize(AVCODEC_MAX_AUDIO_FRAME_SIZE*2);\r
                int written_bytes = buffer1_.size() - FF_INPUT_BUFFER_PADDING_SIZE;\r
@@ -152,7 +152,7 @@ public:
                const auto n_samples = buffer1_.size() / av_get_bytes_per_sample(AV_SAMPLE_FMT_S32);\r
                const auto samples = reinterpret_cast<int32_t*>(buffer1_.data());\r
 \r
-               return std::make_shared<std::vector<int32_t>>(samples, samples + n_samples);\r
+               return std::make_shared<core::audio_buffer>(samples, samples + n_samples);\r
        }\r
 \r
        bool ready() const\r
@@ -164,6 +164,6 @@ public:
 audio_decoder::audio_decoder(const safe_ptr<AVFormatContext>& context, const core::video_format_desc& format_desc) : impl_(new implementation(context, format_desc)){}\r
 void audio_decoder::push(const std::shared_ptr<AVPacket>& packet){impl_->push(packet);}\r
 bool audio_decoder::ready() const{return impl_->ready();}\r
-std::vector<std::shared_ptr<std::vector<int32_t>>> audio_decoder::poll(){return impl_->poll();}\r
+std::vector<std::shared_ptr<core::audio_buffer>> audio_decoder::poll(){return impl_->poll();}\r
 int64_t audio_decoder::nb_frames() const{return impl_->nb_frames_;}\r
 }
\ No newline at end of file

diff --git a/modules/ffmpeg/producer/audio/audio_decoder.h b/modules/ffmpeg/producer/audio/audio_decoder.h
index 8ad020697692e4d2b5090b18f03701306c9e34b7..e2c978cf8db3854b3b523d57f1988f9504f7759d 100644 (file)
--- a/modules/ffmpeg/producer/audio/audio_decoder.h
+++ b/modules/ffmpeg/producer/audio/audio_decoder.h
@@ -19,6 +19,8 @@
 */\r
 #pragma once\r
 \r
+#include <core/mixer/audio/audio_mixer.h>\r
+\r
 #include <common/memory/safe_ptr.h>\r
 \r
 #include <boost/noncopyable.hpp>\r
@@ -43,7 +45,7 @@ public:
        \r
        void push(const std::shared_ptr<AVPacket>& packet);\r
        bool ready() const;\r
-       std::vector<std::shared_ptr<std::vector<int32_t>>> poll();\r
+       std::vector<std::shared_ptr<core::audio_buffer>> poll();\r
 \r
        int64_t nb_frames() const;\r
 \r

diff --git a/modules/ffmpeg/producer/frame_muxer.cpp b/modules/ffmpeg/producer/frame_muxer.cpp
index affb1c329c6469e4270228f6e73075b0a004ded5..4dfbcebf234e71ad09442994fe0902e9e52d1dca 100644 (file)
--- a/modules/ffmpeg/producer/frame_muxer.cpp
+++ b/modules/ffmpeg/producer/frame_muxer.cpp
@@ -136,7 +136,7 @@ display_mode::type get_display_mode(const core::field_mode::type in_mode, double
 struct frame_muxer::implementation : boost::noncopyable\r
 {      \r
        std::deque<std::queue<safe_ptr<write_frame>>>   video_streams_;\r
-       std::deque<std::vector<int32_t>>                                audio_streams_;\r
+       std::deque<core::audio_buffer>                                  audio_streams_;\r
        std::deque<safe_ptr<basic_frame>>                               frame_buffer_;\r
        display_mode::type                                                              display_mode_;\r
        const double                                                                    in_fps_;\r
@@ -238,12 +238,12 @@ struct frame_muxer::implementation : boost::noncopyable
                        BOOST_THROW_EXCEPTION(invalid_operation() << source_info("frame_muxer") << msg_info("video-stream overflow. This can be caused by incorrect frame-rate. Check clip meta-data."));\r
        }\r
 \r
-       void push(const std::shared_ptr<std::vector<int32_t>>& audio_samples)\r
+       void push(const std::shared_ptr<core::audio_buffer>& audio_samples)\r
        {\r
                if(!audio_samples)      \r
                {\r
                        CASPAR_LOG(debug) << L"audio-chunk-count: " << audio_sample_count_/format_desc_.audio_samples_per_frame;\r
-                       audio_streams_.push_back(std::vector<int32_t>());\r
+                       audio_streams_.push_back(core::audio_buffer());\r
                        audio_sample_count_ = 0;\r
                        return;\r
                }\r
@@ -276,14 +276,14 @@ struct frame_muxer::implementation : boost::noncopyable
                return frame;\r
        }\r
 \r
-       std::vector<int32_t> pop_audio()\r
+       core::audio_buffer pop_audio()\r
        {\r
                CASPAR_VERIFY(audio_streams_.front().size() >= format_desc_.audio_samples_per_frame);\r
 \r
                auto begin = audio_streams_.front().begin();\r
                auto end   = begin + format_desc_.audio_samples_per_frame;\r
 \r
-               auto samples = std::vector<int32_t>(begin, end);\r
+               auto samples = core::audio_buffer(begin, end);\r
                audio_streams_.front().erase(begin, end);\r
 \r
                return samples;\r
@@ -410,7 +410,7 @@ struct frame_muxer::implementation : boost::noncopyable
 frame_muxer::frame_muxer(double in_fps, const safe_ptr<core::frame_factory>& frame_factory)\r
        : impl_(new implementation(in_fps, frame_factory)){}\r
 void frame_muxer::push(const std::shared_ptr<AVFrame>& video_frame, int hints){impl_->push(video_frame, hints);}\r
-void frame_muxer::push(const std::shared_ptr<std::vector<int32_t>>& audio_samples){return impl_->push(audio_samples);}\r
+void frame_muxer::push(const std::shared_ptr<core::audio_buffer>& audio_samples){return impl_->push(audio_samples);}\r
 void frame_muxer::commit(){impl_->commit();}\r
 safe_ptr<basic_frame> frame_muxer::pop(){return impl_->pop();}\r
 size_t frame_muxer::size() const {return impl_->size();}\r

diff --git a/modules/ffmpeg/producer/frame_muxer.h b/modules/ffmpeg/producer/frame_muxer.h
index da9f4108649182a478f36da726980e1a5540ceaf..9390f0b8586683e7a2c916be357278d6209e8116 100644 (file)
--- a/modules/ffmpeg/producer/frame_muxer.h
+++ b/modules/ffmpeg/producer/frame_muxer.h
@@ -2,6 +2,8 @@
 \r
 #include <common/memory/safe_ptr.h>\r
 \r
+#include <core/mixer/audio/audio_mixer.h>\r
+\r
 #include <boost/noncopyable.hpp>\r
 \r
 #include <vector>\r
@@ -24,7 +26,7 @@ public:
        frame_muxer(double in_fps, const safe_ptr<core::frame_factory>& frame_factory);\r
        \r
        void push(const std::shared_ptr<AVFrame>& video_frame, int hints = 0);\r
-       void push(const std::shared_ptr<std::vector<int32_t>>& audio_samples);\r
+       void push(const std::shared_ptr<core::audio_buffer>& audio_samples);\r
        \r
        void commit();\r
 \r