From: ronag <ronag@362d55ac-95cf-4e76-9f9a-cbaa9c17b72d>
Date: Fri, 2 Sep 2011 21:55:59 +0000 (+0000)
Subject: 2.0. audio:
X-Git-Tag: 2.0.1~29
X-Git-Url: https://git.sesse.net/?p=casparcg;a=commitdiff_plain;h=ffcb75eabba1d77eab8d83447136261462e3e088

2.0. audio:
        - Fixed bug where channel samples were crossfaded incorrectly.
        - SSE optimized audio mixing.
     decklink_producer: Added 32 bit audio support.


git-svn-id: https://casparcg.svn.sourceforge.net/svnroot/casparcg/server/branches/2.0.0.2@1328 362d55ac-95cf-4e76-9f9a-cbaa9c17b72d
---

diff --git a/core/mixer/audio/audio_mixer.cpp b/core/mixer/audio/audio_mixer.cpp
index e356844d1..dafa6e4dd 100644
--- a/core/mixer/audio/audio_mixer.cpp
+++ b/core/mixer/audio/audio_mixer.cpp
@@ -90,9 +90,9 @@ public:
 		transform_stack_.pop();
 	}
 	
-	std::vector<int32_t> mix()
+	audio_buffer mix()
 	{
-		auto result = std::vector<int32_t>(format_desc_.audio_samples_per_frame, 0);
+		auto result = audio_buffer(format_desc_.audio_samples_per_frame, 0);
 
 		std::map<const void*, core::frame_transform> next_frame_transforms;
 
@@ -109,30 +109,38 @@ public:
 				
 			if(next.volume < 0.001 && prev.volume < 0.001)
 				continue;
-		
-			static const int BASE = 1<<31;
-
-			const auto next_volume = static_cast<int64_t>(next.volume*BASE);
-			const auto prev_volume = static_cast<int64_t>(prev.volume*BASE);
-		
-			const int n_samples = result.size();
-		
-			const auto in_size = static_cast<size_t>(item.audio_data.size());
-			CASPAR_VERIFY(in_size == 0 || in_size == result.size());
-
-			if(in_size > result.size())
+									
+			if(static_cast<size_t>(item.audio_data.size()) != format_desc_.audio_samples_per_frame)
 				continue;
 
+			CASPAR_ASSERT(format_desc_.audio_channels == 2);
+						
+			const float prev_volume = static_cast<float>(prev.volume);
+			const float next_volume = static_cast<float>(next.volume);
+			const float delta		= 1.0f/static_cast<float>(format_desc_.audio_samples_per_frame/2);
+			
 			tbb::parallel_for
 			(
-				tbb::blocked_range<size_t>(0, item.audio_data.size()),
+				tbb::blocked_range<size_t>(0, format_desc_.audio_samples_per_frame/2),
 				[&](const tbb::blocked_range<size_t>& r)
 				{
 					for(size_t n = r.begin(); n < r.end(); ++n)
 					{
-						const auto sample_volume = (prev_volume - (prev_volume * n)/n_samples) + (next_volume * n)/n_samples;
-						const auto sample = static_cast<int32_t>((static_cast<int64_t>(item.audio_data[n])*sample_volume)/BASE);
-						result[n] = result[n] + sample;
+						const float alpha		= n * delta;
+						const float volume		= prev_volume * (1.0f - alpha) + next_volume * alpha;
+
+						auto sample_epi32		= _mm_loadl_epi64(reinterpret_cast<__m128i*>(&item.audio_data[n*2]));
+
+						auto sample_ps			= _mm_cvtepi32_ps(sample_epi32);												
+						sample_ps				= _mm_mul_ps(sample_ps, _mm_set1_ps(volume));	
+
+						auto res_sample_epi32	= _mm_loadl_epi64(reinterpret_cast<__m128i*>(&result[n*2]));
+						auto res_sample_ps		= _mm_cvtepi32_ps(res_sample_epi32);	
+
+						res_sample_ps			= _mm_add_ps(sample_ps, res_sample_ps);
+						res_sample_epi32		= _mm_cvtps_epi32(res_sample_ps);
+						
+						_mm_storel_epi64(reinterpret_cast<__m128i*>(&result[n*2]), res_sample_epi32);
 					}
 				}
 			);
@@ -141,6 +149,7 @@ public:
 		items.clear();
 		prev_frame_transforms_ = std::move(next_frame_transforms);	
 
+		result.resize(format_desc_.audio_samples_per_frame);
 		return std::move(result);
 	}
 };
@@ -149,7 +158,7 @@ audio_mixer::audio_mixer(const core::video_format_desc& format_desc) : impl_(new
 void audio_mixer::begin(core::basic_frame& frame){impl_->begin(frame);}
 void audio_mixer::visit(core::write_frame& frame){impl_->visit(frame);}
 void audio_mixer::end(){impl_->end();}
-std::vector<int32_t> audio_mixer::mix(){return impl_->mix();}
+audio_buffer audio_mixer::mix(){return impl_->mix();}
 audio_mixer& audio_mixer::operator=(audio_mixer&& other)
 {
 	impl_ = std::move(other.impl_);
diff --git a/core/mixer/audio/audio_mixer.h b/core/mixer/audio/audio_mixer.h
index 3a5648230..6523ef253 100644
--- a/core/mixer/audio/audio_mixer.h
+++ b/core/mixer/audio/audio_mixer.h
@@ -25,12 +25,16 @@
 
 #include <boost/noncopyable.hpp>
 
+#include <tbb/cache_aligned_allocator.h>
+
 #include <vector>
 
 namespace caspar { namespace core {
 
 struct video_format_desc;
 	
+typedef std::vector<int32_t, tbb::cache_aligned_allocator<int32_t>> audio_buffer;
+
 class audio_mixer : public core::frame_visitor, boost::noncopyable
 {
 public:
@@ -40,7 +44,7 @@ public:
 	virtual void visit(core::write_frame& frame);
 	virtual void end();
 
-	std::vector<int32_t> mix();
+	audio_buffer mix();
 	
 	audio_mixer& operator=(audio_mixer&& other);
 private:
diff --git a/core/mixer/mixer.cpp b/core/mixer/mixer.cpp
index 95721d044..8353d4170 100644
--- a/core/mixer/mixer.cpp
+++ b/core/mixer/mixer.cpp
@@ -96,7 +96,7 @@ struct mixer::implementation : boost::noncopyable
 	std::unordered_map<int, tweened_transform<core::frame_transform>> transforms_;	
 	std::unordered_map<int, blend_mode::type> blend_modes_;
 
-	std::queue<std::pair<boost::unique_future<safe_ptr<host_buffer>>, std::vector<int32_t>>> buffer_;
+	std::queue<std::pair<boost::unique_future<safe_ptr<host_buffer>>, core::audio_buffer>> buffer_;
 	
 	const size_t buffer_size_;
 
diff --git a/core/mixer/read_frame.cpp b/core/mixer/read_frame.cpp
index 77ab0c3b6..13a97b10b 100644
--- a/core/mixer/read_frame.cpp
+++ b/core/mixer/read_frame.cpp
@@ -35,10 +35,10 @@ struct read_frame::implementation : boost::noncopyable
 	size_t						size_;
 	safe_ptr<host_buffer>		image_data_;
 	tbb::mutex					mutex_;
-	std::vector<int32_t>		audio_data_;
+	audio_buffer				audio_data_;
 
 public:
-	implementation(ogl_device& ogl, size_t size, safe_ptr<host_buffer>&& image_data, std::vector<int32_t>&& audio_data) 
+	implementation(ogl_device& ogl, size_t size, safe_ptr<host_buffer>&& image_data, audio_buffer&& audio_data) 
 		: ogl_(ogl)
 		, size_(size)
 		, image_data_(std::move(image_data))
@@ -65,7 +65,7 @@ public:
 	}
 };
 
-read_frame::read_frame(ogl_device& ogl, size_t size, safe_ptr<host_buffer>&& image_data, std::vector<int32_t>&& audio_data) 
+read_frame::read_frame(ogl_device& ogl, size_t size, safe_ptr<host_buffer>&& image_data, audio_buffer&& audio_data) 
 	: impl_(new implementation(ogl, size, std::move(image_data), std::move(audio_data))){}
 read_frame::read_frame(){}
 const boost::iterator_range<const uint8_t*> read_frame::image_data()
diff --git a/core/mixer/read_frame.h b/core/mixer/read_frame.h
index a3afc36cd..976ca39f9 100644
--- a/core/mixer/read_frame.h
+++ b/core/mixer/read_frame.h
@@ -21,6 +21,8 @@
 
 #include <common/memory/safe_ptr.h>
 
+#include <core/mixer/audio/audio_mixer.h>
+
 #include <boost/noncopyable.hpp>
 #include <boost/range/iterator_range.hpp>
 
@@ -37,7 +39,7 @@ class read_frame : boost::noncopyable
 {
 public:
 	read_frame();
-	read_frame(ogl_device& ogl, size_t size, safe_ptr<host_buffer>&& image_data, std::vector<int32_t>&& audio_data);
+	read_frame(ogl_device& ogl, size_t size, safe_ptr<host_buffer>&& image_data, audio_buffer&& audio_data);
 
 	virtual const boost::iterator_range<const uint8_t*> image_data();
 	virtual const boost::iterator_range<const int32_t*> audio_data();
diff --git a/core/mixer/write_frame.cpp b/core/mixer/write_frame.cpp
index 1fc455349..f7b25f50a 100644
--- a/core/mixer/write_frame.cpp
+++ b/core/mixer/write_frame.cpp
@@ -37,7 +37,7 @@ struct write_frame::implementation
 	ogl_device*									ogl_;
 	std::vector<std::shared_ptr<host_buffer>>	buffers_;
 	std::vector<safe_ptr<device_buffer>>		textures_;
-	std::vector<int32_t>						audio_data_;
+	audio_buffer								audio_data_;
 	const core::pixel_format_desc				desc_;
 	const void*									tag_;
 	core::field_mode::type						mode_;
@@ -142,7 +142,7 @@ write_frame& write_frame::operator=(write_frame&& other)
 void write_frame::swap(write_frame& other){impl_.swap(other.impl_);}
 
 boost::iterator_range<uint8_t*> write_frame::image_data(size_t index){return impl_->image_data(index);}
-std::vector<int32_t>& write_frame::audio_data() { return impl_->audio_data_; }
+audio_buffer& write_frame::audio_data() { return impl_->audio_data_; }
 const boost::iterator_range<const uint8_t*> write_frame::image_data(size_t index) const
 {
 	return boost::iterator_range<const uint8_t*>(impl_->image_data(index).begin(), impl_->image_data(index).end());
diff --git a/core/mixer/write_frame.h b/core/mixer/write_frame.h
index 9f295f819..cf95d1728 100644
--- a/core/mixer/write_frame.h
+++ b/core/mixer/write_frame.h
@@ -23,6 +23,7 @@
 
 #include <core/producer/frame/basic_frame.h>
 #include <core/video_format.h>
+#include <core/mixer/audio/audio_mixer.h>
 
 #include <boost/noncopyable.hpp>
 #include <boost/range/iterator_range.hpp>
@@ -54,7 +55,7 @@ public:
 	boost::iterator_range<uint8_t*> image_data(size_t plane_index = 0);	
 	const boost::iterator_range<const uint8_t*> image_data(size_t plane_index = 0) const;
 
-	std::vector<int32_t>& audio_data();
+	audio_buffer& audio_data();
 	const boost::iterator_range<const int32_t*> audio_data() const;
 	
 	void commit(uint32_t plane_index);
diff --git a/modules/decklink/producer/decklink_producer.cpp b/modules/decklink/producer/decklink_producer.cpp
index 3ab270649..f57347966 100644
--- a/modules/decklink/producer/decklink_producer.cpp
+++ b/modules/decklink/producer/decklink_producer.cpp
@@ -124,7 +124,7 @@ public:
 									<< msg_info(narrow(print()) + " Could not enable video input.")
 									<< boost::errinfo_api_function("EnableVideoInput"));
 
-		if(FAILED(input_->EnableAudioInput(bmdAudioSampleRate48kHz, bmdAudioSampleType16bitInteger, 2))) 
+		if(FAILED(input_->EnableAudioInput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2))) 
 			BOOST_THROW_EXCEPTION(caspar_exception() 
 									<< msg_info(narrow(print()) + " Could not enable audio input.")
 									<< boost::errinfo_api_function("EnableAudioInput"));
@@ -194,11 +194,11 @@ public:
 			if(audio && SUCCEEDED(audio->GetBytes(&bytes)))
 			{
 				auto sample_frame_count = audio->GetSampleFrameCount();
-				auto audio_data = reinterpret_cast<short*>(bytes);
-				muxer_.push(std::make_shared<std::vector<int32_t>>(audio_data, audio_data + sample_frame_count*2));
+				auto audio_data = reinterpret_cast<int32_t*>(bytes);
+				muxer_.push(std::make_shared<core::audio_buffer>(audio_data, audio_data + sample_frame_count*sizeof(int32_t)));
 			}
 			else
-				muxer_.push(std::make_shared<std::vector<int32_t>>(frame_factory_->get_video_format_desc().audio_samples_per_frame, 0));
+				muxer_.push(std::make_shared<core::audio_buffer>(frame_factory_->get_video_format_desc().audio_samples_per_frame, 0));
 					
 			muxer_.commit();
 
diff --git a/modules/ffmpeg/producer/audio/audio_decoder.cpp b/modules/ffmpeg/producer/audio/audio_decoder.cpp
index dd0a9e586..a71c672c2 100644
--- a/modules/ffmpeg/producer/audio/audio_decoder.cpp
+++ b/modules/ffmpeg/producer/audio/audio_decoder.cpp
@@ -95,9 +95,9 @@ public:
 		packets_.push(packet);
 	}	
 	
-	std::vector<std::shared_ptr<std::vector<int32_t>>> poll()
+	std::vector<std::shared_ptr<core::audio_buffer>> poll()
 	{
-		std::vector<std::shared_ptr<std::vector<int32_t>>> result;
+		std::vector<std::shared_ptr<core::audio_buffer>> result;
 
 		if(packets_.empty())
 			return result;
@@ -123,7 +123,7 @@ public:
 		return result;
 	}
 
-	std::vector<std::shared_ptr<std::vector<int32_t>>> empty_poll()
+	std::vector<std::shared_ptr<core::audio_buffer>> empty_poll()
 	{
 		auto packet = packets_.front();
 		packets_.pop();
@@ -131,10 +131,10 @@ public:
 		if(!packet)			
 			return boost::assign::list_of(nullptr);
 		
-		return boost::assign::list_of(std::make_shared<std::vector<int32_t>>(format_desc_.audio_samples_per_frame, 0));	
+		return boost::assign::list_of(std::make_shared<core::audio_buffer>(format_desc_.audio_samples_per_frame, 0));	
 	}
 
-	std::shared_ptr<std::vector<int32_t>> decode(AVPacket& pkt)
+	std::shared_ptr<core::audio_buffer> decode(AVPacket& pkt)
 	{		
 		buffer1_.resize(AVCODEC_MAX_AUDIO_FRAME_SIZE*2);
 		int written_bytes = buffer1_.size() - FF_INPUT_BUFFER_PADDING_SIZE;
@@ -152,7 +152,7 @@ public:
 		const auto n_samples = buffer1_.size() / av_get_bytes_per_sample(AV_SAMPLE_FMT_S32);
 		const auto samples = reinterpret_cast<int32_t*>(buffer1_.data());
 
-		return std::make_shared<std::vector<int32_t>>(samples, samples + n_samples);
+		return std::make_shared<core::audio_buffer>(samples, samples + n_samples);
 	}
 
 	bool ready() const
@@ -164,6 +164,6 @@ public:
 audio_decoder::audio_decoder(const safe_ptr<AVFormatContext>& context, const core::video_format_desc& format_desc) : impl_(new implementation(context, format_desc)){}
 void audio_decoder::push(const std::shared_ptr<AVPacket>& packet){impl_->push(packet);}
 bool audio_decoder::ready() const{return impl_->ready();}
-std::vector<std::shared_ptr<std::vector<int32_t>>> audio_decoder::poll(){return impl_->poll();}
+std::vector<std::shared_ptr<core::audio_buffer>> audio_decoder::poll(){return impl_->poll();}
 int64_t audio_decoder::nb_frames() const{return impl_->nb_frames_;}
 }
\ No newline at end of file
diff --git a/modules/ffmpeg/producer/audio/audio_decoder.h b/modules/ffmpeg/producer/audio/audio_decoder.h
index 8ad020697..e2c978cf8 100644
--- a/modules/ffmpeg/producer/audio/audio_decoder.h
+++ b/modules/ffmpeg/producer/audio/audio_decoder.h
@@ -19,6 +19,8 @@
 */
 #pragma once
 
+#include <core/mixer/audio/audio_mixer.h>
+
 #include <common/memory/safe_ptr.h>
 
 #include <boost/noncopyable.hpp>
@@ -43,7 +45,7 @@ public:
 	
 	void push(const std::shared_ptr<AVPacket>& packet);
 	bool ready() const;
-	std::vector<std::shared_ptr<std::vector<int32_t>>> poll();
+	std::vector<std::shared_ptr<core::audio_buffer>> poll();
 
 	int64_t nb_frames() const;
 
diff --git a/modules/ffmpeg/producer/frame_muxer.cpp b/modules/ffmpeg/producer/frame_muxer.cpp
index affb1c329..4dfbcebf2 100644
--- a/modules/ffmpeg/producer/frame_muxer.cpp
+++ b/modules/ffmpeg/producer/frame_muxer.cpp
@@ -136,7 +136,7 @@ display_mode::type get_display_mode(const core::field_mode::type in_mode, double
 struct frame_muxer::implementation : boost::noncopyable
 {	
 	std::deque<std::queue<safe_ptr<write_frame>>>	video_streams_;
-	std::deque<std::vector<int32_t>>				audio_streams_;
+	std::deque<core::audio_buffer>					audio_streams_;
 	std::deque<safe_ptr<basic_frame>>				frame_buffer_;
 	display_mode::type								display_mode_;
 	const double									in_fps_;
@@ -238,12 +238,12 @@ struct frame_muxer::implementation : boost::noncopyable
 			BOOST_THROW_EXCEPTION(invalid_operation() << source_info("frame_muxer") << msg_info("video-stream overflow. This can be caused by incorrect frame-rate. Check clip meta-data."));
 	}
 
-	void push(const std::shared_ptr<std::vector<int32_t>>& audio_samples)
+	void push(const std::shared_ptr<core::audio_buffer>& audio_samples)
 	{
 		if(!audio_samples)	
 		{
 			CASPAR_LOG(debug) << L"audio-chunk-count: " << audio_sample_count_/format_desc_.audio_samples_per_frame;
-			audio_streams_.push_back(std::vector<int32_t>());
+			audio_streams_.push_back(core::audio_buffer());
 			audio_sample_count_ = 0;
 			return;
 		}
@@ -276,14 +276,14 @@ struct frame_muxer::implementation : boost::noncopyable
 		return frame;
 	}
 
-	std::vector<int32_t> pop_audio()
+	core::audio_buffer pop_audio()
 	{
 		CASPAR_VERIFY(audio_streams_.front().size() >= format_desc_.audio_samples_per_frame);
 
 		auto begin = audio_streams_.front().begin();
 		auto end   = begin + format_desc_.audio_samples_per_frame;
 
-		auto samples = std::vector<int32_t>(begin, end);
+		auto samples = core::audio_buffer(begin, end);
 		audio_streams_.front().erase(begin, end);
 
 		return samples;
@@ -410,7 +410,7 @@ struct frame_muxer::implementation : boost::noncopyable
 frame_muxer::frame_muxer(double in_fps, const safe_ptr<core::frame_factory>& frame_factory)
 	: impl_(new implementation(in_fps, frame_factory)){}
 void frame_muxer::push(const std::shared_ptr<AVFrame>& video_frame, int hints){impl_->push(video_frame, hints);}
-void frame_muxer::push(const std::shared_ptr<std::vector<int32_t>>& audio_samples){return impl_->push(audio_samples);}
+void frame_muxer::push(const std::shared_ptr<core::audio_buffer>& audio_samples){return impl_->push(audio_samples);}
 void frame_muxer::commit(){impl_->commit();}
 safe_ptr<basic_frame> frame_muxer::pop(){return impl_->pop();}
 size_t frame_muxer::size() const {return impl_->size();}
diff --git a/modules/ffmpeg/producer/frame_muxer.h b/modules/ffmpeg/producer/frame_muxer.h
index da9f41086..9390f0b85 100644
--- a/modules/ffmpeg/producer/frame_muxer.h
+++ b/modules/ffmpeg/producer/frame_muxer.h
@@ -2,6 +2,8 @@
 
 #include <common/memory/safe_ptr.h>
 
+#include <core/mixer/audio/audio_mixer.h>
+
 #include <boost/noncopyable.hpp>
 
 #include <vector>
@@ -24,7 +26,7 @@ public:
 	frame_muxer(double in_fps, const safe_ptr<core::frame_factory>& frame_factory);
 	
 	void push(const std::shared_ptr<AVFrame>& video_frame, int hints = 0);
-	void push(const std::shared_ptr<std::vector<int32_t>>& audio_samples);
+	void push(const std::shared_ptr<core::audio_buffer>& audio_samples);
 	
 	void commit();