From: Helge Norberg Date: Fri, 9 Oct 2015 09:54:07 +0000 (+0200) Subject: * Reenabled unit-test project after move to CMake. X-Git-Tag: 2.1.0_Beta1~270 X-Git-Url: https://git.sesse.net/?a=commitdiff_plain;h=a36f794e330f7a29ea4583868290817c483c10ae;p=casparcg * Reenabled unit-test project after move to CMake. * Reimplemented multichannel audio support from 2.0.7 but using ffmpeg's pan filter. - Fixed bug in caspar::array where it assumed std::uint8_t instead of T. - Started using caspar::array for audio as well, to allow for AVFrame to be the storage to avoid unnecessary copying when pan filtering audio. * Made win32_exception a caspar_exception to enable full stack trace information when an access violation or similar occurs. * FFMpeg Consumer now send more data via OSC to enable clients to indicate recording progress. --- diff --git a/CMakeLists.txt b/CMakeLists.txt index b2861c8f5..d518a4ccd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,6 +33,7 @@ set(FREEIMAGE_INCLUDE_PATH "${DEPENDENCIES_FOLDER}/freeimage/include") set(OPENAL_INCLUDE_PATH "${DEPENDENCIES_FOLDER}/openal/include") set(BLUEFISH_INCLUDE_PATH "${DEPENDENCIES_FOLDER}/bluefish/include") set(CEF_INCLUDE_PATH "${DEPENDENCIES_FOLDER}/cef/include") +set(GTEST_INCLUDE_PATH "${DEPENDENCIES_FOLDER}/gtest/include") if (MSVC) set(PLATFORM_FOLDER_NAME "win32") @@ -62,6 +63,7 @@ link_directories("${DEPENDENCIES_FOLDER}/openal/lib/${PLATFORM_FOLDER_NAME}") link_directories("${DEPENDENCIES_FOLDER}/bluefish/lib") link_directories("${DEPENDENCIES_FOLDER}/zlib/lib") link_directories("${DEPENDENCIES_FOLDER}/cef/lib/${PLATFORM_FOLDER_NAME}") +link_directories("${DEPENDENCIES_FOLDER}/gtest/lib/${PLATFORM_FOLDER_NAME}") set_property(GLOBAL PROPERTY USE_FOLDERS ON) @@ -154,3 +156,4 @@ add_subdirectory(modules) add_subdirectory(protocol) add_subdirectory(shell) +add_subdirectory(unit-test) diff --git a/accelerator/cpu/image/image_mixer.cpp b/accelerator/cpu/image/image_mixer.cpp index 8313a9f8c..a2922e56a 100644 --- a/accelerator/cpu/image/image_mixer.cpp +++ b/accelerator/cpu/image/image_mixer.cpp @@ -348,7 +348,7 @@ public: return renderer_(std::move(items_), format_desc); } - core::mutable_frame create_frame(const void* tag, const core::pixel_format_desc& desc) + core::mutable_frame create_frame(const void* tag, const core::pixel_format_desc& desc, const core::audio_channel_layout& channel_layout) { std::vector> buffers; for (auto& plane : desc.planes) @@ -356,7 +356,7 @@ public: auto buf = spl::make_shared(plane.size); buffers.push_back(array(buf->data(), plane.size, true, buf)); } - return core::mutable_frame(std::move(buffers), core::audio_buffer(), tag, desc); + return core::mutable_frame(std::move(buffers), core::mutable_audio_buffer(), tag, desc, channel_layout); } }; @@ -366,6 +366,6 @@ void image_mixer::push(const core::frame_transform& transform){impl_->push(trans void image_mixer::visit(const core::const_frame& frame){impl_->visit(frame);} void image_mixer::pop(){impl_->pop();} std::future> image_mixer::operator()(const core::video_format_desc& format_desc, bool /* straighten_alpha */){return impl_->render(format_desc);} -core::mutable_frame image_mixer::create_frame(const void* tag, const core::pixel_format_desc& desc) {return impl_->create_frame(tag, desc);} +core::mutable_frame image_mixer::create_frame(const void* tag, const core::pixel_format_desc& desc, const core::audio_channel_layout& channel_layout) {return impl_->create_frame(tag, desc, channel_layout);} }}} diff --git a/accelerator/cpu/image/image_mixer.h b/accelerator/cpu/image/image_mixer.h index 37a894204..c357a8ce1 100644 --- a/accelerator/cpu/image/image_mixer.h +++ b/accelerator/cpu/image/image_mixer.h @@ -34,7 +34,7 @@ public: std::future> operator()(const core::video_format_desc& format_desc, bool straighten_alpha) override; - core::mutable_frame create_frame(const void* tag, const core::pixel_format_desc& desc) override; + core::mutable_frame create_frame(const void* tag, const core::pixel_format_desc& desc, const core::audio_channel_layout& channel_layout) override; // Properties diff --git a/accelerator/ogl/image/image_mixer.cpp b/accelerator/ogl/image/image_mixer.cpp index 1ee5e3fb0..2c38af47e 100644 --- a/accelerator/ogl/image/image_mixer.cpp +++ b/accelerator/ogl/image/image_mixer.cpp @@ -361,13 +361,13 @@ public: return renderer_(std::move(layers_), format_desc, straighten_alpha); } - core::mutable_frame create_frame(const void* tag, const core::pixel_format_desc& desc) override + core::mutable_frame create_frame(const void* tag, const core::pixel_format_desc& desc, const core::audio_channel_layout& channel_layout) override { std::vector> buffers; for (auto& plane : desc.planes) buffers.push_back(ogl_->create_array(plane.size)); - return core::mutable_frame(std::move(buffers), core::audio_buffer(), tag, desc); + return core::mutable_frame(std::move(buffers), core::mutable_audio_buffer(), tag, desc, channel_layout); } }; @@ -377,6 +377,6 @@ void image_mixer::push(const core::frame_transform& transform){impl_->push(trans void image_mixer::visit(const core::const_frame& frame){impl_->visit(frame);} void image_mixer::pop(){impl_->pop();} std::future> image_mixer::operator()(const core::video_format_desc& format_desc, bool straighten_alpha){return impl_->render(format_desc, straighten_alpha);} -core::mutable_frame image_mixer::create_frame(const void* tag, const core::pixel_format_desc& desc) {return impl_->create_frame(tag, desc);} +core::mutable_frame image_mixer::create_frame(const void* tag, const core::pixel_format_desc& desc, const core::audio_channel_layout& channel_layout) {return impl_->create_frame(tag, desc, channel_layout);} }}} diff --git a/accelerator/ogl/image/image_mixer.h b/accelerator/ogl/image/image_mixer.h index 372ce3aec..95eebb973 100644 --- a/accelerator/ogl/image/image_mixer.h +++ b/accelerator/ogl/image/image_mixer.h @@ -50,7 +50,7 @@ public: // Methods std::future> operator()(const core::video_format_desc& format_desc, bool straighten_alpha) override; - core::mutable_frame create_frame(const void* tag, const core::pixel_format_desc& desc) override; + core::mutable_frame create_frame(const void* tag, const core::pixel_format_desc& desc, const core::audio_channel_layout& channel_layout) override; // core::image_mixer diff --git a/common/array.h b/common/array.h index 2eceb6948..3c84fdd34 100644 --- a/common/array.h +++ b/common/array.h @@ -15,18 +15,23 @@ namespace caspar { template class array final { - array(const array&); - array& operator=(const array&); + array(const array&); + array& operator=(const array&); template friend class array; public: + // Boost Range support + + typedef T* iterator; + typedef const T* const_iterator; + // Static Members // Constructors template - explicit array(std::uint8_t* ptr, std::size_t size, bool cacheable, T2&& storage) + explicit array(T* ptr, std::size_t size, bool cacheable, T2&& storage) : ptr_(ptr) , size_(size) , cacheable_(cacheable) @@ -61,7 +66,7 @@ public: T* begin() const {return ptr_;} T* data() const {return ptr_;} - T* end() const {return reinterpret_cast(reinterpret_cast(ptr_) + size_);} + T* end() const {return ptr_ + size_;} std::size_t size() const {return size_;} bool empty() const {return size() == 0;} bool cacheable() const {return cacheable_;} @@ -72,9 +77,9 @@ public: return boost::any_cast(storage_.get()); } private: - T* ptr_; - std::size_t size_; - bool cacheable_; + T* ptr_; + std::size_t size_; + bool cacheable_; std::unique_ptr storage_; }; @@ -83,13 +88,18 @@ class array final { public: + // Boost Range support + + typedef const T* iterator; + typedef const T* const_iterator; + // Static Members // Constructors array() = default; // Needed by std::future template - explicit array(const std::uint8_t* ptr, std::size_t size, bool cacheable, T2&& storage) + explicit array(const T* ptr, std::size_t size, bool cacheable, T2&& storage) : ptr_(ptr) , size_(size) , cacheable_(cacheable) @@ -97,7 +107,7 @@ public: { } - explicit array(const std::uint8_t* ptr, std::size_t size, bool cacheable) + explicit array(const T* ptr, std::size_t size, bool cacheable) : ptr_(ptr) , size_(size) , cacheable_(cacheable) @@ -143,7 +153,7 @@ public: const T* begin() const {return ptr_;} const T* data() const {return ptr_;} - const T* end() const {return reinterpret_cast(reinterpret_cast(ptr_) + size_);} + const T* end() const {return ptr_ + size_;} std::size_t size() const {return size_;} bool empty() const {return size() == 0;} bool cacheable() const {return cacheable_;} diff --git a/common/assert.h b/common/assert.h index e077804db..79dd587cf 100644 --- a/common/assert.h +++ b/common/assert.h @@ -22,6 +22,7 @@ #pragma once #include "log.h" +#include "except.h" #ifdef _MSC_VER #define _CASPAR_DBG_BREAK _CrtDbgBreak() @@ -38,6 +39,9 @@ _CASPAR_DBG_BREAK;\ }}while(0); +#define CASPAR_ENSURE(expr) do{if(!(expr)){ CASPAR_THROW_EXCEPTION(programming_error() << msg_info(std::string("Assertion Failed: ") + CASPAR_VERIFY_EXPR_STR(expr))); \ + }}while(0); + #ifdef _DEBUG #define CASPAR_ASSERT(expr) CASPAR_VERIFY(expr) #else diff --git a/common/except.h b/common/except.h index 9bdc05ccf..b06b1cacf 100644 --- a/common/except.h +++ b/common/except.h @@ -61,13 +61,10 @@ typedef boost::error_info nest struct caspar_exception : virtual boost::exception, virtual std::exception { caspar_exception(){} - explicit caspar_exception(const char* msg) : msg_(msg) {} const char* what() const throw() override { - return msg_; + return boost::diagnostic_information_what(*this); } -private: - const char* msg_ = ""; }; struct io_error : virtual caspar_exception {}; diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 744e28857..17c354778 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -10,6 +10,7 @@ set(SOURCES diagnostics/osd_graph.cpp diagnostics/subject_diagnostics.cpp + frame/audio_channel_layout.cpp frame/draw_frame.cpp frame/frame.cpp frame/frame_transform.cpp @@ -24,8 +25,6 @@ set(SOURCES producer/color/color_producer.cpp - producer/draw/freehand_producer.cpp - producer/media_info/in_memory_media_info_repository.cpp producer/scene/const_producer.cpp @@ -64,6 +63,7 @@ set(HEADERS diagnostics/osd_graph.h diagnostics/subject_diagnostics.h + frame/audio_channel_layout.h frame/draw_frame.h frame/frame.h frame/frame_factory.h @@ -87,8 +87,6 @@ set(HEADERS producer/color/color_producer.h - producer/draw/freehand_producer.h - producer/media_info/in_memory_media_info_repository.h producer/media_info/media_info.h producer/media_info/media_info_repository.h @@ -144,7 +142,6 @@ source_group(sources\\frame frame/*) source_group(sources\\help help/*) source_group(sources\\interaction interaction/*) source_group(sources\\mixer mixer/*) -source_group(sources\\producer\\draw producer/draw/*) source_group(sources\\producer\\media_info producer/media_info/*) source_group(sources\\producer\\scene producer/scene/*) source_group(sources\\producer\\text\\utils producer/text/utils/*) diff --git a/core/consumer/frame_consumer.cpp b/core/consumer/frame_consumer.cpp index 320ff13f7..5a0880b5c 100644 --- a/core/consumer/frame_consumer.cpp +++ b/core/consumer/frame_consumer.cpp @@ -29,6 +29,7 @@ #include #include +#include #include @@ -124,16 +125,16 @@ public: }).detach(); } - std::future send(const_frame frame) override {return consumer_->send(std::move(frame));} - virtual void initialize(const video_format_desc& format_desc, int channel_index) override {return consumer_->initialize(format_desc, channel_index);} - std::wstring print() const override {return consumer_->print();} - std::wstring name() const override {return consumer_->name();} - boost::property_tree::wptree info() const override {return consumer_->info();} - bool has_synchronization_clock() const override {return consumer_->has_synchronization_clock();} - int buffer_depth() const override {return consumer_->buffer_depth();} - int index() const override {return consumer_->index();} - int64_t presentation_frame_age_millis() const override {return consumer_->presentation_frame_age_millis();} - monitor::subject& monitor_output() override {return consumer_->monitor_output();} + std::future send(const_frame frame) override {return consumer_->send(std::move(frame));} + void initialize(const video_format_desc& format_desc, const audio_channel_layout& channel_layout, int channel_index) override {return consumer_->initialize(format_desc, channel_layout, channel_index);} + std::wstring print() const override {return consumer_->print();} + std::wstring name() const override {return consumer_->name();} + boost::property_tree::wptree info() const override {return consumer_->info();} + bool has_synchronization_clock() const override {return consumer_->has_synchronization_clock();} + int buffer_depth() const override {return consumer_->buffer_depth();} + int index() const override {return consumer_->index();} + int64_t presentation_frame_age_millis() const override {return consumer_->presentation_frame_age_millis();} + monitor::subject& monitor_output() override {return consumer_->monitor_output();} }; class print_consumer_proxy : public frame_consumer @@ -154,16 +155,16 @@ public: CASPAR_LOG(info) << str << L" Uninitialized."; } - std::future send(const_frame frame) override {return consumer_->send(std::move(frame));} - virtual void initialize(const video_format_desc& format_desc, int channel_index) override {return consumer_->initialize(format_desc, channel_index);} - std::wstring print() const override {return consumer_->print();} - std::wstring name() const override {return consumer_->name();} - boost::property_tree::wptree info() const override {return consumer_->info();} - bool has_synchronization_clock() const override {return consumer_->has_synchronization_clock();} - int buffer_depth() const override {return consumer_->buffer_depth();} - int index() const override {return consumer_->index();} - int64_t presentation_frame_age_millis() const override {return consumer_->presentation_frame_age_millis();} - monitor::subject& monitor_output() override {return consumer_->monitor_output();} + std::future send(const_frame frame) override {return consumer_->send(std::move(frame));} + void initialize(const video_format_desc& format_desc, const audio_channel_layout& channel_layout, int channel_index) override { return consumer_->initialize(format_desc, channel_layout, channel_index); } + std::wstring print() const override {return consumer_->print();} + std::wstring name() const override {return consumer_->name();} + boost::property_tree::wptree info() const override {return consumer_->info();} + bool has_synchronization_clock() const override {return consumer_->has_synchronization_clock();} + int buffer_depth() const override {return consumer_->buffer_depth();} + int index() const override {return consumer_->index();} + int64_t presentation_frame_age_millis() const override {return consumer_->presentation_frame_age_millis();} + monitor::subject& monitor_output() override {return consumer_->monitor_output();} }; class recover_consumer_proxy : public frame_consumer @@ -171,13 +172,14 @@ class recover_consumer_proxy : public frame_consumer std::shared_ptr consumer_; int channel_index_ = -1; video_format_desc format_desc_; + audio_channel_layout channel_layout_ = audio_channel_layout::invalid(); public: recover_consumer_proxy(spl::shared_ptr&& consumer) : consumer_(std::move(consumer)) { } - virtual std::future send(const_frame frame) + std::future send(const_frame frame) override { try { @@ -188,7 +190,7 @@ public: CASPAR_LOG_CURRENT_EXCEPTION(); try { - consumer_->initialize(format_desc_, channel_index_); + consumer_->initialize(format_desc_, channel_layout_, channel_index_); return consumer_->send(frame); } catch(...) @@ -200,11 +202,12 @@ public: } } - virtual void initialize(const video_format_desc& format_desc, int channel_index) + void initialize(const video_format_desc& format_desc, const audio_channel_layout& channel_layout, int channel_index) override { format_desc_ = format_desc; + channel_layout_ = channel_layout; channel_index_ = channel_index; - return consumer_->initialize(format_desc, channel_index); + return consumer_->initialize(format_desc, channel_layout, channel_index); } std::wstring print() const override {return consumer_->print();} @@ -223,6 +226,7 @@ class cadence_guard : public frame_consumer spl::shared_ptr consumer_; std::vector audio_cadence_; video_format_desc format_desc_; + audio_channel_layout channel_layout_ = audio_channel_layout::invalid(); boost::circular_buffer sync_buffer_; public: cadence_guard(const spl::shared_ptr& consumer) @@ -230,12 +234,13 @@ public: { } - void initialize(const video_format_desc& format_desc, int channel_index) override + void initialize(const video_format_desc& format_desc, const audio_channel_layout& channel_layout, int channel_index) override { audio_cadence_ = format_desc.audio_cadence; sync_buffer_ = boost::circular_buffer(format_desc.audio_cadence.size()); format_desc_ = format_desc; - consumer_->initialize(format_desc, channel_index); + channel_layout_ = channel_layout; + consumer_->initialize(format_desc, channel_layout, channel_index); } std::future send(const_frame frame) override @@ -245,7 +250,7 @@ public: std::future result = make_ready_future(true); - if(boost::range::equal(sync_buffer_, audio_cadence_) && audio_cadence_.front() * format_desc_.audio_channels == static_cast(frame.audio_data().size())) + if(boost::range::equal(sync_buffer_, audio_cadence_) && audio_cadence_.front() * channel_layout_.num_channels == static_cast(frame.audio_data().size())) { // Audio sent so far is in sync, now we can send the next chunk. result = consumer_->send(frame); @@ -254,7 +259,7 @@ public: else CASPAR_LOG(trace) << print() << L" Syncing audio."; - sync_buffer_.push_back(static_cast(frame.audio_data().size() / format_desc_.audio_channels)); + sync_buffer_.push_back(static_cast(frame.audio_data().size() / channel_layout_.num_channels)); return std::move(result); } @@ -325,14 +330,14 @@ const spl::shared_ptr& frame_consumer::empty() { public: std::future send(const_frame) override { return make_ready_future(false); } - void initialize(const video_format_desc&, int) override{} + void initialize(const video_format_desc&, const audio_channel_layout&, int) override{} std::wstring print() const override {return L"empty";} std::wstring name() const override {return L"empty";} bool has_synchronization_clock() const override {return false;} int buffer_depth() const override {return 0;}; int index() const override {return -1;} int64_t presentation_frame_age_millis() const override {return -1;} - monitor::subject& monitor_output() override {static monitor::subject monitor_subject(""); return monitor_subject;} + monitor::subject& monitor_output() override {static monitor::subject monitor_subject(""); return monitor_subject;} boost::property_tree::wptree info() const override { boost::property_tree::wptree info; diff --git a/core/consumer/frame_consumer.h b/core/consumer/frame_consumer.h index 9b3937b59..e09e4edcf 100644 --- a/core/consumer/frame_consumer.h +++ b/core/consumer/frame_consumer.h @@ -55,7 +55,7 @@ public: // Methods virtual std::future send(const_frame frame) = 0; - virtual void initialize(const video_format_desc& format_desc, int channel_index) = 0; + virtual void initialize(const video_format_desc& format_desc, const audio_channel_layout& channel_layout, int channel_index) = 0; // monitor::observable diff --git a/core/consumer/output.cpp b/core/consumer/output.cpp index 54578479a..5c6128d1d 100644 --- a/core/consumer/output.cpp +++ b/core/consumer/output.cpp @@ -32,6 +32,7 @@ #include "../video_format.h" #include "../frame/frame.h" +#include "../frame/audio_channel_layout.h" #include #include @@ -57,16 +58,18 @@ struct output::impl spl::shared_ptr monitor_subject_ = spl::make_shared("/output"); const int channel_index_; video_format_desc format_desc_; - std::map ports_; + audio_channel_layout channel_layout_; + std::map ports_; prec_timer sync_timer_; boost::circular_buffer frames_; std::map send_to_consumers_delays_; executor executor_ { L"output " + boost::lexical_cast(channel_index_) }; public: - impl(spl::shared_ptr graph, const video_format_desc& format_desc, int channel_index) + impl(spl::shared_ptr graph, const video_format_desc& format_desc, const audio_channel_layout& channel_layout, int channel_index) : graph_(std::move(graph)) , channel_index_(channel_index) , format_desc_(format_desc) + , channel_layout_(channel_layout) { graph_->set_color("consume-time", diagnostics::color(1.0f, 0.4f, 0.0f, 0.8f)); } @@ -75,7 +78,7 @@ public: { remove(index); - consumer->initialize(format_desc_, channel_index_); + consumer->initialize(format_desc_, channel_layout_, channel_index_); executor_.begin_invoke([this, index, consumer] { @@ -108,11 +111,11 @@ public: remove(consumer->index()); } - void set_video_format_desc(const core::video_format_desc& format_desc) + void change_channel_format(const core::video_format_desc& format_desc, const core::audio_channel_layout& channel_layout) { executor_.invoke([&] { - if(format_desc_ == format_desc) + if(format_desc_ == format_desc && channel_layout_ == channel_layout) return; auto it = ports_.begin(); @@ -120,7 +123,7 @@ public: { try { - it->second.video_format_desc(format_desc); + it->second.change_channel_format(format_desc, channel_layout); ++it; } catch(...) @@ -132,6 +135,7 @@ public: } format_desc_ = format_desc; + channel_layout_ = channel_layout; frames_.clear(); }); } @@ -156,11 +160,11 @@ public: .any(); } - void operator()(const_frame input_frame, const core::video_format_desc& format_desc) + void operator()(const_frame input_frame, const core::video_format_desc& format_desc, const core::audio_channel_layout& channel_layout) { caspar::timer frame_timer; - set_video_format_desc(format_desc); + change_channel_format(format_desc, channel_layout); executor_.invoke([=] { @@ -284,13 +288,13 @@ public: } }; -output::output(spl::shared_ptr graph, const video_format_desc& format_desc, int channel_index) : impl_(new impl(std::move(graph), format_desc, channel_index)){} +output::output(spl::shared_ptr graph, const video_format_desc& format_desc, const core::audio_channel_layout& channel_layout, int channel_index) : impl_(new impl(std::move(graph), format_desc, channel_layout, channel_index)){} void output::add(int index, const spl::shared_ptr& consumer){impl_->add(index, consumer);} void output::add(const spl::shared_ptr& consumer){impl_->add(consumer);} void output::remove(int index){impl_->remove(index);} void output::remove(const spl::shared_ptr& consumer){impl_->remove(consumer);} std::future output::info() const{return impl_->info();} std::future output::delay_info() const{ return impl_->delay_info(); } -void output::operator()(const_frame frame, const video_format_desc& format_desc){ (*impl_)(std::move(frame), format_desc); } +void output::operator()(const_frame frame, const video_format_desc& format_desc, const core::audio_channel_layout& channel_layout){ (*impl_)(std::move(frame), format_desc, channel_layout); } monitor::subject& output::monitor_output() {return *impl_->monitor_subject_;} }} diff --git a/core/consumer/output.h b/core/consumer/output.h index 7af724adc..1f2d4e320 100644 --- a/core/consumer/output.h +++ b/core/consumer/output.h @@ -45,11 +45,11 @@ public: // Constructors - explicit output(spl::shared_ptr graph, const video_format_desc& format_desc, int channel_index); + explicit output(spl::shared_ptr graph, const video_format_desc& format_desc, const core::audio_channel_layout& channel_layout, int channel_index); // Methods - void operator()(const_frame frame, const video_format_desc& format_desc); + void operator()(const_frame frame, const video_format_desc& format_desc, const core::audio_channel_layout& channel_layout); void add(const spl::shared_ptr& consumer); void add(int index, const spl::shared_ptr& consumer); diff --git a/core/consumer/port.cpp b/core/consumer/port.cpp index c91ddddfd..db3c68fa8 100644 --- a/core/consumer/port.cpp +++ b/core/consumer/port.cpp @@ -13,7 +13,7 @@ namespace caspar { namespace core { struct port::impl { int index_; - spl::shared_ptr monitor_subject_ = spl::make_shared("/port" + boost::lexical_cast(index_)); + spl::shared_ptr monitor_subject_ = spl::make_shared("/port/" + boost::lexical_cast(index_)); std::shared_ptr consumer_; int channel_index_; public: @@ -25,9 +25,9 @@ public: consumer_->monitor_output().attach_parent(monitor_subject_); } - void video_format_desc(const core::video_format_desc& format_desc) + void change_channel_format(const core::video_format_desc& format_desc, const audio_channel_layout& channel_layout) { - consumer_->initialize(format_desc, channel_index_); + consumer_->initialize(format_desc, channel_layout, channel_index_); } std::future send(const_frame frame) @@ -72,7 +72,7 @@ port::~port(){} port& port::operator=(port&& other){impl_ = std::move(other.impl_); return *this;} std::future port::send(const_frame frame){return impl_->send(std::move(frame));} monitor::subject& port::monitor_output() {return *impl_->monitor_subject_;} -void port::video_format_desc(const core::video_format_desc& format_desc){impl_->video_format_desc(format_desc);} +void port::change_channel_format(const core::video_format_desc& format_desc, const audio_channel_layout& channel_layout){impl_->change_channel_format(format_desc, channel_layout);} int port::buffer_depth() const{return impl_->buffer_depth();} std::wstring port::print() const{ return impl_->print();} bool port::has_synchronization_clock() const{return impl_->has_synchronization_clock();} diff --git a/core/consumer/port.h b/core/consumer/port.h index 1c84fa2d9..46db42c2d 100644 --- a/core/consumer/port.h +++ b/core/consumer/port.h @@ -34,7 +34,7 @@ public: // Properties - void video_format_desc(const video_format_desc& format_desc); + void change_channel_format(const video_format_desc& format_desc, const audio_channel_layout& channel_layout); std::wstring print() const; int buffer_depth() const; bool has_synchronization_clock() const; diff --git a/core/frame/audio_channel_layout.cpp b/core/frame/audio_channel_layout.cpp new file mode 100644 index 000000000..40f43a018 --- /dev/null +++ b/core/frame/audio_channel_layout.cpp @@ -0,0 +1,247 @@ +/* +* Copyright (c) 2011 Sveriges Television AB +* +* This file is part of CasparCG (www.casparcg.com). +* +* CasparCG is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* CasparCG is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with CasparCG. If not, see . +* +* Author: Helge Norberg, helge.norberg@svt.se +*/ + +#include "../StdAfx.h" + +#include "audio_channel_layout.h" + +#include +#include +#include +#include +#include +#include + +#include + +namespace caspar { namespace core { + +audio_channel_layout::audio_channel_layout() + : num_channels(0) +{ +} + +audio_channel_layout::audio_channel_layout(int num_channels, std::wstring type_, const std::wstring& channel_order_) + : num_channels(num_channels) + , type(std::move(type_)) +{ + if (num_channels < 1) + CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info(L"num_channels cannot be less than 1")); + + if (boost::contains(channel_order_, L"=") || + boost::contains(channel_order_, L"<") || + boost::contains(channel_order_, L"+") || + boost::contains(channel_order_, L"*") || + boost::contains(channel_order_, L"|")) + { + CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info( + channel_order_ + L" contains illegal characters =<+*| reserved for mix config syntax")); + } + + boost::to_upper(type); + boost::split(channel_order, channel_order_, boost::is_any_of(L" "), boost::algorithm::token_compress_on); + + if (channel_order.size() == 1 && channel_order.front().empty()) + channel_order.clear(); + + if (channel_order.size() > num_channels) + CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info( + channel_order_ + L" contains more than " + boost::lexical_cast(num_channels))); +} + +std::vector audio_channel_layout::indexes_of(const std::wstring& channel_name) const +{ + std::vector result; + for (int i = 0; i < channel_order.size(); ++i) + if (channel_name == channel_order.at(i)) + result.push_back(i); + + return result; +} + +std::wstring audio_channel_layout::print() const +{ + auto channels = boost::join(channel_order, L" "); + + return L"[audio_channel_layout] num_channels=" + boost::lexical_cast(num_channels) + L" type=" + type + L" channel_order=" + channels; +} + +const audio_channel_layout& audio_channel_layout::invalid() +{ + static const audio_channel_layout instance; + + return instance; +} + +bool operator==(const audio_channel_layout& lhs, const audio_channel_layout& rhs) +{ + return lhs.num_channels == rhs.num_channels + && boost::equal(lhs.channel_order, rhs.channel_order) + && lhs.type == rhs.type; +} + +bool operator!=(const audio_channel_layout& lhs, const audio_channel_layout& rhs) +{ + return !(lhs == rhs); +} + +struct audio_channel_layout_repository::impl +{ + mutable boost::mutex mutex_; + std::map layouts_; +}; + +audio_channel_layout_repository::audio_channel_layout_repository() + : impl_(new impl) +{ +} + +void audio_channel_layout_repository::register_layout(std::wstring name, audio_channel_layout layout) +{ + auto& self = *impl_; + boost::lock_guard lock(self.mutex_); + + boost::to_upper(name); + self.layouts_.insert(std::make_pair(std::move(name), std::move(layout))); +} + +void audio_channel_layout_repository::register_all_layouts(const boost::property_tree::wptree& layouts) +{ + auto& self = *impl_; + boost::lock_guard lock(self.mutex_); + + for (auto& layout : layouts) + { + CASPAR_VERIFY(layout.first == L"channel-layout"); + + auto name = layout.second.get(L".name"); + auto type = layout.second.get(L".type"); + auto num_channels = layout.second.get(L".num-channels"); + auto channel_order = layout.second.get(L".channel-order", L""); + + boost::to_upper(name); + self.layouts_.insert(std::make_pair( + std::move(name), + audio_channel_layout(num_channels, std::move(type), channel_order))); + } +} + +boost::optional audio_channel_layout_repository::get_layout(const std::wstring& name) const +{ + auto& self = *impl_; + boost::lock_guard lock(self.mutex_); + + auto found = self.layouts_.find(boost::to_upper_copy(name)); + + if (found == self.layouts_.end()) + return boost::none; + + return found->second; +} + +spl::shared_ptr audio_channel_layout_repository::get_default() +{ + static spl::shared_ptr instance; + + return instance; +} + +struct audio_mix_config_repository::impl +{ + mutable boost::mutex mutex_; + std::map> from_to_configs_; +}; + +audio_mix_config_repository::audio_mix_config_repository() + : impl_(new impl) +{ +} + +void audio_mix_config_repository::register_config( + const std::wstring& from_type, + const std::vector& to_types, + const std::wstring& mix_config) +{ + auto& self = *impl_; + boost::lock_guard lock(self.mutex_); + + for (auto& to_type : to_types) + self.from_to_configs_[boost::to_upper_copy(from_type)][boost::to_upper_copy(to_type)] = mix_config; +} + +void audio_mix_config_repository::register_all_configs(const boost::property_tree::wptree& configs) +{ + auto& self = *impl_; + boost::lock_guard lock(self.mutex_); + + for (auto& config : configs) + { + CASPAR_VERIFY(config.first == L"mix-config"); + + auto from_type = config.second.get(L".from-type"); + auto to_types_str = config.second.get(L".to-types"); + auto mix_config = config.second.get(L".mix"); + + boost::to_upper(from_type); + std::vector to_types; + boost::split(to_types, to_types_str, boost::is_any_of(L","), boost::algorithm::token_compress_off); + + for (auto& to_type : to_types) + { + boost::trim(to_type); + + if (to_type.empty()) + continue; + + boost::to_upper(to_type); + self.from_to_configs_[from_type][to_type] = mix_config; + } + } +} + +boost::optional audio_mix_config_repository::get_config( + const std::wstring& from_type, + const std::wstring& to_type) const +{ + auto& self = *impl_; + boost::lock_guard lock(self.mutex_); + + auto from_found = self.from_to_configs_.find(boost::to_upper_copy(from_type)); + + if (from_found == self.from_to_configs_.end()) + return boost::none; + + auto to_found = from_found->second.find(boost::to_upper_copy(to_type)); + + if (to_found == from_found->second.end()) + return boost::none; + + return to_found->second; +} + +spl::shared_ptr audio_mix_config_repository::get_default() +{ + static spl::shared_ptr instance; + + return instance; +} + +}} diff --git a/core/frame/audio_channel_layout.h b/core/frame/audio_channel_layout.h new file mode 100644 index 000000000..47aea7ca9 --- /dev/null +++ b/core/frame/audio_channel_layout.h @@ -0,0 +1,105 @@ +/* +* Copyright (c) 2011 Sveriges Television AB +* +* This file is part of CasparCG (www.casparcg.com). +* +* CasparCG is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* CasparCG is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with CasparCG. If not, see . +* +* Author: Helge Norberg, helge.norberg@svt.se +*/ + +#pragma once + +#include + +#include + +#include +#include +#include + +#include + +namespace caspar { namespace core { + +struct audio_channel_layout final +{ + int num_channels; + std::wstring type; + std::vector channel_order; + + audio_channel_layout(int num_channels, std::wstring type, const std::wstring& channel_order); + std::vector indexes_of(const std::wstring& channel_name) const; + std::wstring print() const; + static const audio_channel_layout& invalid(); +private: + audio_channel_layout(); +}; + +bool operator==(const audio_channel_layout& lhs, const audio_channel_layout& rhs); +bool operator!=(const audio_channel_layout& lhs, const audio_channel_layout& rhs); + +class audio_channel_layout_repository : boost::noncopyable +{ +public: + audio_channel_layout_repository(); + void register_layout(std::wstring name, audio_channel_layout layout); + void register_all_layouts(const boost::property_tree::wptree& layouts); + boost::optional get_layout(const std::wstring& name) const; + static spl::shared_ptr get_default(); +private: + struct impl; + spl::shared_ptr impl_; +}; + +class audio_mix_config_repository : boost::noncopyable +{ +public: + audio_mix_config_repository(); + void register_config( + const std::wstring& from_type, + const std::vector& to_types, + const std::wstring& mix_config); + void register_all_configs(const boost::property_tree::wptree& configs); + boost::optional get_config(const std::wstring& from_type, const std::wstring& to_type) const; + static spl::shared_ptr get_default(); +private: + struct impl; + spl::shared_ptr impl_; +}; + +// Implementation in ffmpeg module. +class audio_channel_remapper : boost::noncopyable +{ +public: + audio_channel_remapper( + audio_channel_layout input_layout, + audio_channel_layout output_layout, + spl::shared_ptr mix_repo = audio_mix_config_repository::get_default()); + + /** + * Perform downmix/upmix/rearranging of audio data if needed. + * + * @param input The input audio buffer. + * + * @return input if the input layout is the same as the output layout. + * otherwise the mixed buffer (valid until the next call). + */ + audio_buffer mix_and_rearrange(audio_buffer input); +private: + struct impl; + spl::shared_ptr impl_; +}; + +}} diff --git a/core/frame/frame.cpp b/core/frame/frame.cpp index 83164f04d..f9b97f991 100644 --- a/core/frame/frame.cpp +++ b/core/frame/frame.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include @@ -43,16 +44,23 @@ namespace caspar { namespace core { struct mutable_frame::impl : boost::noncopyable { std::vector> buffers_; - core::audio_buffer audio_data_; + core::mutable_audio_buffer audio_data_; const core::pixel_format_desc desc_; + const core::audio_channel_layout channel_layout_; const void* tag_; core::frame_geometry geometry_ = frame_geometry::get_default(); caspar::timer since_created_timer_; - impl(std::vector> buffers, audio_buffer audio_buffer, const void* tag, const core::pixel_format_desc& desc) + impl( + std::vector> buffers, + mutable_audio_buffer audio_data, + const void* tag, + const core::pixel_format_desc& desc, + const core::audio_channel_layout& channel_layout) : buffers_(std::move(buffers)) - , audio_data_(std::move(audio_buffer)) + , audio_data_(std::move(audio_data)) , desc_(desc) + , channel_layout_(channel_layout) , tag_(tag) { for (auto& buffer : buffers_) @@ -61,8 +69,13 @@ struct mutable_frame::impl : boost::noncopyable } }; -mutable_frame::mutable_frame(std::vector> image_buffers, audio_buffer audio_buffer, const void* tag, const core::pixel_format_desc& desc) - : impl_(new impl(std::move(image_buffers), std::move(audio_buffer), tag, desc)){} +mutable_frame::mutable_frame( + std::vector> image_buffers, + mutable_audio_buffer audio_data, + const void* tag, + const core::pixel_format_desc& desc, + const core::audio_channel_layout& channel_layout) + : impl_(new impl(std::move(image_buffers), std::move(audio_data), tag, desc, channel_layout)){} mutable_frame::~mutable_frame(){} mutable_frame::mutable_frame(mutable_frame&& other) : impl_(std::move(other.impl_)){} mutable_frame& mutable_frame::operator=(mutable_frame&& other) @@ -72,10 +85,11 @@ mutable_frame& mutable_frame::operator=(mutable_frame&& other) } void mutable_frame::swap(mutable_frame& other){impl_.swap(other.impl_);} const core::pixel_format_desc& mutable_frame::pixel_format_desc() const{return impl_->desc_;} +const core::audio_channel_layout& mutable_frame::audio_channel_layout() const { return impl_->channel_layout_; } const array& mutable_frame::image_data(std::size_t index) const{return impl_->buffers_.at(index);} -const core::audio_buffer& mutable_frame::audio_data() const{return impl_->audio_data_;} +const core::mutable_audio_buffer& mutable_frame::audio_data() const{return impl_->audio_data_;} array& mutable_frame::image_data(std::size_t index){return impl_->buffers_.at(index);} -core::audio_buffer& mutable_frame::audio_data(){return impl_->audio_data_;} +core::mutable_audio_buffer& mutable_frame::audio_data(){return impl_->audio_data_;} std::size_t mutable_frame::width() const{return impl_->desc_.planes.at(0).width;} std::size_t mutable_frame::height() const{return impl_->desc_.planes.at(0).height;} const void* mutable_frame::stream_tag()const{return impl_->tag_;} @@ -94,8 +108,9 @@ const const_frame& const_frame::empty() struct const_frame::impl : boost::noncopyable { mutable std::vector>> future_buffers_; - core::audio_buffer audio_data_; + mutable core::audio_buffer audio_data_; const core::pixel_format_desc desc_; + const core::audio_channel_layout channel_layout_; const void* tag_; core::frame_geometry geometry_; caspar::timer since_created_timer_; @@ -103,7 +118,9 @@ struct const_frame::impl : boost::noncopyable mutable tbb::atomic recorded_age_; impl(const void* tag) - : desc_(core::pixel_format::invalid) + : audio_data_(0, 0, true, 0) + , desc_(core::pixel_format::invalid) + , channel_layout_(audio_channel_layout::invalid()) , tag_(tag) , geometry_(frame_geometry::get_default()) , should_record_age_(true) @@ -111,9 +128,15 @@ struct const_frame::impl : boost::noncopyable recorded_age_ = 0; } - impl(std::shared_future> image, audio_buffer audio_buffer, const void* tag, const core::pixel_format_desc& desc) - : audio_data_(std::move(audio_buffer)) + impl( + std::shared_future> image, + audio_buffer audio_data, + const void* tag, + const core::pixel_format_desc& desc, + const core::audio_channel_layout& channel_layout) + : audio_data_(std::move(audio_data)) , desc_(desc) + , channel_layout_(channel_layout) , tag_(tag) , geometry_(frame_geometry::get_default()) , should_record_age_(false) @@ -125,13 +148,18 @@ struct const_frame::impl : boost::noncopyable } impl(mutable_frame&& other) - : audio_data_(other.audio_data()) + : audio_data_(0, 0, true, 0) // Complex init done in body instead. , desc_(other.pixel_format_desc()) + , channel_layout_(other.audio_channel_layout()) , tag_(other.stream_tag()) , geometry_(other.geometry()) , since_created_timer_(other.since_created()) , should_record_age_(true) { + spl::shared_ptr shared_audio_data(new mutable_audio_buffer(std::move(other.audio_data()))); + // pointer returned by vector::data() should be the same after move, but just to be safe. + audio_data_ = audio_buffer(shared_audio_data->data(), shared_audio_data->size(), true, std::move(shared_audio_data)); + for (std::size_t n = 0; n < desc_.planes.size(); ++n) { future_buffers_.push_back(make_ready_future>(std::move(other.image_data(n))).share()); @@ -175,8 +203,13 @@ struct const_frame::impl : boost::noncopyable }; const_frame::const_frame(const void* tag) : impl_(new impl(tag)){} -const_frame::const_frame(std::shared_future> image, audio_buffer audio_buffer, const void* tag, const core::pixel_format_desc& desc) - : impl_(new impl(std::move(image), std::move(audio_buffer), tag, desc)){} +const_frame::const_frame( + std::shared_future> image, + audio_buffer audio_data, + const void* tag, + const core::pixel_format_desc& desc, + const core::audio_channel_layout& channel_layout) + : impl_(new impl(std::move(image), std::move(audio_data), tag, desc, channel_layout)){} const_frame::const_frame(mutable_frame&& other) : impl_(new impl(std::move(other))){} const_frame::~const_frame(){} const_frame::const_frame(const_frame&& other) : impl_(std::move(other.impl_)){} @@ -196,6 +229,7 @@ bool const_frame::operator!=(const const_frame& other){return !(*this == other); bool const_frame::operator<(const const_frame& other){return impl_ < other.impl_;} bool const_frame::operator>(const const_frame& other){return impl_ > other.impl_;} const core::pixel_format_desc& const_frame::pixel_format_desc()const{return impl_->desc_;} +const core::audio_channel_layout& const_frame::audio_channel_layout()const { return impl_->channel_layout_; } array const_frame::image_data(int index)const{return impl_->image_data(index);} const core::audio_buffer& const_frame::audio_data()const{return impl_->audio_data_;} std::size_t const_frame::width()const{return impl_->width();} diff --git a/core/frame/frame.h b/core/frame/frame.h index aca02495e..f2a93bfd1 100644 --- a/core/frame/frame.h +++ b/core/frame/frame.h @@ -19,7 +19,8 @@ FORWARD1(boost, template class shared_future); namespace caspar { namespace core { -typedef cache_aligned_vector audio_buffer; +typedef caspar::array audio_buffer; +typedef cache_aligned_vector mutable_audio_buffer; class frame_geometry; class mutable_frame final @@ -33,9 +34,10 @@ public: // Constructors explicit mutable_frame(std::vector> image_buffers, - audio_buffer audio_buffer, + mutable_audio_buffer audio_data, const void* tag, - const pixel_format_desc& desc); + const pixel_format_desc& desc, + const audio_channel_layout& channel_layout); ~mutable_frame(); // Methods @@ -48,12 +50,13 @@ public: // Properties const core::pixel_format_desc& pixel_format_desc() const; + const core::audio_channel_layout& audio_channel_layout() const; const array& image_data(std::size_t index = 0) const; - const core::audio_buffer& audio_data() const; + const core::mutable_audio_buffer& audio_data() const; array& image_data(std::size_t index = 0); - core::audio_buffer& audio_data(); + core::mutable_audio_buffer& audio_data(); std::size_t width() const; std::size_t height() const; @@ -83,9 +86,10 @@ public: explicit const_frame(const void* tag = nullptr); explicit const_frame(std::shared_future> image, - audio_buffer audio_buffer, + audio_buffer audio_data, const void* tag, - const pixel_format_desc& desc); + const pixel_format_desc& desc, + const audio_channel_layout& channel_layout); const_frame(mutable_frame&& other); ~const_frame(); @@ -99,6 +103,7 @@ public: // Properties const core::pixel_format_desc& pixel_format_desc() const; + const core::audio_channel_layout& audio_channel_layout() const; array image_data(int index = 0) const; const core::audio_buffer& audio_data() const; diff --git a/core/frame/frame_factory.h b/core/frame/frame_factory.h index 5aa865af0..ac2cda113 100644 --- a/core/frame/frame_factory.h +++ b/core/frame/frame_factory.h @@ -42,7 +42,10 @@ public: // Methods - virtual mutable_frame create_frame(const void* video_stream_tag, const pixel_format_desc& desc) = 0; + virtual mutable_frame create_frame( + const void* video_stream_tag, + const pixel_format_desc& desc, + const core::audio_channel_layout& channel_layout) = 0; // Properties }; diff --git a/core/fwd.h b/core/fwd.h index 6a0129bcf..433798a8b 100644 --- a/core/fwd.h +++ b/core/fwd.h @@ -50,3 +50,6 @@ FORWARD2(caspar, core, class help_sink); FORWARD2(caspar, core, class help_repository); FORWARD2(caspar, core, struct module_dependencies); FORWARD2(caspar, core, class frame_producer_registry); +FORWARD2(caspar, core, struct audio_channel_layout); +FORWARD2(caspar, core, class audio_channel_layout_repository); +FORWARD2(caspar, core, class audio_mix_config_repository); diff --git a/core/mixer/audio/audio_mixer.cpp b/core/mixer/audio/audio_mixer.cpp index a030526e9..e2bb430d1 100644 --- a/core/mixer/audio/audio_mixer.cpp +++ b/core/mixer/audio/audio_mixer.cpp @@ -25,6 +25,7 @@ #include #include +#include #include #include @@ -42,9 +43,10 @@ namespace caspar { namespace core { struct audio_item { - const void* tag = nullptr; - audio_transform transform; - audio_buffer audio_data; + const void* tag = nullptr; + audio_transform transform; + audio_buffer audio_data; + audio_channel_layout channel_layout = audio_channel_layout::invalid(); audio_item() { @@ -54,6 +56,7 @@ struct audio_item : tag(std::move(other.tag)) , transform(std::move(other.transform)) , audio_data(std::move(other.audio_data)) + , channel_layout(std::move(other.channel_layout)) { } }; @@ -62,9 +65,11 @@ typedef cache_aligned_vector audio_buffer_ps; struct audio_stream { - audio_transform prev_transform; - audio_buffer_ps audio_data; - bool is_still = false; + audio_transform prev_transform; + audio_buffer_ps audio_data; + std::unique_ptr channel_remapper; + bool remapping_failed = false; + bool is_still = false; }; struct audio_mixer::impl : boost::noncopyable @@ -75,6 +80,7 @@ struct audio_mixer::impl : boost::noncopyable std::vector items_; std::vector audio_cadence_; video_format_desc format_desc_; + audio_channel_layout channel_layout_ = audio_channel_layout::invalid(); float master_volume_ = 1.0f; float previous_master_volume_ = master_volume_; spl::shared_ptr graph_; @@ -97,9 +103,10 @@ public: return; audio_item item; - item.tag = frame.stream_tag(); - item.transform = transform_stack_.top(); - item.audio_data = frame.audio_data(); + item.tag = frame.stream_tag(); + item.transform = transform_stack_.top(); + item.audio_data = frame.audio_data(); + item.channel_layout = frame.audio_channel_layout(); if(item.transform.is_still) item.transform.volume = 0.0; @@ -127,13 +134,14 @@ public: return master_volume_; } - audio_buffer mix(const video_format_desc& format_desc) + audio_buffer mix(const video_format_desc& format_desc, const audio_channel_layout& channel_layout) { - if(format_desc_ != format_desc) + if(format_desc_ != format_desc || channel_layout_ != channel_layout) { audio_streams_.clear(); audio_cadence_ = format_desc.audio_cadence; format_desc_ = format_desc; + channel_layout_ = channel_layout; } std::map next_audio_streams; @@ -142,6 +150,8 @@ public: for (auto& item : items_) { audio_buffer_ps next_audio; + std::unique_ptr channel_remapper; + bool remapping_failed = false; auto next_transform = item.transform; auto prev_transform = next_transform; @@ -154,32 +164,66 @@ public: used_tags.push_back(tag); const auto it = audio_streams_.find(tag); - if(it != audio_streams_.end()) - { - prev_transform = it->second.prev_transform; - next_audio = std::move(it->second.audio_data); + if (it != audio_streams_.end()) + { + prev_transform = it->second.prev_transform; + next_audio = std::move(it->second.audio_data); + channel_remapper = std::move(it->second.channel_remapper); + remapping_failed = it->second.remapping_failed; } + if (remapping_failed) + { + CASPAR_LOG(trace) << "[audio_mixer] audio channel remapping already failed for stream."; + next_audio_streams[tag].remapping_failed = true; + continue; + } + // Skip it if there is no existing audio stream and item has no audio-data. if(it == audio_streams_.end() && item.audio_data.empty()) continue; - + + if (item.channel_layout == audio_channel_layout::invalid()) + { + CASPAR_LOG(debug) << "[audio_mixer] invalid audio channel layout for item"; + continue; + } + + if (!channel_remapper) + { + try + { + channel_remapper.reset(new audio_channel_remapper(item.channel_layout, channel_layout_)); + } + catch (...) + { + CASPAR_LOG_CURRENT_EXCEPTION(); + CASPAR_LOG(error) << "[audio_mixer] audio channel remapping failed for stream."; + next_audio_streams[tag].remapping_failed = true; + continue; + } + } + + item.audio_data = channel_remapper->mix_and_rearrange(item.audio_data); + const float prev_volume = static_cast(prev_transform.volume) * previous_master_volume_; const float next_volume = static_cast(next_transform.volume) * master_volume_; // TODO: Move volume mixing into code below, in order to support audio sample counts not corresponding to frame audio samples. - auto alpha = (next_volume-prev_volume)/static_cast(item.audio_data.size()/format_desc.audio_channels); + auto alpha = (next_volume-prev_volume)/static_cast(item.audio_data.size()/channel_layout_.num_channels); for(size_t n = 0; n < item.audio_data.size(); ++n) { - auto sample_multiplier = (prev_volume + (n/format_desc_.audio_channels) * alpha); - next_audio.push_back(item.audio_data[n] * sample_multiplier); + auto sample_multiplier = (prev_volume + (n / channel_layout_.num_channels) * alpha); + next_audio.push_back(item.audio_data.data()[n] * sample_multiplier); } - next_audio_streams[tag].prev_transform = std::move(next_transform); // Store all active tags, inactive tags will be removed at the end. - next_audio_streams[tag].audio_data = std::move(next_audio); - next_audio_streams[tag].is_still = item.transform.is_still; - } + next_audio_streams[tag].prev_transform = std::move(next_transform); // Store all active tags, inactive tags will be removed at the end. + next_audio_streams[tag].audio_data = std::move(next_audio); + next_audio_streams[tag].channel_remapper = std::move(channel_remapper); + next_audio_streams[tag].remapping_failed = remapping_failed; + next_audio_streams[tag].is_still = item.transform.is_still; + } previous_master_volume_ = master_volume_; items_.clear(); @@ -193,7 +237,10 @@ public: auto nb_invalid_streams = cpplinq::from(audio_streams_) .select(values()) - .where([&](const audio_stream& x) { return x.audio_data.size() < audio_size(audio_cadence_.front()); }) + .where([&](const audio_stream& x) + { + return !x.remapping_failed && x.audio_data.size() < audio_size(audio_cadence_.front()); + }) .count(); if(nb_invalid_streams > 0) @@ -212,11 +259,12 @@ public: boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1); - audio_buffer result; + auto result_owner = spl::make_shared(); + auto& result = *result_owner; result.reserve(result_ps.size()); boost::range::transform(result_ps, std::back_inserter(result), [](float sample){return static_cast(sample);}); - const int num_channels = format_desc_.audio_channels; + const int num_channels = channel_layout_.num_channels; monitor_subject_ << monitor::message("/nb_channels") % num_channels; auto max = std::vector(num_channels, std::numeric_limits::min()); @@ -242,12 +290,12 @@ public: graph_->set_value("volume", static_cast(*boost::max_element(max)) / std::numeric_limits::max()); - return result; + return caspar::array(result.data(), result.size(), true, std::move(result_owner)); } size_t audio_size(size_t num_samples) const { - return num_samples * format_desc_.audio_channels; + return num_samples * channel_layout_.num_channels; } }; @@ -257,7 +305,7 @@ void audio_mixer::visit(const const_frame& frame){impl_->visit(frame);} void audio_mixer::pop(){impl_->pop();} void audio_mixer::set_master_volume(float volume) { impl_->set_master_volume(volume); } float audio_mixer::get_master_volume() { return impl_->get_master_volume(); } -audio_buffer audio_mixer::operator()(const video_format_desc& format_desc){ return impl_->mix(format_desc); } +audio_buffer audio_mixer::operator()(const video_format_desc& format_desc, const audio_channel_layout& channel_layout){ return impl_->mix(format_desc, channel_layout); } monitor::subject& audio_mixer::monitor_output(){ return impl_->monitor_subject_; } }} diff --git a/core/mixer/audio/audio_mixer.h b/core/mixer/audio/audio_mixer.h index 1a93008fe..12e31a177 100644 --- a/core/mixer/audio/audio_mixer.h +++ b/core/mixer/audio/audio_mixer.h @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -35,7 +36,7 @@ FORWARD2(caspar, diagnostics, class graph); namespace caspar { namespace core { -typedef cache_aligned_vector audio_buffer; +typedef caspar::array audio_buffer; class audio_mixer final : public frame_visitor { @@ -47,11 +48,11 @@ public: // Constructors - audio_mixer(spl::shared_ptr graph); + audio_mixer(spl::shared_ptr<::caspar::diagnostics::graph> graph); // Methods - audio_buffer operator()(const struct video_format_desc& format_desc); + audio_buffer operator()(const struct video_format_desc& format_desc, const struct audio_channel_layout& channel_layout); void set_master_volume(float volume); float get_master_volume(); monitor::subject& monitor_output(); diff --git a/core/mixer/image/image_mixer.h b/core/mixer/image/image_mixer.h index 316677bc5..5c1a01292 100644 --- a/core/mixer/image/image_mixer.h +++ b/core/mixer/image/image_mixer.h @@ -61,7 +61,7 @@ public: virtual std::future> operator()(const struct video_format_desc& format_desc, bool straighten_alpha) = 0; - virtual class mutable_frame create_frame(const void* tag, const struct pixel_format_desc& desc) = 0; + virtual class mutable_frame create_frame(const void* tag, const struct pixel_format_desc& desc, const core::audio_channel_layout& channel_layout) = 0; // Properties }; diff --git a/core/mixer/mixer.cpp b/core/mixer/mixer.cpp index 30061d522..52efcbe45 100644 --- a/core/mixer/mixer.cpp +++ b/core/mixer/mixer.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include @@ -77,7 +78,7 @@ public: audio_mixer_.monitor_output().attach_parent(monitor_subject_); } - const_frame operator()(std::map frames, const video_format_desc& format_desc) + const_frame operator()(std::map frames, const video_format_desc& format_desc, const core::audio_channel_layout& channel_layout) { caspar::timer frame_timer; @@ -97,11 +98,11 @@ public: } auto image = (*image_mixer_)(format_desc, straighten_alpha_); - auto audio = audio_mixer_(format_desc); + auto audio = audio_mixer_(format_desc, channel_layout); auto desc = core::pixel_format_desc(core::pixel_format::bgra); desc.planes.push_back(core::pixel_format_desc::plane(format_desc.width, format_desc.height, 4)); - return const_frame(std::move(image), std::move(audio), this, desc); + return const_frame(std::move(image), std::move(audio), this, desc, channel_layout); } catch(...) { @@ -174,7 +175,7 @@ void mixer::set_straight_alpha_output(bool value) { impl_->set_straight_alpha_ou bool mixer::get_straight_alpha_output() { return impl_->get_straight_alpha_output(); } std::future mixer::info() const{return impl_->info();} std::future mixer::delay_info() const{ return impl_->delay_info(); } -const_frame mixer::operator()(std::map frames, const video_format_desc& format_desc){ return (*impl_)(std::move(frames), format_desc); } -mutable_frame mixer::create_frame(const void* tag, const core::pixel_format_desc& desc) {return impl_->image_mixer_->create_frame(tag, desc);} +const_frame mixer::operator()(std::map frames, const video_format_desc& format_desc, const core::audio_channel_layout& channel_layout){ return (*impl_)(std::move(frames), format_desc, channel_layout); } +mutable_frame mixer::create_frame(const void* tag, const core::pixel_format_desc& desc, const core::audio_channel_layout& channel_layout) {return impl_->image_mixer_->create_frame(tag, desc, channel_layout);} monitor::subject& mixer::monitor_output() { return *impl_->monitor_subject_; } }} diff --git a/core/mixer/mixer.h b/core/mixer/mixer.h index 9b4b61283..7203191ab 100644 --- a/core/mixer/mixer.h +++ b/core/mixer/mixer.h @@ -49,18 +49,18 @@ public: // Constructors - explicit mixer(int channel_index, spl::shared_ptr graph, spl::shared_ptr image_mixer); + explicit mixer(int channel_index, spl::shared_ptr graph, spl::shared_ptr image_mixer); // Methods - const_frame operator()(std::map frames, const video_format_desc& format_desc); + const_frame operator()(std::map frames, const video_format_desc& format_desc, const core::audio_channel_layout& channel_layout); void set_master_volume(float volume); float get_master_volume(); void set_straight_alpha_output(bool value); bool get_straight_alpha_output(); - mutable_frame create_frame(const void* tag, const pixel_format_desc& desc); + mutable_frame create_frame(const void* tag, const pixel_format_desc& desc, const core::audio_channel_layout& channel_layout); // Properties diff --git a/core/producer/color/color_producer.cpp b/core/producer/color/color_producer.cpp index df1059d84..dd82ec198 100644 --- a/core/producer/color/color_producer.cpp +++ b/core/producer/color/color_producer.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -177,7 +178,7 @@ draw_frame create_color_frame(void* tag, const spl::shared_ptr& f { core::pixel_format_desc desc(pixel_format::bgra); desc.planes.push_back(core::pixel_format_desc::plane(1, 1, 4)); - auto frame = frame_factory->create_frame(tag, desc); + auto frame = frame_factory->create_frame(tag, desc, core::audio_channel_layout::invalid()); *reinterpret_cast(frame.image_data(0).begin()) = value; diff --git a/core/producer/draw/freehand_producer.cpp b/core/producer/draw/freehand_producer.cpp deleted file mode 100644 index cdc6639bc..000000000 --- a/core/producer/draw/freehand_producer.cpp +++ /dev/null @@ -1,181 +0,0 @@ -/* -* Copyright (c) 2011 Sveriges Television AB -* -* This file is part of CasparCG (www.casparcg.com). -* -* CasparCG is free software: you can redistribute it and/or modify -* it under the terms of the GNU General Public License as published by -* the Free Software Foundation, either version 3 of the License, or -* (at your option) any later version. -* -* CasparCG is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with CasparCG. If not, see . -* -* Author: Helge Norberg, helge.norberg@svt.se -*/ - -#include "../../StdAfx.h" - -#include "freehand_producer.h" - -#include -#include -#include -#include -#include -#include - -namespace caspar { namespace core { - -cache_aligned_vector empty_drawing(int width, int height) -{ - cache_aligned_vector result; - - result.resize(width * height * 4); - std::fill(result.begin(), result.end(), 0); - - return std::move(result); -} - -class freehand_producer : public frame_producer_base -{ - monitor::subject monitor_subject_; - cache_aligned_vector drawing_; - spl::shared_ptr frame_factory_; - constraints constraints_; - draw_frame frame_; - bool button_pressed_; - bool modified_; -public: - explicit freehand_producer( - const spl::shared_ptr& frame_factory, - int width, - int height) - : drawing_(std::move(empty_drawing(width, height))) - , frame_factory_(frame_factory) - , constraints_(width, height) - , frame_(create_frame()) - , button_pressed_(false) - , modified_(false) - { - CASPAR_LOG(info) << print() << L" Initialized"; - } - - draw_frame create_frame() - { - pixel_format_desc desc(pixel_format::bgra); - desc.planes.push_back(pixel_format_desc::plane( - static_cast(constraints_.width.get()), - static_cast(constraints_.height.get()), - 4)); - auto frame = frame_factory_->create_frame(this, desc); - - std::memcpy(frame.image_data().begin(), drawing_.data(), drawing_.size()); - - return draw_frame(std::move(frame)); - } - - // frame_producer - - void on_interaction(const interaction_event::ptr& event) override - { - if (is(event) && button_pressed_) - { - auto mouse_move = as(event); - - int x = static_cast(mouse_move->x * constraints_.width.get()); - int y = static_cast(mouse_move->y * constraints_.height.get()); - - if (x >= constraints_.width.get() - || y >= constraints_.height.get() - || x < 0 - || y < 0) - return; - - uint8_t* b = drawing_.data() + (y * static_cast(constraints_.width.get()) + x) * 4; - uint8_t* g = b + 1; - uint8_t* r = b + 2; - uint8_t* a = b + 3; - - *b = 255; - *g = 255; - *r = 255; - *a = 255; - - modified_ = true; - } - else if (is(event)) - { - auto button_event = as(event); - - if (button_event->button == 0) - button_pressed_ = button_event->pressed; - else if (button_event->button == 1 && button_event->pressed) - { - std::memset(drawing_.data(), 0, drawing_.size()); - modified_ = true; - } - } - } - - bool collides(double x, double y) const override - { - return true; - } - - draw_frame receive_impl() override - { - if (modified_) - { - frame_ = create_frame(); - modified_ = false; - } - - return frame_; - } - - constraints& pixel_constraints() override - { - return constraints_; - } - - std::wstring print() const override - { - return L"freehand[]"; - } - - std::wstring name() const override - { - return L"freehand"; - } - - boost::property_tree::wptree info() const override - { - boost::property_tree::wptree info; - info.add(L"type", L"freehand"); - return info; - } - - monitor::subject& monitor_output() - { - return monitor_subject_; - } -}; - -spl::shared_ptr create_freehand_producer(const spl::shared_ptr& frame_factory, const std::vector& params) -{ - if(params.size() < 3 || !boost::iequals(params.at(0), L"[FREEHAND]")) - return core::frame_producer::empty(); - - int width = boost::lexical_cast(params.at(1)); - int height = boost::lexical_cast(params.at(2)); - - return spl::make_shared(frame_factory, width, height); -} - -}} diff --git a/core/producer/draw/freehand_producer.h b/core/producer/draw/freehand_producer.h deleted file mode 100644 index b8ac1c79e..000000000 --- a/core/producer/draw/freehand_producer.h +++ /dev/null @@ -1,35 +0,0 @@ -/* -* Copyright (c) 2011 Sveriges Television AB -* -* This file is part of CasparCG (www.casparcg.com). -* -* CasparCG is free software: you can redistribute it and/or modify -* it under the terms of the GNU General Public License as published by -* the Free Software Foundation, either version 3 of the License, or -* (at your option) any later version. -* -* CasparCG is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with CasparCG. If not, see . -* -* Author: Helge Norberg, helge.norberg@svt.se -*/ - -#pragma once - -#include - -#include - -#include -#include - -namespace caspar { namespace core { - -spl::shared_ptr create_freehand_producer(const spl::shared_ptr& frame_factory, const std::vector& params); - -}} diff --git a/core/producer/frame_producer.cpp b/core/producer/frame_producer.cpp index dacf09203..7312937f3 100644 --- a/core/producer/frame_producer.cpp +++ b/core/producer/frame_producer.cpp @@ -27,7 +27,6 @@ #include "../frame/frame_transform.h" #include "color/color_producer.h" -#include "draw/freehand_producer.h" #include "separated/separated_producer.h" #include "variable.h" @@ -330,9 +329,6 @@ spl::shared_ptr do_create_producer(const frame_producer_de if(producer == frame_producer::empty()) producer = create_color_producer(dependencies.frame_factory, params); - if (producer == frame_producer::empty()) - producer = create_freehand_producer(dependencies.frame_factory, params); - if(producer == frame_producer::empty()) return producer; diff --git a/core/producer/stage.h b/core/producer/stage.h index a25d72e99..9912258fa 100644 --- a/core/producer/stage.h +++ b/core/producer/stage.h @@ -57,7 +57,7 @@ public: // Constructors - explicit stage(int channel_index, spl::shared_ptr graph); + explicit stage(int channel_index, spl::shared_ptr graph); // Methods diff --git a/core/producer/text/text_producer.cpp b/core/producer/text/text_producer.cpp index 27424f5c7..89756f555 100644 --- a/core/producer/text/text_producer.cpp +++ b/core/producer/text/text_producer.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -188,7 +189,7 @@ public: text::string_metrics metrics; font_.set_tracking(static_cast(tracking_.value().get())); auto vertex_stream = font_.create_vertex_stream(text_.value().get(), x_, y_, parent_width_, parent_height_, &metrics); - auto frame = frame_factory_->create_frame(vertex_stream.data(), pfd); + auto frame = frame_factory_->create_frame(vertex_stream.data(), pfd, core::audio_channel_layout::invalid()); memcpy(frame.image_data().data(), atlas_.data(), frame.image_data().size()); frame.set_geometry(frame_geometry(frame_geometry::geometry_type::quad_list, std::move(vertex_stream))); diff --git a/core/thumbnail_generator.cpp b/core/thumbnail_generator.cpp index 085fee819..fd3ccb188 100644 --- a/core/thumbnail_generator.cpp +++ b/core/thumbnail_generator.cpp @@ -44,6 +44,7 @@ #include "frame/frame.h" #include "frame/draw_frame.h" #include "frame/frame_transform.h" +#include "frame/audio_channel_layout.h" #include "producer/media_info/media_info.h" #include "producer/media_info/media_info_repository.h" @@ -330,7 +331,7 @@ public: std::shared_ptr ticket(nullptr, [&thumbnail_ready](void*) { thumbnail_ready.set_value(); }); - auto mixed_frame = mixer_(std::move(frames), format_desc_); + auto mixed_frame = mixer_(std::move(frames), format_desc_, audio_channel_layout(2, L"stereo", L"")); output_->send(std::move(mixed_frame), ticket); ticket.reset(); diff --git a/core/video_channel.cpp b/core/video_channel.cpp index d91aee379..78520b060 100644 --- a/core/video_channel.cpp +++ b/core/video_channel.cpp @@ -31,6 +31,7 @@ #include "frame/frame.h" #include "frame/draw_frame.h" #include "frame/frame_factory.h" +#include "frame/audio_channel_layout.h" #include #include @@ -58,7 +59,9 @@ struct video_channel::impl final mutable tbb::spin_mutex format_desc_mutex_; core::video_format_desc format_desc_; - + mutable tbb::spin_mutex channel_layout_mutex_; + core::audio_channel_layout channel_layout_; + const spl::shared_ptr graph_ = [](int index) { core::diagnostics::scoped_call_context save; @@ -73,12 +76,17 @@ struct video_channel::impl final executor executor_ { L"video_channel " + boost::lexical_cast(index_) }; public: - impl(int index, const core::video_format_desc& format_desc, std::unique_ptr image_mixer) + impl( + int index, + const core::video_format_desc& format_desc, + const core::audio_channel_layout& channel_layout, + std::unique_ptr image_mixer) : monitor_subject_(spl::make_shared( "/channel/" + boost::lexical_cast(index))) , index_(index) , format_desc_(format_desc) - , output_(graph_, format_desc, index) + , channel_layout_(channel_layout) + , output_(graph_, format_desc, channel_layout, index) , image_mixer_(std::move(image_mixer)) , mixer_(index, graph_, image_mixer_) , stage_(index, graph_) @@ -100,7 +108,7 @@ public: { CASPAR_LOG(info) << print() << " Uninitializing."; } - + core::video_format_desc video_format_desc() const { return lock(format_desc_mutex_, [&] @@ -108,7 +116,7 @@ public: return format_desc_; }); } - + void video_format_desc(const core::video_format_desc& format_desc) { lock(format_desc_mutex_, [&] @@ -118,12 +126,30 @@ public: }); } + core::audio_channel_layout audio_channel_layout() const + { + return lock(channel_layout_mutex_, [&] + { + return channel_layout_; + }); + } + + void audio_channel_layout(const core::audio_channel_layout& channel_layout) + { + lock(channel_layout_mutex_, [&] + { + channel_layout_ = channel_layout; + stage_.clear(); + }); + } + void tick() { try { - auto format_desc = video_format_desc(); + auto format_desc = video_format_desc(); + auto channel_layout = audio_channel_layout(); caspar::timer frame_timer; @@ -133,11 +159,11 @@ public: // Mix - auto mixed_frame = mixer_(std::move(stage_frames), format_desc); + auto mixed_frame = mixer_(std::move(stage_frames), format_desc, channel_layout); // Consume - output_(std::move(mixed_frame), format_desc); + output_(std::move(mixed_frame), format_desc, channel_layout); auto frame_time = frame_timer.elapsed()*format_desc.fps*0.5; graph_->set_value("tick-time", frame_time); @@ -172,7 +198,8 @@ public: auto mixer_info = mixer_.info(); auto output_info = output_.info(); - info.add(L"video-mode", format_desc_.name); + info.add(L"video-mode", video_format_desc().name); + info.add(L"audio-channel-layout", audio_channel_layout().print()); info.add_child(L"stage", stage_info.get()); info.add_child(L"mixer", mixer_info.get()); info.add_child(L"output", output_info.get()); @@ -197,17 +224,23 @@ public: } }; -video_channel::video_channel(int index, const core::video_format_desc& format_desc, std::unique_ptr image_mixer) : impl_(new impl(index, format_desc, std::move(image_mixer))){} +video_channel::video_channel( + int index, + const core::video_format_desc& format_desc, + const core::audio_channel_layout& channel_layout, + std::unique_ptr image_mixer) : impl_(new impl(index, format_desc, channel_layout, std::move(image_mixer))){} video_channel::~video_channel(){} -const stage& video_channel::stage() const { return impl_->stage_;} -stage& video_channel::stage() { return impl_->stage_;} -const mixer& video_channel::mixer() const{ return impl_->mixer_;} -mixer& video_channel::mixer() { return impl_->mixer_;} -const output& video_channel::output() const { return impl_->output_;} -output& video_channel::output() { return impl_->output_;} -spl::shared_ptr video_channel::frame_factory() { return impl_->image_mixer_;} +const stage& video_channel::stage() const { return impl_->stage_;} +stage& video_channel::stage() { return impl_->stage_;} +const mixer& video_channel::mixer() const{ return impl_->mixer_;} +mixer& video_channel::mixer() { return impl_->mixer_;} +const output& video_channel::output() const { return impl_->output_;} +output& video_channel::output() { return impl_->output_;} +spl::shared_ptr video_channel::frame_factory() { return impl_->image_mixer_;} core::video_format_desc video_channel::video_format_desc() const{return impl_->video_format_desc();} void core::video_channel::video_format_desc(const core::video_format_desc& format_desc){impl_->video_format_desc(format_desc);} +core::audio_channel_layout video_channel::audio_channel_layout() const { return impl_->audio_channel_layout(); } +void core::video_channel::audio_channel_layout(const core::audio_channel_layout& channel_layout) { impl_->audio_channel_layout(channel_layout); } boost::property_tree::wptree video_channel::info() const{return impl_->info();} boost::property_tree::wptree video_channel::delay_info() const { return impl_->delay_info(); } int video_channel::index() const { return impl_->index(); } diff --git a/core/video_channel.h b/core/video_channel.h index 05bb79bda..0e7d6f994 100644 --- a/core/video_channel.h +++ b/core/video_channel.h @@ -43,7 +43,11 @@ public: // Constructors - explicit video_channel(int index, const video_format_desc& format_desc, std::unique_ptr image_mixer); + explicit video_channel( + int index, + const video_format_desc& format_desc, + const audio_channel_layout& channel_layout, + std::unique_ptr image_mixer); ~video_channel(); // Methods @@ -61,6 +65,8 @@ public: core::video_format_desc video_format_desc() const; void video_format_desc(const core::video_format_desc& format_desc); + core::audio_channel_layout audio_channel_layout() const; + void audio_channel_layout(const core::audio_channel_layout& channel_layout); spl::shared_ptr frame_factory(); diff --git a/core/video_format.cpp b/core/video_format.cpp index 61171c2af..792afb462 100644 --- a/core/video_format.cpp +++ b/core/video_format.cpp @@ -91,7 +91,6 @@ video_format_desc::video_format_desc( , size(width*height*4) , name(name) , audio_sample_rate(48000) - , audio_channels(2) , audio_cadence(audio_cadence) { } diff --git a/core/video_format.h b/core/video_format.h index 8e2a78117..90ff1f806 100644 --- a/core/video_format.h +++ b/core/video_format.h @@ -98,7 +98,6 @@ struct video_format_desc final std::wstring name; // name of output format int audio_sample_rate; - int audio_channels; std::vector audio_cadence; // rotating optimal number of samples per frame video_format_desc(video_format format, diff --git a/dependencies64/gtest/lib/gtest.lib b/dependencies64/gtest/lib/gtest.lib deleted file mode 100644 index 0956036c8..000000000 Binary files a/dependencies64/gtest/lib/gtest.lib and /dev/null differ diff --git a/dependencies64/gtest/lib/gtestd.lib b/dependencies64/gtest/lib/gtestd.lib deleted file mode 100644 index 17ff697c9..000000000 Binary files a/dependencies64/gtest/lib/gtestd.lib and /dev/null differ diff --git a/dependencies64/gtest/lib/win32/gtest.lib b/dependencies64/gtest/lib/win32/gtest.lib new file mode 100644 index 000000000..70e17bf2e Binary files /dev/null and b/dependencies64/gtest/lib/win32/gtest.lib differ diff --git a/dependencies64/gtest/lib/win32/gtestd.lib b/dependencies64/gtest/lib/win32/gtestd.lib new file mode 100644 index 000000000..e02284277 Binary files /dev/null and b/dependencies64/gtest/lib/win32/gtestd.lib differ diff --git a/modules/bluefish/consumer/bluefish_consumer.cpp b/modules/bluefish/consumer/bluefish_consumer.cpp index b5af351e9..77cc4d852 100644 --- a/modules/bluefish/consumer/bluefish_consumer.cpp +++ b/modules/bluefish/consumer/bluefish_consumer.cpp @@ -27,6 +27,7 @@ #include #include +#include #include #include @@ -58,34 +59,45 @@ namespace caspar { namespace bluefish { struct bluefish_consumer : boost::noncopyable { - spl::shared_ptr blue_; - const unsigned int device_index_; - const core::video_format_desc format_desc_; - const int channel_index_; - - const std::wstring model_name_; - - spl::shared_ptr graph_; - boost::timer frame_timer_; - boost::timer tick_timer_; - boost::timer sync_timer_; + spl::shared_ptr blue_; + const unsigned int device_index_; + const core::video_format_desc format_desc_; + const core::audio_channel_layout channel_layout_; + core::audio_channel_remapper channel_remapper_; + const int channel_index_; + + const std::wstring model_name_; + + spl::shared_ptr graph_; + boost::timer frame_timer_; + boost::timer tick_timer_; + boost::timer sync_timer_; - unsigned int vid_fmt_; + unsigned int vid_fmt_; - std::array reserved_frames_; - tbb::concurrent_bounded_queue frame_buffer_; - tbb::atomic presentation_delay_millis_; - core::const_frame previous_frame_ = core::const_frame::empty(); + std::array reserved_frames_; + tbb::concurrent_bounded_queue frame_buffer_; + tbb::atomic presentation_delay_millis_; + core::const_frame previous_frame_ = core::const_frame::empty(); - const bool embedded_audio_; - const bool key_only_; + const bool embedded_audio_; + const bool key_only_; - executor executor_; + executor executor_; public: - bluefish_consumer(const core::video_format_desc& format_desc, int device_index, bool embedded_audio, bool key_only, int channel_index) + bluefish_consumer( + const core::video_format_desc& format_desc, + const core::audio_channel_layout& in_channel_layout, + const core::audio_channel_layout& out_channel_layout, + int device_index, + bool embedded_audio, + bool key_only, + int channel_index) : blue_(create_blue(device_index)) , device_index_(device_index) - , format_desc_(format_desc) + , format_desc_(format_desc) + , channel_layout_(out_channel_layout) + , channel_remapper_(in_channel_layout, out_channel_layout) , channel_index_(channel_index) , model_name_(get_card_desc(*blue_)) , vid_fmt_(get_video_mode(*blue_, format_desc)) @@ -142,7 +154,19 @@ public: } else { - if(BLUE_FAIL(set_card_property(blue_, EMBEDEDDED_AUDIO_OUTPUT, blue_emb_audio_enable | blue_emb_audio_group1_enable))) + ULONG audio_value = + EMBEDDED_AUDIO_OUTPUT | blue_emb_audio_group1_enable; + + if (channel_layout_.num_channels > 4) + audio_value |= blue_emb_audio_group2_enable; + + if (channel_layout_.num_channels > 8) + audio_value |= blue_emb_audio_group3_enable; + + if (channel_layout_.num_channels > 12) + audio_value |= blue_emb_audio_group4_enable; + + if(BLUE_FAIL(set_card_property(blue_, EMBEDEDDED_AUDIO_OUTPUT, audio_value))) CASPAR_LOG(warning) << print() << TEXT(" Failed to enable embedded audio."); CASPAR_LOG(info) << print() << TEXT(" Enabled embedded-audio."); } @@ -245,12 +269,13 @@ public: // Send and display if(embedded_audio_) - { - auto frame_audio = core::audio_32_to_24(frame.audio_data()); + { + auto remapped_audio = channel_remapper_.mix_and_rearrange(frame.audio_data()); + auto frame_audio = core::audio_32_to_24(remapped_audio); encode_hanc(reinterpret_cast(reserved_frames_.front()->hanc_data()), frame_audio.data(), - static_cast(frame.audio_data().size()/format_desc_.audio_channels), - static_cast(format_desc_.audio_channels)); + static_cast(frame.audio_data().size()/channel_layout_.num_channels), + static_cast(channel_layout_.num_channels)); blue_->system_buffer_write_async(const_cast(reserved_frames_.front()->image_data()), static_cast(reserved_frames_.front()->image_size()), @@ -284,7 +309,16 @@ public: void encode_hanc(BLUE_UINT32* hanc_data, void* audio_data, int audio_samples, int audio_nchannels) { const auto sample_type = AUDIO_CHANNEL_24BIT | AUDIO_CHANNEL_LITTLEENDIAN; - const auto emb_audio_flag = blue_emb_audio_enable | blue_emb_audio_group1_enable; + auto emb_audio_flag = blue_emb_audio_enable | blue_emb_audio_group1_enable; + + if (audio_nchannels > 4) + emb_audio_flag |= blue_emb_audio_group2_enable; + + if (audio_nchannels > 8) + emb_audio_flag |= blue_emb_audio_group3_enable; + + if (audio_nchannels > 12) + emb_audio_flag |= blue_emb_audio_group4_enable; hanc_stream_info_struct hanc_stream_info; memset(&hanc_stream_info, 0, sizeof(hanc_stream_info)); @@ -325,30 +359,37 @@ struct bluefish_consumer_proxy : public core::frame_consumer std::vector audio_cadence_; core::video_format_desc format_desc_; + core::audio_channel_layout in_channel_layout_ = core::audio_channel_layout::invalid(); + core::audio_channel_layout out_channel_layout_; public: - bluefish_consumer_proxy(int device_index, bool embedded_audio, bool key_only) + bluefish_consumer_proxy(int device_index, bool embedded_audio, bool key_only, const core::audio_channel_layout& out_channel_layout) : device_index_(device_index) , embedded_audio_(embedded_audio) , key_only_(key_only) + , out_channel_layout_(out_channel_layout) { } // frame_consumer - void initialize(const core::video_format_desc& format_desc, int channel_index) override + void initialize(const core::video_format_desc& format_desc, const core::audio_channel_layout& channel_layout, int channel_index) override { - format_desc_ = format_desc; - audio_cadence_ = format_desc.audio_cadence; + format_desc_ = format_desc; + in_channel_layout_ = channel_layout; + audio_cadence_ = format_desc.audio_cadence; + + if (out_channel_layout_ == core::audio_channel_layout::invalid()) + out_channel_layout_ = in_channel_layout_; consumer_.reset(); - consumer_.reset(new bluefish_consumer(format_desc, device_index_, embedded_audio_, key_only_, channel_index)); + consumer_.reset(new bluefish_consumer(format_desc, in_channel_layout_, out_channel_layout_, device_index_, embedded_audio_, key_only_, channel_index)); } std::future send(core::const_frame frame) override { - CASPAR_VERIFY(audio_cadence_.front() * format_desc_.audio_channels == static_cast(frame.audio_data().size())); + CASPAR_VERIFY(audio_cadence_.front() * in_channel_layout_.num_channels == static_cast(frame.audio_data().size())); boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1); return consumer_->send(frame); } @@ -399,7 +440,7 @@ public: void describe_consumer(core::help_sink& sink, const core::help_repository& repo) { sink.short_description(L"Sends video on an SDI output using Bluefish video cards."); - sink.syntax(L"BLUEFISH {[device_index:int]|1} {[embedded_audio:EMBEDDED_AUDIO]} {[key_only:KEY_ONLY]}"); + sink.syntax(L"BLUEFISH {[device_index:int]|1} {[embedded_audio:EMBEDDED_AUDIO]} {[key_only:KEY_ONLY]} {CHANNEL_LAYOUT [channel_layout:string]}"); sink.para() ->text(L"Sends video on an SDI output using Bluefish video cards. Multiple video cards can be ") ->text(L"installed in the same machine and used at the same time, they will be addressed via ") @@ -409,6 +450,7 @@ void describe_consumer(core::help_sink& sink, const core::help_repository& repo) ->text(L"Specifying ")->code(L"key_only")->text(L" will extract only the alpha channel from the ") ->text(L"channel. This is useful when you have two SDI video cards, and neither has native support ") ->text(L"for separate fill/key output"); + sink.para()->text(L"Specify ")->code(L"channel_layout")->text(L" to output a different audio channel layout than the channel uses."); sink.para()->text(L"Examples:"); sink.example(L">> ADD 1 BLUEFISH", L"uses the default device_index of 1."); sink.example(L">> ADD 1 BLUEFISH 2", L"for device_index 2."); @@ -425,20 +467,46 @@ spl::shared_ptr create_consumer( const auto device_index = params.size() > 1 ? boost::lexical_cast(params.at(1)) : 1; - const auto embedded_audio = contains_param(L"EMBEDDED_AUDIO", params); - const auto key_only = contains_param(L"KEY_ONLY", params); + const auto embedded_audio = contains_param( L"EMBEDDED_AUDIO", params); + const auto key_only = contains_param( L"KEY_ONLY", params); + const auto channel_layout = get_param( L"CHANNEL_LAYOUT", params); - return spl::make_shared(device_index, embedded_audio, key_only); + auto layout = core::audio_channel_layout::invalid(); + + if (!channel_layout.empty()) + { + auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(channel_layout); + + if (!found_layout) + CASPAR_THROW_EXCEPTION(file_not_found() << msg_info(L"Channel layout " + channel_layout + L" not found")); + + layout = *found_layout; + } + + return spl::make_shared(device_index, embedded_audio, key_only, layout); } spl::shared_ptr create_preconfigured_consumer( const boost::property_tree::wptree& ptree, core::interaction_sink*) { - const auto device_index = ptree.get(L"device", 1); - const auto embedded_audio = ptree.get(L"embedded-audio", false); - const auto key_only = ptree.get(L"key-only", false); + const auto device_index = ptree.get( L"device", 1); + const auto embedded_audio = ptree.get( L"embedded-audio", false); + const auto key_only = ptree.get( L"key-only", false); + const auto channel_layout = ptree.get_optional( L"channel-layout"); + + auto layout = core::audio_channel_layout::invalid(); + + if (channel_layout) + { + auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(*channel_layout); + + if (!found_layout) + CASPAR_THROW_EXCEPTION(file_not_found() << msg_info(L"Channel layout " + *channel_layout + L" not found")); + + layout = *found_layout; + } - return spl::make_shared(device_index, embedded_audio, key_only); + return spl::make_shared(device_index, embedded_audio, key_only, layout); } }} \ No newline at end of file diff --git a/modules/decklink/consumer/decklink_consumer.cpp b/modules/decklink/consumer/decklink_consumer.cpp index 41b49d66d..418179586 100644 --- a/modules/decklink/consumer/decklink_consumer.cpp +++ b/modules/decklink/consumer/decklink_consumer.cpp @@ -28,6 +28,7 @@ #include "../decklink_api.h" #include +#include #include #include #include @@ -71,13 +72,14 @@ struct configuration default_latency }; - int device_index = 1; - int key_device_idx = 0; - bool embedded_audio = true; - keyer_t keyer = keyer_t::default_keyer; - latency_t latency = latency_t::default_latency; - bool key_only = false; - int base_buffer_depth = 3; + int device_index = 1; + int key_device_idx = 0; + bool embedded_audio = true; + keyer_t keyer = keyer_t::default_keyer; + latency_t latency = latency_t::default_latency; + bool key_only = false; + int base_buffer_depth = 3; + core::audio_channel_layout out_channel_layout = core::audio_channel_layout::invalid(); int buffer_depth() const { @@ -88,6 +90,31 @@ struct configuration { return key_device_idx == 0 ? device_index + 1 : key_device_idx; } + + core::audio_channel_layout get_adjusted_layout(const core::audio_channel_layout& in_layout) const + { + auto adjusted = out_channel_layout == core::audio_channel_layout::invalid() ? in_layout : out_channel_layout; + + if (adjusted.num_channels == 1) // Duplicate mono-signal into both left and right. + { + adjusted.num_channels = 2; + adjusted.channel_order.push_back(adjusted.channel_order.at(0)); // Usually FC -> FC FC + } + else if (adjusted.num_channels == 2) + { + adjusted.num_channels = 2; + } + else if (adjusted.num_channels <= 8) + { + adjusted.num_channels = 8; + } + else // Over 8 always pad to 16 or drop >16 + { + adjusted.num_channels = 16; + } + + return adjusted; + } }; void set_latency( @@ -317,6 +344,9 @@ struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLink const std::wstring model_name_ = get_model_name(decklink_); const core::video_format_desc format_desc_; + const core::audio_channel_layout in_channel_layout_; + const core::audio_channel_layout out_channel_layout_ = config_.get_adjusted_layout(in_channel_layout_); + core::audio_channel_remapper channel_remapper_ { in_channel_layout_, out_channel_layout_ }; const int buffer_size_ = config_.buffer_depth(); // Minimum buffer-size 3. long long video_scheduled_ = 0; @@ -337,10 +367,15 @@ struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLink std::unique_ptr key_context_; public: - decklink_consumer(const configuration& config, const core::video_format_desc& format_desc, int channel_index) + decklink_consumer( + const configuration& config, + const core::video_format_desc& format_desc, + const core::audio_channel_layout& in_channel_layout, + int channel_index) : channel_index_(channel_index) , config_(config) , format_desc_(format_desc) + , in_channel_layout_(in_channel_layout) { is_running_ = true; current_presentation_delay_ = 0; @@ -406,7 +441,7 @@ public: void enable_audio() { - if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped))) + if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, out_channel_layout_.num_channels, bmdAudioOutputStreamTimestamped))) CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable audio output.")); if(FAILED(output_->SetAudioCallback(this))) @@ -472,7 +507,7 @@ public: { graph_->set_tag("late-frame"); video_scheduled_ += format_desc_.duration; - audio_scheduled_ += dframe->audio_data().size() / format_desc_.audio_channels; + audio_scheduled_ += dframe->audio_data().size() / out_channel_layout_.num_channels; //++video_scheduled_; //audio_scheduled_ += format_desc_.audio_cadence[0]; //++audio_scheduled_; @@ -519,7 +554,7 @@ public: } else { - schedule_next_audio(core::audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()] * format_desc_.audio_channels, 0)); + schedule_next_audio(core::mutable_audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()] * out_channel_layout_.num_channels, 0)); } } else @@ -529,13 +564,13 @@ public: while(audio_frame_buffer_.try_pop(frame)) { send_completion_.try_completion(); - schedule_next_audio(frame.audio_data()); + schedule_next_audio(channel_remapper_.mix_and_rearrange(frame.audio_data())); } } UINT32 buffered; output_->GetBufferedAudioSampleFrameCount(&buffered); - graph_->set_value("buffered-audio", static_cast(buffered) / (format_desc_.audio_cadence[0] * format_desc_.audio_channels * 2)); + graph_->set_value("buffered-audio", static_cast(buffered) / (format_desc_.audio_cadence[0] * config_.buffer_depth())); } catch(...) { @@ -550,7 +585,7 @@ public: template void schedule_next_audio(const T& audio_data) { - auto sample_frame_count = static_cast(audio_data.size()/format_desc_.audio_channels); + auto sample_frame_count = static_cast(audio_data.size()/out_channel_layout_.num_channels); audio_container_.push_back(std::vector(audio_data.begin(), audio_data.end())); @@ -664,13 +699,13 @@ public: // frame_consumer - void initialize(const core::video_format_desc& format_desc, int channel_index) override + void initialize(const core::video_format_desc& format_desc, const core::audio_channel_layout& channel_layout, int channel_index) override { format_desc_ = format_desc; executor_.invoke([=] { consumer_.reset(); - consumer_.reset(new decklink_consumer(config_, format_desc, channel_index)); + consumer_.reset(new decklink_consumer(config_, format_desc, channel_layout, channel_index)); }); } @@ -737,7 +772,8 @@ void describe_consumer(core::help_sink& sink, const core::help_repository& repo) L"{[keyer:INTERNAL_KEY,EXTERNAL_KEY,EXTERNAL_SEPARATE_DEVICE_KEY]} " L"{[low_latency:LOW_LATENCY]} " L"{[embedded_audio:EMBEDDED_AUDIO]} " - L"{[key_only:KEY_ONLY]}"); + L"{[key_only:KEY_ONLY]} " + L"{CHANNEL_LAYOUT [channel_layout:string]}"); sink.para()->text(L"Sends video on an SDI output using Blackmagic Decklink video cards."); sink.definitions() ->item(L"device_index", L"The Blackmagic video card to use (See Blackmagic control panel for card order). Default is 1.") @@ -749,7 +785,8 @@ void describe_consumer(core::help_sink& sink, const core::help_repository& repo) ->item(L"key_only", L" will extract only the alpha channel from the " L"channel. This is useful when you have two SDI video cards, and neither has native support " - L"for separate fill/key output"); + L"for separate fill/key output") + ->item(L"channel_layout", L"If specified, overrides the audio channel layout used by the channel."); sink.para()->text(L"Examples:"); sink.example(L">> ADD 1 DECKLINK", L"for using the default device_index of 1."); sink.example(L">> ADD 1 DECKLINK 2", L"uses device_index 2."); @@ -788,6 +825,18 @@ spl::shared_ptr create_consumer( config.embedded_audio = contains_param(L"EMBEDDED_AUDIO", params); config.key_only = contains_param(L"KEY_ONLY", params); + auto channel_layout = get_param(L"CHANNEL_LAYOUT", params); + + if (!channel_layout.empty()) + { + auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(channel_layout); + + if (!found_layout) + CASPAR_THROW_EXCEPTION(file_not_found() << msg_info(L"Channel layout " + channel_layout + L" not found.")); + + config.out_channel_layout = *found_layout; + } + return spl::make_shared(config); } @@ -810,6 +859,18 @@ spl::shared_ptr create_preconfigured_consumer( else if(latency == L"normal") config.latency = configuration::latency_t::normal_latency; + auto channel_layout = ptree.get_optional(L"channel-layout"); + + if (channel_layout) + { + auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(*channel_layout); + + if (!found_layout) + CASPAR_THROW_EXCEPTION(file_not_found() << msg_info(L"Channel layout " + *channel_layout + L" not found.")); + + config.out_channel_layout = *found_layout; + } + config.key_only = ptree.get(L"key-only", config.key_only); config.device_index = ptree.get(L"device", config.device_index); config.key_device_idx = ptree.get(L"key-device", config.key_device_idx); diff --git a/modules/decklink/producer/decklink_producer.cpp b/modules/decklink/producer/decklink_producer.cpp index e74b1bfa3..3f6cc8b7d 100644 --- a/modules/decklink/producer/decklink_producer.cpp +++ b/modules/decklink/producer/decklink_producer.cpp @@ -37,6 +37,7 @@ #include #include +#include #include #include #include @@ -71,6 +72,18 @@ extern "C" #include namespace caspar { namespace decklink { + +core::audio_channel_layout get_adjusted_channel_layout(core::audio_channel_layout layout) +{ + if (layout.num_channels <= 2) + layout.num_channels = 2; + else if (layout.num_channels <= 8) + layout.num_channels = 8; + else + layout.num_channels = 16; + + return layout; +} class decklink_producer : boost::noncopyable, public IDeckLinkInputCallback { @@ -91,30 +104,33 @@ class decklink_producer : boost::noncopyable, public IDeckLinkInputCallback std::vector audio_cadence_ = out_format_desc_.audio_cadence; boost::circular_buffer sync_buffer_ { audio_cadence_.size() }; spl::shared_ptr frame_factory_; - ffmpeg::frame_muxer muxer_ { in_format_desc_.fps, frame_factory_, out_format_desc_, filter_ }; + core::audio_channel_layout channel_layout_; + ffmpeg::frame_muxer muxer_ { in_format_desc_.fps, frame_factory_, out_format_desc_, channel_layout_, filter_ }; core::constraints constraints_ { in_format_desc_.width, in_format_desc_.height }; tbb::concurrent_bounded_queue frame_buffer_; - std::exception_ptr exception_; + std::exception_ptr exception_; public: decklink_producer( const core::video_format_desc& in_format_desc, int device_index, const spl::shared_ptr& frame_factory, - const core::video_format_desc& out_format_desc, + const core::video_format_desc& out_format_desc, + const core::audio_channel_layout& channel_layout, const std::wstring& filter) : device_index_(device_index) , filter_(filter) , in_format_desc_(in_format_desc) , out_format_desc_(out_format_desc) , frame_factory_(frame_factory) - { + , channel_layout_(get_adjusted_channel_layout(channel_layout)) + { frame_buffer_.set_capacity(2); - graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f)); + graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f)); graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f)); graph_->set_color("frame-time", diagnostics::color(1.0f, 0.0f, 0.0f)); graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f)); @@ -130,8 +146,8 @@ public: << msg_info(print() + L" Could not enable video input.") << boost::errinfo_api_function("EnableVideoInput")); - if(FAILED(input_->EnableAudioInput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, static_cast(in_format_desc.audio_channels)))) - CASPAR_THROW_EXCEPTION(caspar_exception() + if(FAILED(input_->EnableAudioInput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, static_cast(channel_layout_.num_channels)))) + CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable audio input.") << boost::errinfo_api_function("EnableAudioInput")); @@ -219,7 +235,7 @@ public: auto audio_frame = ffmpeg::create_frame(); audio_frame->data[0] = reinterpret_cast(audio_bytes); - audio_frame->linesize[0] = audio->GetSampleFrameCount()*out_format_desc_.audio_channels*sizeof(int32_t); + audio_frame->linesize[0] = audio->GetSampleFrameCount() * channel_layout_.num_channels * sizeof(int32_t); audio_frame->nb_samples = audio->GetSampleFrameCount(); audio_frame->format = AV_SAMPLE_FMT_S32; @@ -301,18 +317,21 @@ class decklink_producer_proxy : public core::frame_producer_base const uint32_t length_; executor executor_; public: - explicit decklink_producer_proxy(const core::video_format_desc& in_format_desc, - const spl::shared_ptr& frame_factory, - const core::video_format_desc& out_format_desc, - int device_index, - const std::wstring& filter_str, uint32_t length) + explicit decklink_producer_proxy( + const core::video_format_desc& in_format_desc, + const spl::shared_ptr& frame_factory, + const core::video_format_desc& out_format_desc, + const core::audio_channel_layout& channel_layout, + int device_index, + const std::wstring& filter_str, + uint32_t length) : executor_(L"decklink_producer[" + boost::lexical_cast(device_index) + L"]") , length_(length) { executor_.invoke([=] { com_initialize(); - producer_.reset(new decklink_producer(in_format_desc, device_index, frame_factory, out_format_desc, filter_str)); + producer_.reset(new decklink_producer(in_format_desc, device_index, frame_factory, out_format_desc, channel_layout, filter_str)); }); } @@ -368,17 +387,19 @@ public: void describe_producer(core::help_sink& sink, const core::help_repository& repo) { sink.short_description(L"Allows video sources to be input from BlackMagic Design cards."); - sink.syntax(L"DECKLINK [device:int],DEVICE [device:int] {FILTER [filter:string]} {LENGTH [length:int]} {FORMAT [format:string]}"); + sink.syntax(L"DECKLINK [device:int],DEVICE [device:int] {FILTER [filter:string]} {LENGTH [length:int]} {FORMAT [format:string]} {CHANNEL_LAYOUT [channel_layout:string]}"); sink.para()->text(L"Allows video sources to be input from BlackMagic Design cards. Parameters:"); sink.definitions() ->item(L"device", L"The decklink device to stream the input from. See the Blackmagic control panel for the order of devices in your system.") ->item(L"filter", L"If specified, sets an FFmpeg video filter to use.") ->item(L"length", L"Optionally specify a limit on how many frames to produce.") - ->item(L"format", L"Specifies what video format to expect on the incoming SDI/HDMI signal. If not specified the video format of the channel is assumed."); + ->item(L"format", L"Specifies what video format to expect on the incoming SDI/HDMI signal. If not specified the video format of the channel is assumed.") + ->item(L"channel_layout", L"Specifies what audio channel layout to expect on the incoming SDI/HDMI signal. If not specified, stereo is assumed."); sink.para()->text(L"Examples:"); sink.example(L">> PLAY 1-10 DECKLINK DEVICE 2", L"Play using decklink device 2 expecting the video signal to have the same video format as the channel."); sink.example(L">> PLAY 1-10 DECKLINK DEVICE 2 FORMAT PAL FILTER yadif=1:-1", L"Play using decklink device 2 expecting the video signal to be in PAL and deinterlace it."); sink.example(L">> PLAY 1-10 DECKLINK DEVICE 2 LENGTH 1000", L"Play using decklink device 2 but only produce 1000 frames."); + sink.example(L">> PLAY 1-10 DECKLINK DEVICE 2 CHANNEL_LAYOUT smpte", L"Play using decklink device 2 and expect smpte surround sound."); } spl::shared_ptr create_producer(const core::frame_producer_dependencies& dependencies, const std::vector& params) @@ -396,8 +417,28 @@ spl::shared_ptr create_producer(const core::frame_producer if(in_format_desc.format == core::video_format::invalid) in_format_desc = dependencies.format_desc; + + auto channel_layout_spec = get_param(L"CHANNEL_LAYOUT", params); + auto channel_layout = *core::audio_channel_layout_repository::get_default()->get_layout(L"stereo"); + + if (!channel_layout_spec.empty()) + { + auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(channel_layout_spec); + + if (!found_layout) + CASPAR_THROW_EXCEPTION(file_not_found() << msg_info(L"Channel layout not found.")); + + channel_layout = *found_layout; + } - return create_destroy_proxy(spl::make_shared(in_format_desc, dependencies.frame_factory, dependencies.format_desc, device_index, filter_str, length)); + return create_destroy_proxy(spl::make_shared( + in_format_desc, + dependencies.frame_factory, + dependencies.format_desc, + channel_layout, + device_index, + filter_str, + length)); } }} diff --git a/modules/ffmpeg/CMakeLists.txt b/modules/ffmpeg/CMakeLists.txt index d9e526dd6..fe343425c 100644 --- a/modules/ffmpeg/CMakeLists.txt +++ b/modules/ffmpeg/CMakeLists.txt @@ -7,6 +7,7 @@ set(SOURCES producer/audio/audio_decoder.cpp + producer/filter/audio_filter.cpp producer/filter/filter.cpp producer/input/input.cpp @@ -21,6 +22,7 @@ set(SOURCES producer/ffmpeg_producer.cpp producer/tbb_avcodec.cpp + audio_channel_remapper.cpp ffmpeg.cpp ffmpeg_error.cpp StdAfx.cpp @@ -31,6 +33,7 @@ set(HEADERS producer/audio/audio_decoder.h + producer/filter/audio_filter.h producer/filter/filter.h producer/input/input.h diff --git a/modules/ffmpeg/audio_channel_remapper.cpp b/modules/ffmpeg/audio_channel_remapper.cpp new file mode 100644 index 000000000..f70f9c558 --- /dev/null +++ b/modules/ffmpeg/audio_channel_remapper.cpp @@ -0,0 +1,225 @@ +/* +* Copyright (c) 2011 Sveriges Television AB +* +* This file is part of CasparCG (www.casparcg.com). +* +* CasparCG is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* CasparCG is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with CasparCG. If not, see . +* +* Author: Helge Norberg, helge.norberg@svt.se +*/ + +#include "StdAfx.h" + +#include + +#include +#include + +#include "producer/filter/audio_filter.h" +#include "producer/util/util.h" + +#include + +#include +#include + +#include +#include + +#if defined(_MSC_VER) +#pragma warning (push) +#pragma warning (disable : 4244) +#endif +extern "C" +{ +#include +#include +} +#if defined(_MSC_VER) +#pragma warning (pop) +#endif + +namespace caspar { namespace core { + +std::wstring generate_pan_filter_str( + const audio_channel_layout& input, + const audio_channel_layout& output, + boost::optional mix_config) +{ + std::wstringstream result; + + result << L"pan=" << (boost::wformat(L"0x%|1$x|") % ffmpeg::create_channel_layout_bitmask(output.num_channels)).str(); + + if (!mix_config) + { + if (input.type == output.type && !input.channel_order.empty() && !input.channel_order.empty()) + { // No config needed because the layouts are of the same type. Generate mix config string. + std::vector mappings; + + for (auto& input_name : input.channel_order) + mappings.push_back(input_name + L"=" + input_name); + + mix_config = boost::join(mappings, L"|"); + } + else + { // Fallback to passthru c0=c0| c1=c1 | ... + for (int i = 0; i < output.num_channels; ++i) + result << L"|c" << i << L"=c" << i; + + CASPAR_LOG(debug) << "[audio_channel_remapper] Passthru " << input.num_channels << " channels into " << output.num_channels; + + return result.str(); + } + } + + CASPAR_LOG(debug) << L"[audio_channel_remapper] Using mix config: " << *mix_config; + + // Split on | to find the output sections + std::vector output_sections; + boost::split(output_sections, *mix_config, boost::is_any_of(L"|"), boost::algorithm::token_compress_off); + + for (auto& output_section : output_sections) + { + bool normalize_ratios = boost::contains(output_section, L"<"); + std::wstring mix_char = normalize_ratios ? L"<" : L"="; + + // Split on either = or < to get the output name and mix spec + std::vector output_and_spec; + boost::split(output_and_spec, output_section, boost::is_any_of(mix_char), boost::algorithm::token_compress_off); + auto& mix_spec = output_and_spec.at(1); + + // Replace each occurance of each channel name with c + for (int i = 0; i < input.channel_order.size(); ++i) + boost::replace_all(mix_spec, input.channel_order.at(i), L"c" + boost::lexical_cast(i)); + + auto output_name = boost::trim_copy(output_and_spec.at(0)); + auto actual_output_indexes = output.indexes_of(output_name); + + for (auto actual_output_index : actual_output_indexes) + { + result << L"|c" << actual_output_index << L" " << mix_char; + result << mix_spec; + } + } + + return result.str(); +} + +struct audio_channel_remapper::impl +{ + const audio_channel_layout input_layout_; + const audio_channel_layout output_layout_; + const bool the_same_layouts_ = input_layout_ == output_layout_; + std::unique_ptr filter_; + + impl( + audio_channel_layout input_layout, + audio_channel_layout output_layout, + spl::shared_ptr mix_repo) + : input_layout_(std::move(input_layout)) + , output_layout_(std::move(output_layout)) + { + if (input_layout_ == audio_channel_layout::invalid()) + CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info(L"Input audio channel layout is invalid")); + + if (output_layout_ == audio_channel_layout::invalid()) + CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info(L"Output audio channel layout is invalid")); + + CASPAR_LOG(debug) << L"[audio_channel_remapper] Input: " << input_layout_.print(); + CASPAR_LOG(debug) << L"[audio_channel_remapper] Output: " << output_layout_.print(); + + if (!the_same_layouts_) + { + auto mix_config = mix_repo->get_config(input_layout_.type, output_layout_.type); + auto pan_filter = u8(generate_pan_filter_str(input_layout_, output_layout_, mix_config)); + + CASPAR_LOG(debug) << "[audio_channel_remapper] Using audio filter: " << pan_filter; + filter_.reset(new ffmpeg::audio_filter( + boost::rational(1, 1), + 48000, + AV_SAMPLE_FMT_S32, + ffmpeg::create_channel_layout_bitmask(input_layout_.num_channels), + { 48000 }, + { AV_SAMPLE_FMT_S32 }, + { ffmpeg::create_channel_layout_bitmask(output_layout_.num_channels) }, + pan_filter)); + } + else + CASPAR_LOG(debug) << "[audio_channel_remapper] No remapping/mixing needed because the input and output layout is equal."; + } + + audio_buffer mix_and_rearrange(audio_buffer input) + { + CASPAR_ENSURE(input.size() % input_layout_.num_channels == 0); + + if (the_same_layouts_) + return std::move(input); + + auto num_samples = input.size() / input_layout_.num_channels; + auto expected_output_size = num_samples * output_layout_.num_channels; + auto input_frame = std::shared_ptr(av_frame_alloc(), [](AVFrame* p) + { + if (p) + av_frame_free(&p); + }); + + input_frame->channels = input_layout_.num_channels; + input_frame->channel_layout = ffmpeg::create_channel_layout_bitmask(input_layout_.num_channels); + input_frame->sample_rate = 48000; + input_frame->nb_samples = static_cast(num_samples); + input_frame->format = AV_SAMPLE_FMT_S32; + input_frame->pts = 0; + + av_samples_fill_arrays( + input_frame->extended_data, + input_frame->linesize, + reinterpret_cast(input.data()), + input_frame->channels, + input_frame->nb_samples, + static_cast(input_frame->format), + 16); + + filter_->push(input_frame); + + auto frames = filter_->poll_all(); + + CASPAR_ENSURE(frames.size() == 1); // Expect 1:1 from pan filter + + auto& frame = frames.front(); + auto output_size = frame->channels * frame->nb_samples; + + CASPAR_ENSURE(output_size == expected_output_size); + + return audio_buffer( + reinterpret_cast(frame->extended_data[0]), + output_size, + true, + std::move(frame)); + } +}; + +audio_channel_remapper::audio_channel_remapper( + audio_channel_layout input_layout, + audio_channel_layout output_layout, + spl::shared_ptr mix_repo) + : impl_(new impl(std::move(input_layout), std::move(output_layout), std::move(mix_repo))) +{ +} + +audio_buffer audio_channel_remapper::mix_and_rearrange(audio_buffer input) +{ + return impl_->mix_and_rearrange(std::move(input)); +} + +}} diff --git a/modules/ffmpeg/consumer/ffmpeg_consumer.cpp b/modules/ffmpeg/consumer/ffmpeg_consumer.cpp index 4a5aa2ecd..7d1c5f793 100644 --- a/modules/ffmpeg/consumer/ffmpeg_consumer.cpp +++ b/modules/ffmpeg/consumer/ffmpeg_consumer.cpp @@ -28,6 +28,7 @@ #include "../producer/tbb_avcodec.h" #include +#include #include #include #include @@ -261,9 +262,11 @@ typedef cache_aligned_vector byte_vector; struct ffmpeg_consumer : boost::noncopyable { const spl::shared_ptr graph_; - const std::string filename_; + const std::string filename_; + const std::string full_filename_ = u8(env::media_folder()) + filename_; const std::shared_ptr oc_ { avformat_alloc_context(), avformat_free_context }; - const core::video_format_desc format_desc_; + const core::video_format_desc format_desc_; + const core::audio_channel_layout channel_layout_; core::monitor::subject monitor_subject_; @@ -287,10 +290,16 @@ struct ffmpeg_consumer : boost::noncopyable executor executor_; public: - ffmpeg_consumer(const std::string& filename, const core::video_format_desc& format_desc, std::vector