From: ronag Date: Tue, 29 Nov 2011 02:07:16 +0000 (+0000) Subject: 2.0.2: Merged ntsc-audio-exp branch for proper/native NTSC audio support. X-Git-Tag: 2.0.2~85 X-Git-Url: https://git.sesse.net/?a=commitdiff_plain;h=00353365bb04c5343d487306ff284ad1951f6a3c;hp=ef56b8a7e0374545c6f1f02753771e35a364a8f0;p=casparcg 2.0.2: Merged ntsc-audio-exp branch for proper/native NTSC audio support. git-svn-id: https://casparcg.svn.sourceforge.net/svnroot/casparcg/server/branches/2.0.2@1709 362d55ac-95cf-4e76-9f9a-cbaa9c17b72d --- diff --git a/core/consumer/frame_consumer.cpp b/core/consumer/frame_consumer.cpp index 4f4b55693..35f651b8f 100644 --- a/core/consumer/frame_consumer.cpp +++ b/core/consumer/frame_consumer.cpp @@ -24,6 +24,10 @@ #include #include #include +#include +#include + +#include namespace caspar { namespace core { @@ -59,4 +63,67 @@ safe_ptr create_consumer(const std::vector& return consumer; } +// This class is used to guarantee that audio cadence is correct. This is important for NTSC audio. +class cadence_guard : public frame_consumer +{ + safe_ptr consumer_; + std::vector audio_cadence_; + boost::circular_buffer sync_buffer_; + bool synced_; +public: + cadence_guard(safe_ptr&& consumer) + : consumer_(std::move(consumer)) + { + } + + virtual void initialize(const video_format_desc& format_desc, int channel_index, int sub_index) override + { + audio_cadence_ = format_desc.audio_cadence; + sync_buffer_ = boost::circular_buffer(format_desc.audio_cadence.size()); + consumer_->initialize(format_desc, channel_index, sub_index); + } + + virtual bool send(const safe_ptr& frame) override + { + sync_buffer_.push_back(static_cast(frame->audio_data().size())); + + if(!boost::range::equal(sync_buffer_, audio_cadence_)) + { + synced_ = false; + CASPAR_LOG(trace) << L"[cadence_guard] Audio cadence unsynced. Skipping frame."; + return true; + } + else if(!synced_) + { + synced_ = true; + boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1); + return true; + } + + boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1); + + return consumer_->send(frame); + } + + virtual std::wstring print() const override + { + return consumer_->print(); + } + + virtual bool has_synchronization_clock() const override + { + return consumer_->has_synchronization_clock(); + } + + virtual size_t buffer_depth() const override + { + return consumer_->buffer_depth(); + } +}; + +safe_ptr create_consumer_cadence_guard(safe_ptr&& consumer) +{ + return make_safe(std::move(consumer)); +} + }} \ No newline at end of file diff --git a/core/consumer/frame_consumer.h b/core/consumer/frame_consumer.h index 090c2ff21..e4aafca2b 100644 --- a/core/consumer/frame_consumer.h +++ b/core/consumer/frame_consumer.h @@ -19,8 +19,6 @@ */ #pragma once -#include "../video_format.h" - #include #include @@ -59,6 +57,8 @@ struct frame_consumer : boost::noncopyable } }; +safe_ptr create_consumer_cadence_guard(safe_ptr&& consumer); + typedef std::function(const std::vector&)> consumer_factory_t; void register_consumer_factory(const consumer_factory_t& factory); diff --git a/core/consumer/output.cpp b/core/consumer/output.cpp index 4da6f05bd..ff16c280e 100644 --- a/core/consumer/output.cpp +++ b/core/consumer/output.cpp @@ -40,7 +40,7 @@ #include namespace caspar { namespace core { - + struct output::implementation { typedef std::pair, safe_ptr> fill_and_key; @@ -79,6 +79,7 @@ public: consumers_.erase(index); }); + consumer = create_consumer_cadence_guard(std::move(consumer)); consumer->initialize(format_desc_, channel_index_, index); executor_.invoke([&] diff --git a/core/mixer/audio/audio_mixer.cpp b/core/mixer/audio/audio_mixer.cpp index 627826b05..60e7dd812 100644 --- a/core/mixer/audio/audio_mixer.cpp +++ b/core/mixer/audio/audio_mixer.cpp @@ -24,9 +24,16 @@ #include #include -#include +#include + +#include +#include + +#include #include +#include #include +#include namespace caspar { namespace core { @@ -35,16 +42,38 @@ struct audio_item const void* tag; frame_transform transform; audio_buffer audio_data; + + audio_item() + { + } + + audio_item(audio_item&& other) + : tag(std::move(other.tag)) + , transform(std::move(other.transform)) + , audio_data(std::move(other.audio_data)) + { + } }; + +typedef std::vector> audio_buffer_ps; -struct audio_mixer::implementation +struct audio_stream { - std::stack transform_stack_; - std::map prev_frame_transforms_; - std::vector items_; + frame_transform transform; + audio_buffer_ps audio_data; +}; +struct audio_mixer::implementation +{ + std::stack transform_stack_; + std::map audio_streams_; + std::vector items_; + std::vector audio_cadence_; + video_format_desc format_desc_; + public: implementation() + : format_desc_(video_format_desc::get(video_format::invalid)) { transform_stack_.push(core::frame_transform()); } @@ -63,8 +92,8 @@ public: item.tag = frame.tag(); item.transform = transform_stack_.top(); item.audio_data = std::move(frame.audio_data()); // Note: We don't need to care about upper/lower since audio_data is removed/moved from the last field. - - items_.push_back(item); + + items_.push_back(std::move(item)); } void begin(const core::frame_transform& transform) @@ -79,74 +108,81 @@ public: audio_buffer mix(const video_format_desc& format_desc) { - //CASPAR_ASSERT(format_desc.audio_channels == 2); - //CASPAR_ASSERT(format_desc.audio_samples_per_frame % 4 == 0); - - // NOTE: auto data should be larger than format_desc_.audio_samples_per_frame to allow sse to read/write beyond size. - - auto intermediate = std::vector>(format_desc.audio_samples_per_frame+128, 0.0f); - auto result = audio_buffer(format_desc.audio_samples_per_frame+128); - auto result_128 = reinterpret_cast<__m128i*>(result.data()); + if(format_desc_ != format_desc) + { + audio_streams_.clear(); + audio_cadence_ = format_desc.audio_cadence; + format_desc_ = format_desc; + } + + std::map next_audio_streams_; - std::map next_frame_transforms; - BOOST_FOREACH(auto& item, items_) - { - auto next = item.transform; - auto prev = next; + { + audio_buffer_ps next_audio; - const auto it = prev_frame_transforms_.find(item.tag); - if(it != prev_frame_transforms_.end()) - prev = it->second; - - next_frame_transforms[item.tag] = next; // Store all active tags, inactive tags will be removed at the end. - - if(prev.volume < 0.001 && next.volume < 0.001) - continue; + auto next_transform = item.transform; + auto prev_transform = next_transform; - if(item.audio_data.size() != format_desc.audio_samples_per_frame) - continue; + const auto it = audio_streams_.find(item.tag); + if(it != audio_streams_.end()) + { + prev_transform = it->second.transform; + next_audio = std::move(it->second.audio_data); + } - const float prev_volume = static_cast(prev.volume); - const float next_volume = static_cast(next.volume); + if(prev_transform.volume < 0.001 && next_transform.volume < 0.001) + continue; + + const float prev_volume = static_cast(prev_transform.volume); + const float next_volume = static_cast(next_transform.volume); - auto alpha = (next_volume-prev_volume)/static_cast(format_desc.audio_samples_per_frame/format_desc.audio_channels); - auto alpha_ps = _mm_set_ps1(alpha*2.0f); - auto volume_ps = _mm_setr_ps(prev_volume, prev_volume, prev_volume+alpha, prev_volume+alpha); - - if(&item != &items_.back()) + auto alpha = (next_volume-prev_volume)/static_cast(item.audio_data.size()/format_desc.audio_channels); + + audio_buffer_ps result; + result.reserve(item.audio_data.size()); + for(size_t n = 0; n < item.audio_data.size()/2; ++n) { - for(size_t n = 0; n < format_desc.audio_samples_per_frame/4; ++n) - { - auto sample_ps = _mm_cvtepi32_ps(_mm_load_si128(reinterpret_cast<__m128i*>(&item.audio_data[n*4]))); - auto res_sample_ps = _mm_load_ps(&intermediate[n*4]); - sample_ps = _mm_mul_ps(sample_ps, volume_ps); - res_sample_ps = _mm_add_ps(sample_ps, res_sample_ps); - - volume_ps = _mm_add_ps(volume_ps, alpha_ps); - - _mm_store_ps(&intermediate[n*4], res_sample_ps); - } + result.push_back(item.audio_data[n*2+0] * (prev_volume + n * alpha)); + result.push_back(item.audio_data[n*2+1] * (prev_volume + n * alpha)); } - else + + boost::range::push_back(next_audio, std::move(result)); + + next_audio_streams_[item.tag].transform = std::move(next_transform); // Store all active tags, inactive tags will be removed at the end. + next_audio_streams_[item.tag].audio_data = std::move(next_audio); + } + items_.clear(); + + audio_streams_ = std::move(next_audio_streams_); + next_audio_streams_.clear(); + + if(audio_streams_.empty()) + audio_streams_[nullptr].audio_data = audio_buffer_ps(audio_cadence_.front(), 0.0f); + + std::vector result_ps(audio_cadence_.front(), 0.0f); + BOOST_FOREACH(auto& stream, audio_streams_) + { + auto& audio_data = stream.second.audio_data; + + if(audio_data.size() < audio_cadence_.front()) { - for(size_t n = 0; n < format_desc.audio_samples_per_frame/4; ++n) - { - auto sample_ps = _mm_cvtepi32_ps(_mm_load_si128(reinterpret_cast<__m128i*>(&item.audio_data[n*4]))); - auto res_sample_ps = _mm_load_ps(&intermediate[n*4]); - sample_ps = _mm_mul_ps(sample_ps, volume_ps); - res_sample_ps = _mm_add_ps(sample_ps, res_sample_ps); - - _mm_stream_si128(result_128++, _mm_cvtps_epi32(res_sample_ps)); - } + CASPAR_LOG(warning) << "[audio_mixer] Incorrect frame audio cadence, prepending silence: " << audio_cadence_.front()-audio_data.size(); + boost::range::push_back(audio_data, audio_buffer_ps(audio_cadence_.front()-audio_data.size(), 0.0f)); } - } - items_.clear(); - prev_frame_transforms_ = std::move(next_frame_transforms); + for(size_t n = 0; n < audio_cadence_.front(); ++n) + result_ps[n] += audio_data[n]; - result.resize(format_desc.audio_samples_per_frame); - return std::move(result); + audio_data.erase(std::begin(audio_data), std::begin(audio_data) + audio_cadence_.front()); + } + + boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1); + + audio_buffer result; + result.reserve(result_ps.size()); + boost::range::transform(result_ps, std::back_inserter(result), [](float sample){return static_cast(sample);}); + return result; } }; diff --git a/core/producer/frame_producer.cpp b/core/producer/frame_producer.cpp index 772d5647b..ef93ee8cd 100644 --- a/core/producer/frame_producer.cpp +++ b/core/producer/frame_producer.cpp @@ -103,7 +103,7 @@ public: virtual int64_t file_frame_number() const override {return (*producer_)->file_frame_number();} }; -safe_ptr create_destroy_proxy(safe_ptr&& producer) +safe_ptr create_producer_destroy_proxy(safe_ptr&& producer) { return make_safe(std::move(producer)); } diff --git a/core/producer/frame_producer.h b/core/producer/frame_producer.h index d07ce3258..ac7ba1ea6 100644 --- a/core/producer/frame_producer.h +++ b/core/producer/frame_producer.h @@ -82,7 +82,7 @@ typedef std::function(const safe_ptr create_producer(const safe_ptr&, const std::vector& params); safe_ptr create_producer(const safe_ptr&, const std::wstring& params); -safe_ptr create_destroy_proxy(safe_ptr&& producer); +safe_ptr create_producer_destroy_proxy(safe_ptr&& producer); template typename std::decay::type get_param(const std::wstring& name, const std::vector& params, T fail_value) diff --git a/core/video_format.cpp b/core/video_format.cpp index cb160fff6..ee33c19c6 100644 --- a/core/video_format.cpp +++ b/core/video_format.cpp @@ -23,8 +23,9 @@ #include "video_format.h" #include +#include -#define DEFINE_VIDEOFORMATDESC(fmt, w, h, sw, sh, m, scale, duration, name) \ +#define DEFINE_VIDEOFORMATDESC(fmt, w, h, sw, sh, m, scale, duration, audio_samples, name) \ { \ (fmt), \ (w), \ @@ -40,30 +41,31 @@ (name),\ (48000),\ (2),\ - (static_cast(48000.0/((double)scale/(double)duration)+0.99)*2)\ + (audio_samples)\ } + namespace caspar { namespace core { const video_format_desc format_descs[video_format::count] = { - DEFINE_VIDEOFORMATDESC(video_format::pal ,720, 576, 1024, 576, field_mode::upper, 25, 1, L"PAL"), - DEFINE_VIDEOFORMATDESC(video_format::ntsc ,720, 486, 720, 534, field_mode::lower, 30000, 1001, L"NTSC"), - DEFINE_VIDEOFORMATDESC(video_format::x576p2500 ,720, 576, 720, 576, field_mode::progressive, 25, 1, L"576p2500"), - DEFINE_VIDEOFORMATDESC(video_format::x720p2500 ,1280, 720, 1280, 720, field_mode::progressive, 25, 1, L"720p2500"), - DEFINE_VIDEOFORMATDESC(video_format::x720p5000 ,1280, 720, 1280, 720, field_mode::progressive, 50, 1, L"720p5000"), - DEFINE_VIDEOFORMATDESC(video_format::x720p5994 ,1280, 720, 1280, 720, field_mode::progressive, 60000, 1001, L"720p5994"), - DEFINE_VIDEOFORMATDESC(video_format::x720p6000 ,1280, 720, 1280, 720, field_mode::progressive, 60, 1, L"720p6000"), - DEFINE_VIDEOFORMATDESC(video_format::x1080p2397 ,1920, 1080, 1920, 1080, field_mode::progressive, 24000, 1001, L"1080p2398"), - DEFINE_VIDEOFORMATDESC(video_format::x1080p2400 ,1920, 1080, 1920, 1080, field_mode::progressive, 24, 1, L"1080p2400"), - DEFINE_VIDEOFORMATDESC(video_format::x1080i5000 ,1920, 1080, 1920, 1080, field_mode::upper, 25, 1, L"1080i5000"), - DEFINE_VIDEOFORMATDESC(video_format::x1080i5994 ,1920, 1080, 1920, 1080, field_mode::upper, 30000, 1001, L"1080i5994"), - DEFINE_VIDEOFORMATDESC(video_format::x1080i6000 ,1920, 1080, 1920, 1080, field_mode::upper, 30, 1, L"1080i6000"), - DEFINE_VIDEOFORMATDESC(video_format::x1080p2500 ,1920, 1080, 1920, 1080, field_mode::progressive, 25, 1, L"1080p2500"), - DEFINE_VIDEOFORMATDESC(video_format::x1080p2997 ,1920, 1080, 1920, 1080, field_mode::progressive, 30000, 1001, L"1080p2997"), - DEFINE_VIDEOFORMATDESC(video_format::x1080p3000 ,1920, 1080, 1920, 1080, field_mode::progressive, 30, 1, L"1080p3000"), - DEFINE_VIDEOFORMATDESC(video_format::x1080p5000 ,1920, 1080, 1920, 1080, field_mode::progressive, 50, 1, L"1080p5000"), - DEFINE_VIDEOFORMATDESC(video_format::invalid ,0, 0, 0, 0, field_mode::progressive, 1, 1, L"invalid") + DEFINE_VIDEOFORMATDESC(video_format::pal ,720, 576, 1024, 576, field_mode::upper, 25, 1, boost::assign::list_of(3840), L"PAL"), + DEFINE_VIDEOFORMATDESC(video_format::ntsc ,720, 486, 720, 534, field_mode::lower, 30000, 1001, boost::assign::list_of(3204)(3202)(3204)(3202)(3204), L"NTSC"), + DEFINE_VIDEOFORMATDESC(video_format::x576p2500 ,720, 576, 720, 576, field_mode::progressive, 25, 1, boost::assign::list_of(3840), L"576p2500"), + DEFINE_VIDEOFORMATDESC(video_format::x720p2500 ,1280, 720, 1280, 720, field_mode::progressive, 25, 1, boost::assign::list_of(3840), L"720p2500"), + DEFINE_VIDEOFORMATDESC(video_format::x720p5000 ,1280, 720, 1280, 720, field_mode::progressive, 50, 1, boost::assign::list_of(1920), L"720p5000"), + DEFINE_VIDEOFORMATDESC(video_format::x720p5994 ,1280, 720, 1280, 720, field_mode::progressive, 60000, 1001, boost::assign::list_of(1602)(1601)(1602)(1601)(1602), L"720p5994"), + DEFINE_VIDEOFORMATDESC(video_format::x720p6000 ,1280, 720, 1280, 720, field_mode::progressive, 60, 1, boost::assign::list_of(1600), L"720p6000"), + DEFINE_VIDEOFORMATDESC(video_format::x1080p2397 ,1920, 1080, 1920, 1080, field_mode::progressive, 24000, 1001, boost::assign::list_of(4004), L"1080p2398"), + DEFINE_VIDEOFORMATDESC(video_format::x1080p2400 ,1920, 1080, 1920, 1080, field_mode::progressive, 24, 1, boost::assign::list_of(4000), L"1080p2400"), + DEFINE_VIDEOFORMATDESC(video_format::x1080i5000 ,1920, 1080, 1920, 1080, field_mode::upper, 25, 1, boost::assign::list_of(3840), L"1080i5000"), + DEFINE_VIDEOFORMATDESC(video_format::x1080i5994 ,1920, 1080, 1920, 1080, field_mode::upper, 30000, 1001, boost::assign::list_of(3204)(3202)(3204)(3202)(3204), L"1080i5994"), + DEFINE_VIDEOFORMATDESC(video_format::x1080i6000 ,1920, 1080, 1920, 1080, field_mode::upper, 30, 1, boost::assign::list_of(3200), L"1080i6000"), + DEFINE_VIDEOFORMATDESC(video_format::x1080p2500 ,1920, 1080, 1920, 1080, field_mode::progressive, 25, 1, boost::assign::list_of(3840), L"1080p2500"), + DEFINE_VIDEOFORMATDESC(video_format::x1080p2997 ,1920, 1080, 1920, 1080, field_mode::progressive, 30000, 1001, boost::assign::list_of(3204)(3202)(3204)(3202)(3204), L"1080p2997"), + DEFINE_VIDEOFORMATDESC(video_format::x1080p3000 ,1920, 1080, 1920, 1080, field_mode::progressive, 30, 1, boost::assign::list_of(3200), L"1080p3000"), + DEFINE_VIDEOFORMATDESC(video_format::x1080p5000 ,1920, 1080, 1920, 1080, field_mode::progressive, 50, 1, boost::assign::list_of(1920), L"1080p5000"), + DEFINE_VIDEOFORMATDESC(video_format::invalid ,0, 0, 0, 0, field_mode::progressive, 1, 1, boost::assign::list_of(1), L"invalid") }; const video_format_desc& video_format_desc::get(video_format::type format) diff --git a/core/video_format.h b/core/video_format.h index 58634a827..c809a2b65 100644 --- a/core/video_format.h +++ b/core/video_format.h @@ -20,10 +20,9 @@ #pragma once +#include #include -#include - namespace caspar { namespace core { struct video_format @@ -95,21 +94,21 @@ struct video_format_desc size_t audio_sample_rate; size_t audio_channels; - size_t audio_samples_per_frame; + std::vector audio_cadence; static const video_format_desc& get(video_format::type format); static const video_format_desc& get(const std::wstring& name); -}; - -inline bool operator==(const video_format_desc& rhs, const video_format_desc& lhs) -{ - return rhs.format == lhs.format; -} + + bool operator==(const video_format_desc& lhs) + { + return format == lhs.format; + } -inline bool operator!=(const video_format_desc& rhs, const video_format_desc& lhs) -{ - return !(rhs == lhs); -} + bool operator!=(const video_format_desc& lhs) + { + return !(*this == lhs); + } +}; inline std::wostream& operator<<(std::wostream& out, const video_format_desc& format_desc) { diff --git a/modules/bluefish/consumer/bluefish_consumer.cpp b/modules/bluefish/consumer/bluefish_consumer.cpp index d95c956bc..1312a0eff 100644 --- a/modules/bluefish/consumer/bluefish_consumer.cpp +++ b/modules/bluefish/consumer/bluefish_consumer.cpp @@ -24,6 +24,7 @@ #include "../util/blue_velvet.h" #include "../util/memory.h" +#include #include #include @@ -39,6 +40,7 @@ #include #include +#include #include #include @@ -67,6 +69,8 @@ struct bluefish_consumer : boost::noncopyable const bool embedded_audio_; const bool key_only_; + + uint64_t frame_number_; executor executor_; public: @@ -80,6 +84,7 @@ public: , vid_fmt_(get_video_mode(*blue_, format_desc)) , embedded_audio_(embedded_audio) , key_only_(key_only) + , frame_number_(0) , executor_(print()) { executor_.set_capacity(1); @@ -146,9 +151,9 @@ public: if(blue_->GetHDCardType(device_index_) != CRD_HD_INVALID) blue_->Set_DownConverterSignalType(vid_fmt_ == VID_FMT_PAL ? SD_SDI : HD_SDI); - unsigned long engine_mode = VIDEO_ENGINE_FRAMESTORE; - if(BLUE_FAIL(blue_->set_video_engine(engine_mode))) - BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Failed to set video engine.")); + //unsigned long engine_mode = VIDEO_ENGINE_FRAMESTORE; + //if(BLUE_FAIL(blue_->set_video_engine(engine_mode))) + // BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Failed to set video engine.")); enable_video_output(); @@ -193,61 +198,10 @@ public: { try { - frame_timer_.restart(); - - // Copy to local buffers - - if(!frame->image_data().empty()) - { - if(key_only_) - fast_memshfl(reserved_frames_.front()->image_data(), frame->image_data().begin(), frame->image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303); - else - fast_memcpy(reserved_frames_.front()->image_data(), frame->image_data().begin(), frame->image_data().size()); - } - else - fast_memclr(reserved_frames_.front()->image_data(), reserved_frames_.front()->image_size()); - - // Sync - - sync_timer_.restart(); - unsigned long n_field = 0; - blue_->wait_output_video_synch(UPD_FMT_FRAME, n_field); - graph_->update_value("sync-time", sync_timer_.elapsed()*format_desc_.fps*0.5); + frame_timer_.restart(); - // Send and display + display_frame(frame); - if(embedded_audio_) - { - auto frame_audio16 = core::audio_32_to_16_sse(frame->audio_data()); - - encode_hanc(reinterpret_cast(reserved_frames_.front()->hanc_data()), frame_audio16.data(), frame->audio_data().size(), format_desc_.audio_channels); - - blue_->system_buffer_write_async(const_cast(reserved_frames_.front()->image_data()), - reserved_frames_.front()->image_size(), - nullptr, - BlueImage_HANC_DMABuffer(reserved_frames_.front()->id(), BLUE_DATA_IMAGE)); - - blue_->system_buffer_write_async(reserved_frames_.front()->hanc_data(), - reserved_frames_.front()->hanc_size(), - nullptr, - BlueImage_HANC_DMABuffer(reserved_frames_.front()->id(), BLUE_DATA_HANC)); - - if(BLUE_FAIL(blue_->render_buffer_update(BlueBuffer_Image_HANC(reserved_frames_.front()->id())))) - CASPAR_LOG(warning) << print() << TEXT(" render_buffer_update failed."); - } - else - { - blue_->system_buffer_write_async(const_cast(reserved_frames_.front()->image_data()), - reserved_frames_.front()->image_size(), - nullptr, - BlueImage_DMABuffer(reserved_frames_.front()->id(), BLUE_DATA_IMAGE)); - - if(BLUE_FAIL(blue_->render_buffer_update(BlueBuffer_Image(reserved_frames_.front()->id())))) - CASPAR_LOG(warning) << print() << TEXT(" render_buffer_update failed."); - } - - std::rotate(reserved_frames_.begin(), reserved_frames_.begin() + 1, reserved_frames_.end()); - graph_->update_value("frame-time", static_cast(frame_timer_.elapsed()*format_desc_.fps*0.5)); graph_->update_value("tick-time", static_cast(tick_timer_.elapsed()*format_desc_.fps*0.5)); @@ -260,9 +214,80 @@ public: }); } + void display_frame(const safe_ptr& frame) + { + // Copy to local buffers + + if(!frame->image_data().empty()) + { + if(key_only_) + fast_memshfl(reserved_frames_.front()->image_data(), frame->image_data().begin(), frame->image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303); + else + fast_memcpy(reserved_frames_.front()->image_data(), frame->image_data().begin(), frame->image_data().size()); + } + else + fast_memclr(reserved_frames_.front()->image_data(), reserved_frames_.front()->image_size()); + + // Sync + + sync_timer_.restart(); + unsigned long n_field = 0; + blue_->wait_output_video_synch(UPD_FMT_FRAME, n_field); + graph_->update_value("sync-time", sync_timer_.elapsed()*format_desc_.fps*0.5); + + // Send and display + + if(embedded_audio_) + { + auto frame_audio = core::audio_32_to_16_sse(frame->audio_data()); + + if(format_desc_.format == core::video_format::ntsc) + { + size_t cadence[] = {1602,1601,1602,1601,1602}; + CASPAR_VERIFY(cadence[(frame_number_++) % 5] == (frame->audio_data().size()/format_desc_.audio_channels)); + } + + // 24 bit audio causes access violation in encode_hanc. + //auto frame_audio24 = core::audio_32_to_24(frame->audio_data()); + + //static_assert(sizeof(frame_audio24.front()) == 1, ""); + //static_assert(sizeof(frame->audio_data().front()) == 4, ""); + //CASPAR_VERIFY(frame_audio24.size() / 3 == static_cast(frame->audio_data().size())); + + // audio cadence is guaranteed by input, see output.cpp and frame_producer.cpp - cadence_guard. + + encode_hanc(reinterpret_cast(reserved_frames_.front()->hanc_data()), frame_audio.data(), frame->audio_data().size(), format_desc_.audio_channels); + + blue_->system_buffer_write_async(const_cast(reserved_frames_.front()->image_data()), + reserved_frames_.front()->image_size(), + nullptr, + BlueImage_HANC_DMABuffer(reserved_frames_.front()->id(), BLUE_DATA_IMAGE)); + + blue_->system_buffer_write_async(reserved_frames_.front()->hanc_data(), + reserved_frames_.front()->hanc_size(), + nullptr, + BlueImage_HANC_DMABuffer(reserved_frames_.front()->id(), BLUE_DATA_HANC)); + + if(BLUE_FAIL(blue_->render_buffer_update(BlueBuffer_Image_HANC(reserved_frames_.front()->id())))) + CASPAR_LOG(warning) << print() << TEXT(" render_buffer_update failed."); + } + else + { + blue_->system_buffer_write_async(const_cast(reserved_frames_.front()->image_data()), + reserved_frames_.front()->image_size(), + nullptr, + BlueImage_DMABuffer(reserved_frames_.front()->id(), BLUE_DATA_IMAGE)); + + if(BLUE_FAIL(blue_->render_buffer_update(BlueBuffer_Image(reserved_frames_.front()->id())))) + CASPAR_LOG(warning) << print() << TEXT(" render_buffer_update failed."); + } + + std::rotate(reserved_frames_.begin(), reserved_frames_.begin() + 1, reserved_frames_.end()); + } + void encode_hanc(BLUE_UINT32* hanc_data, void* audio_data, size_t audio_samples, size_t audio_nchannels) { - const auto sample_type = AUDIO_CHANNEL_16BIT | AUDIO_CHANNEL_LITTLEENDIAN; + const auto sample_type = AUDIO_CHANNEL_16BIT | AUDIO_CHANNEL_LITTLEENDIAN; // AUDIO_CHANNEL_24BIT | AUDIO_CHANNEL_LITTLEENDIAN; const auto emb_audio_flag = blue_emb_audio_enable | blue_emb_audio_group1_enable; hanc_stream_info_struct hanc_stream_info; @@ -294,6 +319,7 @@ struct bluefish_consumer_proxy : public core::frame_consumer const size_t device_index_; const bool embedded_audio_; const bool key_only_; + std::vector audio_cadence_; public: bluefish_consumer_proxy(size_t device_index, bool embedded_audio, bool key_only) @@ -318,11 +344,15 @@ public: virtual void initialize(const core::video_format_desc& format_desc, int channel_index, int sub_index) override { consumer_.reset(new bluefish_consumer(format_desc, device_index_, embedded_audio_, key_only_, channel_index, sub_index)); + audio_cadence_ = format_desc.audio_cadence; CASPAR_LOG(info) << print() << L" Successfully Initialized."; } virtual bool send(const safe_ptr& frame) override { + CASPAR_VERIFY(audio_cadence_.front() == static_cast(frame->audio_data().size())); + boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1); + consumer_->send(frame); return true; } diff --git a/modules/decklink/consumer/decklink_consumer.cpp b/modules/decklink/consumer/decklink_consumer.cpp index 4861946b3..85a191d07 100644 --- a/modules/decklink/consumer/decklink_consumer.cpp +++ b/modules/decklink/consumer/decklink_consumer.cpp @@ -59,7 +59,7 @@ struct configuration : device_index(1) , embedded_audio(false) , internal_key(false) - , low_latency(false) + , low_latency(true) , key_only(false) , base_buffer_depth(3) , buffer_depth(base_buffer_depth + (low_latency ? 0 : 1) + (embedded_audio ? 1 : 0)){} @@ -81,6 +81,11 @@ public: { ref_count_ = 0; } + + const boost::iterator_range audio_data() + { + return frame_->audio_data(); + } STDMETHOD (QueryInterface(REFIID, LPVOID*)) {return E_NOINTERFACE;} STDMETHOD_(ULONG, AddRef()) @@ -132,9 +137,9 @@ public: struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable { - const configuration config_; const int channel_index_; const int sub_index_; + const configuration config_; CComPtr decklink_; CComQIPtr output_; @@ -150,7 +155,7 @@ struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLink const core::video_format_desc format_desc_; const size_t buffer_size_; - long long frames_scheduled_; + long long video_scheduled_; long long audio_scheduled_; size_t preroll_count_; @@ -165,9 +170,9 @@ struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLink public: decklink_consumer(const configuration& config, const core::video_format_desc& format_desc, int channel_index, int sub_index) - : config_(config) - , channel_index_(channel_index) + : channel_index_(channel_index) , sub_index_(sub_index) + , config_(config) , decklink_(get_device(config.device_index)) , output_(decklink_) , configuration_(decklink_) @@ -175,7 +180,7 @@ public: , model_name_(get_model_name(decklink_)) , format_desc_(format_desc) , buffer_size_(config.buffer_depth) // Minimum buffer-size 3. - , frames_scheduled_(0) + , video_scheduled_(0) , audio_scheduled_(0) , preroll_count_(0) , audio_container_(buffer_size_+1) @@ -313,8 +318,11 @@ public: if(result == bmdOutputFrameDisplayedLate) { graph_->add_tag("late-frame"); - ++frames_scheduled_; - ++audio_scheduled_; + video_scheduled_ += format_desc_.duration; + audio_scheduled_ += reinterpret_cast(completed_frame)->audio_data().size()/format_desc_.audio_channels; + //++video_scheduled_; + //audio_scheduled_ += format_desc_.audio_cadence[0]; + //++audio_scheduled_; } else if(result == bmdOutputFrameDropped) graph_->add_tag("dropped-frame"); @@ -354,18 +362,18 @@ public: start_playback(); } else - schedule_next_audio(make_safe()); + schedule_next_audio(core::audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()], 0)); } else { std::shared_ptr frame; audio_frame_buffer_.pop(frame); - schedule_next_audio(make_safe_ptr(frame)); + schedule_next_audio(frame->audio_data()); } unsigned long buffered; output_->GetBufferedAudioSampleFrameCount(&buffered); - graph_->update_value("buffered-audio", static_cast(buffered)/(format_desc_.audio_samples_per_frame*2)); + graph_->update_value("buffered-audio", static_cast(buffered)/(format_desc_.audio_cadence[0]*2)); } catch(...) { @@ -377,22 +385,27 @@ public: return S_OK; } - void schedule_next_audio(const safe_ptr& frame) + template + void schedule_next_audio(const T& audio_data) { - const int sample_frame_count = frame->audio_data().size()/format_desc_.audio_channels; + const int sample_frame_count = audio_data.size()/format_desc_.audio_channels; - audio_container_.push_back(std::vector(frame->audio_data().begin(), frame->audio_data().end())); + audio_container_.push_back(std::vector(audio_data.begin(), audio_data.end())); - if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, (audio_scheduled_++) * sample_frame_count, format_desc_.audio_sample_rate, nullptr))) + if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr))) CASPAR_LOG(error) << print() << L" Failed to schedule audio."; + + audio_scheduled_ += sample_frame_count; } void schedule_next_video(const safe_ptr& frame) { CComPtr frame2(new decklink_frame(frame, format_desc_, config_.key_only)); - if(FAILED(output_->ScheduleVideoFrame(frame2, (frames_scheduled_++) * format_desc_.duration, format_desc_.duration, format_desc_.time_scale))) + if(FAILED(output_->ScheduleVideoFrame(frame2, video_scheduled_, format_desc_.duration, format_desc_.time_scale))) CASPAR_LOG(error) << print() << L" Failed to schedule video."; + video_scheduled_ += format_desc_.duration; + graph_->update_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5); tick_timer_.restart(); } @@ -424,6 +437,7 @@ struct decklink_consumer_proxy : public core::frame_consumer { const configuration config_; com_context context_; + std::vector audio_cadence_; public: decklink_consumer_proxy(const configuration& config) @@ -447,12 +461,16 @@ public: virtual void initialize(const core::video_format_desc& format_desc, int channel_index, int sub_index) override { context_.reset([&]{return new decklink_consumer(config_, format_desc, channel_index, sub_index);}); - + audio_cadence_ = format_desc.audio_cadence; + CASPAR_LOG(info) << print() << L" Successfully Initialized."; } virtual bool send(const safe_ptr& frame) override { + CASPAR_VERIFY(audio_cadence_.front() == static_cast(frame->audio_data().size())); + boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1); + context_->send(frame); return true; } diff --git a/modules/decklink/producer/decklink_producer.cpp b/modules/decklink/producer/decklink_producer.cpp index 97892034c..105f8d05a 100644 --- a/modules/decklink/producer/decklink_producer.cpp +++ b/modules/decklink/producer/decklink_producer.cpp @@ -87,6 +87,7 @@ class decklink_producer : boost::noncopyable, public IDeckLinkInputCallback boost::timer frame_timer_; safe_ptr frame_factory_; + std::vector audio_cadence_; tbb::concurrent_bounded_queue> frame_buffer_; @@ -102,6 +103,7 @@ public: , format_desc_(format_desc) , device_index_(device_index) , frame_factory_(frame_factory) + , audio_cadence_(frame_factory->get_video_format_desc().audio_cadence) , muxer_(format_desc.fps, frame_factory, filter) { frame_buffer_.set_capacity(2); @@ -137,10 +139,7 @@ public: BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Failed to start input stream.") << boost::errinfo_api_function("StartStreams")); - - if(format_desc_.duration == 1001) - CASPAR_LOG(warning) << print() << L"Audio not supported in NTSC frame-rates."; - + CASPAR_LOG(info) << print() << L" Successfully Initialized."; } @@ -192,15 +191,18 @@ public: muxer_.push(av_frame); // It is assumed that audio is always equal or ahead of video. - if(audio && SUCCEEDED(audio->GetBytes(&bytes)) && format_desc_.duration != 1001) + if(audio && SUCCEEDED(audio->GetBytes(&bytes))) { auto sample_frame_count = audio->GetSampleFrameCount(); auto audio_data = reinterpret_cast(bytes); muxer_.push(std::make_shared(audio_data, audio_data + sample_frame_count*format_desc_.audio_channels)); } else - muxer_.push(std::make_shared(frame_factory_->get_video_format_desc().audio_samples_per_frame, 0)); - + { + muxer_.push(std::make_shared(audio_cadence_.front(), 0)); + std::rotate(std::begin(audio_cadence_), std::begin(audio_cadence_)+1, std::end(audio_cadence_)); + } + for(auto frame = muxer_.poll(); frame; frame = muxer_.poll()) { if(!frame_buffer_.try_push(make_safe_ptr(frame))) @@ -303,7 +305,7 @@ safe_ptr create_producer(const safe_ptrget_video_format_desc(); - return create_destroy_proxy(make_safe(frame_factory, format_desc, device_index, filter_str, length)); + return create_producer_destroy_proxy(make_safe(frame_factory, format_desc, device_index, filter_str, length)); } }} \ No newline at end of file diff --git a/modules/ffmpeg/producer/audio/audio_resampler.cpp b/modules/ffmpeg/producer/audio/audio_resampler.cpp index 9f953f130..e92d3a8b0 100644 --- a/modules/ffmpeg/producer/audio/audio_resampler.cpp +++ b/modules/ffmpeg/producer/audio/audio_resampler.cpp @@ -49,7 +49,7 @@ struct audio_resampler::implementation buffer2_.resize(AVCODEC_MAX_AUDIO_FRAME_SIZE*2); CASPAR_LOG(warning) << L"Resampling." << - L" sample_rate:" << input_channels << + L" sample_rate:" << input_sample_rate << L" audio_channels:" << input_channels << L" sample_fmt:" << input_sample_format; diff --git a/modules/ffmpeg/producer/ffmpeg_producer.cpp b/modules/ffmpeg/producer/ffmpeg_producer.cpp index 48e8e7f6d..8ba264c35 100644 --- a/modules/ffmpeg/producer/ffmpeg_producer.cpp +++ b/modules/ffmpeg/producer/ffmpeg_producer.cpp @@ -340,7 +340,7 @@ safe_ptr create_producer(const safe_ptr(frame_factory, path, filter_str, loop, start, length)); + return create_producer_destroy_proxy(make_safe(frame_factory, path, filter_str, loop, start, length)); } }} \ No newline at end of file diff --git a/modules/ffmpeg/producer/muxer/frame_muxer.cpp b/modules/ffmpeg/producer/muxer/frame_muxer.cpp index 3f5deef5c..ca6f4a44a 100644 --- a/modules/ffmpeg/producer/muxer/frame_muxer.cpp +++ b/modules/ffmpeg/producer/muxer/frame_muxer.cpp @@ -53,6 +53,8 @@ struct frame_muxer::implementation : boost::noncopyable const double in_fps_; const video_format_desc format_desc_; bool auto_transcode_; + + std::vector audio_cadence_; size_t audio_sample_count_; size_t video_frame_count_; @@ -67,6 +69,7 @@ struct frame_muxer::implementation : boost::noncopyable , in_fps_(in_fps) , format_desc_(frame_factory->get_video_format_desc()) , auto_transcode_(env::properties().get("configuration.auto-transcode", true)) + , audio_cadence_(format_desc_.audio_cadence) , audio_sample_count_(0) , video_frame_count_(0) , frame_factory_(frame_factory) @@ -74,6 +77,8 @@ struct frame_muxer::implementation : boost::noncopyable { video_streams_.push(std::queue>()); audio_streams_.push(core::audio_buffer()); + boost::range::sort(audio_cadence_); + boost::range::reverse(audio_cadence_); } void push(const std::shared_ptr& video_frame, int hints) @@ -83,7 +88,6 @@ struct frame_muxer::implementation : boost::noncopyable if(video_frame == flush_video()) { - CASPAR_LOG(trace) << L"video-frame-count: " << static_cast(video_frame_count_); video_frame_count_ = 0; video_streams_.push(std::queue>()); } @@ -127,13 +131,12 @@ struct frame_muxer::implementation : boost::noncopyable if(audio == flush_audio()) { - CASPAR_LOG(trace) << L"audio-frame-count: " << audio_sample_count_/format_desc_.audio_samples_per_frame; audio_sample_count_ = 0; audio_streams_.push(core::audio_buffer()); } else if(audio == empty_audio()) { - boost::range::push_back(audio_streams_.back(), core::audio_buffer(format_desc_.audio_samples_per_frame, 0)); + boost::range::push_back(audio_streams_.back(), core::audio_buffer(audio_cadence_.front(), 0)); audio_sample_count_ += audio->size(); } else @@ -142,7 +145,7 @@ struct frame_muxer::implementation : boost::noncopyable audio_sample_count_ += audio->size(); } - if(audio_streams_.back().size() > 32*format_desc_.audio_samples_per_frame) + if(audio_streams_.back().size() > 32*audio_cadence_.front()) BOOST_THROW_EXCEPTION(invalid_operation() << source_info("frame_muxer") << msg_info("audio-stream overflow. This can be caused by incorrect frame-rate. Check clip meta-data.")); } @@ -174,9 +177,9 @@ struct frame_muxer::implementation : boost::noncopyable switch(display_mode_) { case display_mode::duplicate: - return audio_streams_.front().size()/2 >= format_desc_.audio_samples_per_frame; + return audio_streams_.front().size()/2 >= audio_cadence_.front(); default: - return audio_streams_.front().size() >= format_desc_.audio_samples_per_frame; + return audio_streams_.front().size() >= audio_cadence_.front(); } } @@ -253,13 +256,15 @@ struct frame_muxer::implementation : boost::noncopyable core::audio_buffer pop_audio() { - CASPAR_VERIFY(audio_streams_.front().size() >= format_desc_.audio_samples_per_frame); + CASPAR_VERIFY(audio_streams_.front().size() >= audio_cadence_.front()); auto begin = audio_streams_.front().begin(); - auto end = begin + format_desc_.audio_samples_per_frame; + auto end = begin + audio_cadence_.front(); - auto samples = core::audio_buffer(begin, end); + core::audio_buffer samples(begin, end); audio_streams_.front().erase(begin, end); + + boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1); return samples; } diff --git a/modules/flash/producer/flash_producer.cpp b/modules/flash/producer/flash_producer.cpp index 8078464a8..da8a34d26 100644 --- a/modules/flash/producer/flash_producer.cpp +++ b/modules/flash/producer/flash_producer.cpp @@ -443,7 +443,7 @@ safe_ptr create_producer(const safe_ptrget_video_format_desc()); - return create_destroy_proxy(make_safe(frame_factory, env::template_folder() + L"\\" + widen(template_host.filename), template_host.width, template_host.height)); + return create_producer_destroy_proxy(make_safe(frame_factory, env::template_folder() + L"\\" + widen(template_host.filename), template_host.width, template_host.height)); } std::wstring find_template(const std::wstring& template_name) diff --git a/modules/oal/consumer/oal_consumer.cpp b/modules/oal/consumer/oal_consumer.cpp index 85f365fe0..86e954cfd 100644 --- a/modules/oal/consumer/oal_consumer.cpp +++ b/modules/oal/consumer/oal_consumer.cpp @@ -41,6 +41,8 @@ namespace caspar { namespace oal { +typedef std::vector> audio_buffer_16; + struct oal_consumer : public core::frame_consumer, public sf::SoundStream { safe_ptr graph_; @@ -48,9 +50,10 @@ struct oal_consumer : public core::frame_consumer, public sf::SoundStream int channel_index_; int sub_index_; - tbb::concurrent_bounded_queue>>> input_; - boost::circular_buffer>> container_; + tbb::concurrent_bounded_queue> input_; + boost::circular_buffer container_; tbb::atomic is_running_; + core::audio_buffer temp; core::video_format_desc format_desc_; public: @@ -71,11 +74,11 @@ public: ~oal_consumer() { is_running_ = false; - input_.try_push(std::make_shared>>()); - input_.try_push(std::make_shared>>()); + input_.try_push(std::make_shared()); + input_.try_push(std::make_shared()); Stop(); - input_.try_push(std::make_shared>>()); - input_.try_push(std::make_shared>>()); + input_.try_push(std::make_shared()); + input_.try_push(std::make_shared()); CASPAR_LOG(info) << print() << L" Successfully Uninitialized."; } @@ -97,8 +100,7 @@ public: virtual bool send(const safe_ptr& frame) override { - input_.push(std::make_shared>>(core::audio_32_to_16_sse(frame->audio_data()))); - + input_.push(std::make_shared(core::audio_32_to_16_sse(frame->audio_data()))); return true; } @@ -116,7 +118,7 @@ public: virtual bool OnGetData(sf::SoundStream::Chunk& data) override { - std::shared_ptr>> audio_data; + std::shared_ptr audio_data; input_.pop(audio_data); container_.push_back(std::move(*audio_data));