2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Robert Nagy, ronag89@gmail.com
22 #include "../StdAfx.h"
24 #include "decklink_consumer.h"
26 #include "../util/util.h"
28 #include "../interop/DeckLinkAPI_h.h"
30 #include <core/frame/frame.h>
31 #include <core/mixer/audio/audio_mixer.h>
33 #include <common/executor.h>
34 #include <common/lock.h>
35 #include <common/diagnostics/graph.h>
36 #include <common/except.h>
37 #include <common/memshfl.h>
38 #include <common/array.h>
39 #include <common/future.h>
41 #include <core/consumer/frame_consumer.h>
43 #include <tbb/concurrent_queue.h>
44 #include <tbb/cache_aligned_allocator.h>
46 #include <common/assert.h>
47 #include <boost/lexical_cast.hpp>
48 #include <boost/circular_buffer.hpp>
49 #include <boost/timer.hpp>
50 #include <boost/property_tree/ptree.hpp>
52 namespace caspar { namespace decklink {
75 int base_buffer_depth;
79 , embedded_audio(true)
80 , keyer(default_keyer)
81 , latency(default_latency)
83 , base_buffer_depth(3)
87 int buffer_depth() const
89 return base_buffer_depth + (latency == low_latency ? 0 : 1) + (embedded_audio ? 1 : 0);
93 class decklink_frame : public IDeckLinkVideoFrame
95 tbb::atomic<int> ref_count_;
96 core::const_frame frame_;
97 const core::video_format_desc format_desc_;
100 std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>> data_;
102 decklink_frame(core::const_frame frame, const core::video_format_desc& format_desc, bool key_only)
104 , format_desc_(format_desc)
105 , key_only_(key_only)
112 STDMETHOD (QueryInterface(REFIID, LPVOID*))
114 return E_NOINTERFACE;
117 STDMETHOD_(ULONG, AddRef())
122 STDMETHOD_(ULONG, Release())
124 if(--ref_count_ == 0)
129 // IDecklinkVideoFrame
131 STDMETHOD_(long, GetWidth()) {return static_cast<long>(format_desc_.width);}
132 STDMETHOD_(long, GetHeight()) {return static_cast<long>(format_desc_.height);}
133 STDMETHOD_(long, GetRowBytes()) {return static_cast<long>(format_desc_.width*4);}
134 STDMETHOD_(BMDPixelFormat, GetPixelFormat()) {return bmdFormat8BitBGRA;}
135 STDMETHOD_(BMDFrameFlags, GetFlags()) {return bmdFrameFlagDefault;}
137 STDMETHOD(GetBytes(void** buffer))
141 if(static_cast<int>(frame_.image_data().size()) != format_desc_.size)
143 data_.resize(format_desc_.size, 0);
144 *buffer = data_.data();
150 data_.resize(frame_.image_data().size());
151 aligned_memshfl(data_.data(), frame_.image_data().begin(), frame_.image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
153 *buffer = data_.data();
156 *buffer = const_cast<uint8_t*>(frame_.image_data().begin());
160 CASPAR_LOG_CURRENT_EXCEPTION();
167 STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}
168 STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)) {return S_FALSE;}
172 const core::audio_buffer& audio_data()
174 return frame_.audio_data();
178 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
180 const int channel_index_;
181 const configuration config_;
183 CComPtr<IDeckLink> decklink_;
184 CComQIPtr<IDeckLinkOutput> output_;
185 CComQIPtr<IDeckLinkConfiguration> configuration_;
186 CComQIPtr<IDeckLinkKeyer> keyer_;
187 CComQIPtr<IDeckLinkAttributes> attributes_;
189 tbb::spin_mutex exception_mutex_;
190 std::exception_ptr exception_;
192 tbb::atomic<bool> is_running_;
194 const std::wstring model_name_;
195 const core::video_format_desc format_desc_;
196 const int buffer_size_;
198 long long video_scheduled_;
199 long long audio_scheduled_;
203 boost::circular_buffer<std::vector<int32_t>> audio_container_;
205 tbb::concurrent_bounded_queue<core::const_frame> video_frame_buffer_;
206 tbb::concurrent_bounded_queue<core::const_frame> audio_frame_buffer_;
208 spl::shared_ptr<diagnostics::graph> graph_;
209 boost::timer tick_timer_;
210 retry_task<bool> send_completion_;
213 decklink_consumer(const configuration& config, const core::video_format_desc& format_desc, int channel_index)
214 : channel_index_(channel_index)
216 , decklink_(get_device(config.device_index))
218 , configuration_(decklink_)
220 , attributes_(decklink_)
221 , model_name_(get_model_name(decklink_))
222 , format_desc_(format_desc)
223 , buffer_size_(config.buffer_depth()) // Minimum buffer-size 3.
224 , video_scheduled_(0)
225 , audio_scheduled_(0)
227 , audio_container_(buffer_size_+1)
231 video_frame_buffer_.set_capacity(1);
232 audio_frame_buffer_.set_capacity(1);
234 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
235 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
236 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
237 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
238 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
239 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
240 graph_->set_text(print());
241 diagnostics::register_graph(graph_);
243 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));
245 if(config.embedded_audio)
248 set_latency(config.latency);
249 set_keyer(config.keyer);
251 if(config.embedded_audio)
252 output_->BeginAudioPreroll();
254 for(int n = 0; n < buffer_size_; ++n)
255 schedule_next_video(core::const_frame::empty());
257 if(!config.embedded_audio)
264 video_frame_buffer_.try_push(core::const_frame::empty());
265 audio_frame_buffer_.try_push(core::const_frame::empty());
267 if(output_ != nullptr)
269 output_->StopScheduledPlayback(0, nullptr, 0);
270 if(config_.embedded_audio)
271 output_->DisableAudioOutput();
272 output_->DisableVideoOutput();
276 void set_latency(configuration::latency_t latency)
278 if(latency == configuration::low_latency)
280 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
281 CASPAR_LOG(info) << print() << L" Enabled low-latency mode.";
283 else if(latency == configuration::normal_latency)
285 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
286 CASPAR_LOG(info) << print() << L" Disabled low-latency mode.";
290 void set_keyer(configuration::keyer_t keyer)
292 if(keyer == configuration::internal_keyer)
295 if(SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
296 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
297 else if(FAILED(keyer_->Enable(FALSE)))
298 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
299 else if(FAILED(keyer_->SetLevel(255)))
300 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
302 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";
304 else if(keyer == configuration::external_keyer)
307 if(SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
308 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
309 else if(FAILED(keyer_->Enable(TRUE)))
310 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
311 else if(FAILED(keyer_->SetLevel(255)))
312 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
314 CASPAR_LOG(info) << print() << L" Enabled external keyer.";
320 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped)))
321 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable audio output."));
323 if(FAILED(output_->SetAudioCallback(this)))
324 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not set audio callback."));
326 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
329 void enable_video(BMDDisplayMode display_mode)
331 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
332 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable video output."));
334 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
335 CASPAR_THROW_EXCEPTION(caspar_exception()
336 << msg_info(u8(print()) + " Failed to set playback completion callback.")
337 << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
340 void start_playback()
342 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
343 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Failed to schedule playback."));
346 STDMETHOD (QueryInterface(REFIID, LPVOID*)) {return E_NOINTERFACE;}
347 STDMETHOD_(ULONG, AddRef()) {return 1;}
348 STDMETHOD_(ULONG, Release()) {return 1;}
350 STDMETHOD(ScheduledPlaybackHasStopped())
353 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
357 STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))
364 if(result == bmdOutputFrameDisplayedLate)
366 graph_->set_tag("late-frame");
367 video_scheduled_ += format_desc_.duration;
368 audio_scheduled_ += reinterpret_cast<decklink_frame*>(completed_frame)->audio_data().size()/format_desc_.audio_channels;
369 //++video_scheduled_;
370 //audio_scheduled_ += format_desc_.audio_cadence[0];
371 //++audio_scheduled_;
373 else if(result == bmdOutputFrameDropped)
374 graph_->set_tag("dropped-frame");
375 else if(result == bmdOutputFrameFlushed)
376 graph_->set_tag("flushed-frame");
378 auto frame = core::const_frame::empty();
379 video_frame_buffer_.pop(frame);
380 send_completion_.try_completion();
381 schedule_next_video(frame);
383 unsigned long buffered;
384 output_->GetBufferedVideoFrameCount(&buffered);
385 graph_->set_value("buffered-video", static_cast<double>(buffered)/format_desc_.fps);
389 lock(exception_mutex_, [&]
391 exception_ = std::current_exception();
399 STDMETHOD(RenderAudioSamples(BOOL preroll))
408 if(++preroll_count_ >= buffer_size_)
410 output_->EndAudioPreroll();
414 schedule_next_audio(core::audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()], 0));
418 auto frame = core::const_frame::empty();
419 audio_frame_buffer_.pop(frame);
420 send_completion_.try_completion();
421 schedule_next_audio(frame.audio_data());
424 unsigned long buffered;
425 output_->GetBufferedAudioSampleFrameCount(&buffered);
426 graph_->set_value("buffered-audio", static_cast<double>(buffered)/(format_desc_.audio_cadence[0]*2));
430 tbb::spin_mutex::scoped_lock lock(exception_mutex_);
431 exception_ = std::current_exception();
439 void schedule_next_audio(const T& audio_data)
441 auto sample_frame_count = static_cast<int>(audio_data.size()/format_desc_.audio_channels);
443 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
445 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
446 CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
448 audio_scheduled_ += sample_frame_count;
451 void schedule_next_video(core::const_frame frame)
453 CComPtr<IDeckLinkVideoFrame> frame2(new decklink_frame(frame, format_desc_, config_.key_only));
454 if(FAILED(output_->ScheduleVideoFrame(frame2, video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
455 CASPAR_LOG(error) << print() << L" Failed to schedule video.";
457 video_scheduled_ += format_desc_.duration;
459 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
460 tick_timer_.restart();
463 boost::unique_future<bool> send(core::const_frame frame)
465 auto exception = lock(exception_mutex_, [&]
470 if(exception != nullptr)
471 std::rethrow_exception(exception);
474 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Is not running."));
476 bool audio_ready = !config_.embedded_audio;
477 bool video_ready = false;
479 auto enqueue_task = [audio_ready, video_ready, frame, this]() mutable -> boost::optional<bool>
482 audio_ready = audio_frame_buffer_.try_push(frame);
485 video_ready = video_frame_buffer_.try_push(frame);
487 if (audio_ready && video_ready)
490 return boost::optional<bool>();
494 return wrap_as_future(true);
496 send_completion_.set_task(enqueue_task);
498 return send_completion_.get_future();
501 std::wstring print() const
503 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_) + L"-" +
504 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
508 struct decklink_consumer_proxy : public core::frame_consumer
510 const configuration config_;
511 std::unique_ptr<decklink_consumer> consumer_;
515 decklink_consumer_proxy(const configuration& config)
517 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
519 executor_.begin_invoke([=]
521 ::CoInitialize(nullptr);
525 ~decklink_consumer_proxy()
535 void initialize(const core::video_format_desc& format_desc, int channel_index) override
540 consumer_.reset(new decklink_consumer(config_, format_desc, channel_index));
544 boost::unique_future<bool> send(core::const_frame frame) override
546 return consumer_->send(frame);
549 std::wstring print() const override
551 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
554 std::wstring name() const override
559 boost::property_tree::wptree info() const override
561 boost::property_tree::wptree info;
562 info.add(L"type", L"decklink");
563 info.add(L"key-only", config_.key_only);
564 info.add(L"device", config_.device_index);
565 info.add(L"low-latency", config_.low_latency);
566 info.add(L"embedded-audio", config_.embedded_audio);
567 info.add(L"low-latency", config_.low_latency);
568 //info.add(L"internal-key", config_.internal_key);
572 int buffer_depth() const override
574 return config_.buffer_depth();
577 int index() const override
579 return 300 + config_.device_index;
582 void subscribe(const monitor::observable::observer_ptr& o) override
586 void unsubscribe(const monitor::observable::observer_ptr& o) override
591 spl::shared_ptr<core::frame_consumer> create_consumer(const std::vector<std::wstring>& params)
593 if(params.size() < 1 || params[0] != L"DECKLINK")
594 return core::frame_consumer::empty();
596 configuration config;
598 if(params.size() > 1)
599 config.device_index = boost::lexical_cast<int>(params[1]);
601 if(std::find(params.begin(), params.end(), L"INTERNAL_KEY") != params.end())
602 config.keyer = configuration::internal_keyer;
603 else if(std::find(params.begin(), params.end(), L"EXTERNAL_KEY") != params.end())
604 config.keyer = configuration::external_keyer;
606 config.keyer = configuration::default_keyer;
608 if(std::find(params.begin(), params.end(), L"LOW_LATENCY") != params.end())
609 config.latency = configuration::low_latency;
611 config.embedded_audio = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();
612 config.key_only = std::find(params.begin(), params.end(), L"KEY_ONLY") != params.end();
614 return spl::make_shared<decklink_consumer_proxy>(config);
617 spl::shared_ptr<core::frame_consumer> create_consumer(const boost::property_tree::wptree& ptree)
619 configuration config;
621 auto keyer = ptree.get(L"keyer", L"default");
622 if(keyer == L"external")
623 config.keyer = configuration::external_keyer;
624 else if(keyer == L"internal")
625 config.keyer = configuration::internal_keyer;
627 auto latency = ptree.get(L"latency", L"normal");
628 if(latency == L"low")
629 config.latency = configuration::low_latency;
630 else if(latency == L"normal")
631 config.latency = configuration::normal_latency;
633 config.key_only = ptree.get(L"key-only", config.key_only);
634 config.device_index = ptree.get(L"device", config.device_index);
635 config.embedded_audio = ptree.get(L"embedded-audio", config.embedded_audio);
636 config.base_buffer_depth = ptree.get(L"buffer-depth", config.base_buffer_depth);
638 return spl::make_shared<decklink_consumer_proxy>(config);
644 ##############################################################################
650 BMD Developer Support
651 developer@blackmagic-design.com
653 -----------------------------------------------------------------------------
655 Thanks for your inquiry. The minimum number of frames that you can preroll
656 for scheduled playback is three frames for video and four frames for audio.
657 As you mentioned if you preroll less frames then playback will not start or
658 playback will be very sporadic. From our experience with Media Express, we
659 recommended that at least seven frames are prerolled for smooth playback.
661 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
662 There can be around 3 frames worth of latency on scheduled output.
663 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
664 reduced or removed for scheduled playback. If the DisplayVideoFrameSync()
665 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
666 guarantee that the provided frame will be output as soon the previous
667 frame output has been completed.
668 ################################################################################
672 ##############################################################################
673 Async DMA Transfer without redundant copying
678 BMD Developer Support
679 developer@blackmagic-design.com
681 -----------------------------------------------------------------------------
683 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
684 and providing a pointer to your video buffer when GetBytes() is called.
685 This may help to keep copying to a minimum. Please ensure that the pixel
686 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
687 have to colourspace convert which may result in additional copying.
688 ################################################################################