2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Robert Nagy, ronag89@gmail.com
22 #include "../StdAfx.h"
24 #include "decklink_consumer.h"
26 #include "../util/util.h"
28 #include "../decklink_api.h"
30 #include <core/frame/frame.h>
31 #include <core/mixer/audio/audio_mixer.h>
32 #include <core/consumer/frame_consumer.h>
33 #include <core/diagnostics/call_context.h>
34 #include <core/help/help_sink.h>
35 #include <core/help/help_repository.h>
37 #include <common/executor.h>
38 #include <common/lock.h>
39 #include <common/diagnostics/graph.h>
40 #include <common/except.h>
41 #include <common/memshfl.h>
42 #include <common/array.h>
43 #include <common/future.h>
44 #include <common/cache_aligned_vector.h>
45 #include <common/timer.h>
46 #include <common/param.h>
48 #include <tbb/concurrent_queue.h>
50 #include <common/assert.h>
51 #include <boost/lexical_cast.hpp>
52 #include <boost/circular_buffer.hpp>
53 #include <boost/property_tree/ptree.hpp>
55 namespace caspar { namespace decklink {
74 bool embedded_audio = true;
75 keyer_t keyer = keyer_t::default_keyer;
76 latency_t latency = latency_t::default_latency;
77 bool key_only = false;
78 int base_buffer_depth = 3;
80 int buffer_depth() const
82 return base_buffer_depth + (latency == latency_t::low_latency ? 0 : 1) + (embedded_audio ? 1 : 0);
86 class decklink_frame : public IDeckLinkVideoFrame
88 tbb::atomic<int> ref_count_;
89 core::const_frame frame_;
90 const core::video_format_desc format_desc_;
93 cache_aligned_vector<uint8_t> data_;
95 decklink_frame(core::const_frame frame, const core::video_format_desc& format_desc, bool key_only)
97 , format_desc_(format_desc)
105 virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)
107 return E_NOINTERFACE;
110 virtual ULONG STDMETHODCALLTYPE AddRef()
115 virtual ULONG STDMETHODCALLTYPE Release()
117 if(--ref_count_ == 0)
122 // IDecklinkVideoFrame
124 virtual long STDMETHODCALLTYPE GetWidth() {return static_cast<long>(format_desc_.width);}
125 virtual long STDMETHODCALLTYPE GetHeight() {return static_cast<long>(format_desc_.height);}
126 virtual long STDMETHODCALLTYPE GetRowBytes() {return static_cast<long>(format_desc_.width*4);}
127 virtual BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat() {return bmdFormat8BitBGRA;}
128 virtual BMDFrameFlags STDMETHODCALLTYPE GetFlags() {return bmdFrameFlagDefault;}
130 virtual HRESULT STDMETHODCALLTYPE GetBytes(void** buffer)
134 if(static_cast<int>(frame_.image_data().size()) != format_desc_.size)
136 data_.resize(format_desc_.size, 0);
137 *buffer = data_.data();
143 data_.resize(frame_.image_data().size());
144 aligned_memshfl(data_.data(), frame_.image_data().begin(), frame_.image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
146 *buffer = data_.data();
149 *buffer = const_cast<uint8_t*>(frame_.image_data().begin());
153 CASPAR_LOG_CURRENT_EXCEPTION();
160 virtual HRESULT STDMETHODCALLTYPE GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode) {return S_FALSE;}
161 virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary) {return S_FALSE;}
165 const core::audio_buffer& audio_data()
167 return frame_.audio_data();
171 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
173 const int channel_index_;
174 const configuration config_;
176 com_ptr<IDeckLink> decklink_ = get_device(config_.device_index);
177 com_iface_ptr<IDeckLinkOutput> output_ = iface_cast<IDeckLinkOutput>(decklink_);
178 com_iface_ptr<IDeckLinkConfiguration> configuration_ = iface_cast<IDeckLinkConfiguration>(decklink_);
179 com_iface_ptr<IDeckLinkKeyer> keyer_ = iface_cast<IDeckLinkKeyer>(decklink_);
180 com_iface_ptr<IDeckLinkAttributes> attributes_ = iface_cast<IDeckLinkAttributes>(decklink_);
182 tbb::spin_mutex exception_mutex_;
183 std::exception_ptr exception_;
185 tbb::atomic<bool> is_running_;
187 const std::wstring model_name_ = get_model_name(decklink_);
188 const core::video_format_desc format_desc_;
189 const int buffer_size_ = config_.buffer_depth(); // Minimum buffer-size 3.
191 long long video_scheduled_ = 0;
192 long long audio_scheduled_ = 0;
194 int preroll_count_ = 0;
196 boost::circular_buffer<std::vector<int32_t>> audio_container_ { buffer_size_ + 1 };
198 tbb::concurrent_bounded_queue<core::const_frame> video_frame_buffer_;
199 tbb::concurrent_bounded_queue<core::const_frame> audio_frame_buffer_;
201 spl::shared_ptr<diagnostics::graph> graph_;
202 caspar::timer tick_timer_;
203 retry_task<bool> send_completion_;
206 decklink_consumer(const configuration& config, const core::video_format_desc& format_desc, int channel_index)
207 : channel_index_(channel_index)
209 , format_desc_(format_desc)
213 video_frame_buffer_.set_capacity(1);
215 // Blackmagic calls RenderAudioSamples() 50 times per second
216 // regardless of video mode so we sometimes need to give them
217 // samples from 2 frames in order to keep up
218 audio_frame_buffer_.set_capacity((format_desc.fps > 50.0) ? 2 : 1);
220 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
221 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
222 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
223 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
224 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
225 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
226 graph_->set_text(print());
227 diagnostics::register_graph(graph_);
229 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));
231 if(config.embedded_audio)
234 set_latency(config.latency);
235 set_keyer(config.keyer);
237 if(config.embedded_audio)
238 output_->BeginAudioPreroll();
240 for(int n = 0; n < buffer_size_; ++n)
241 schedule_next_video(core::const_frame::empty());
243 if(!config.embedded_audio)
250 video_frame_buffer_.try_push(core::const_frame::empty());
251 audio_frame_buffer_.try_push(core::const_frame::empty());
253 if(output_ != nullptr)
255 output_->StopScheduledPlayback(0, nullptr, 0);
256 if(config_.embedded_audio)
257 output_->DisableAudioOutput();
258 output_->DisableVideoOutput();
262 void set_latency(configuration::latency_t latency)
264 if(latency == configuration::latency_t::low_latency)
266 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
267 CASPAR_LOG(info) << print() << L" Enabled low-latency mode.";
269 else if(latency == configuration::latency_t::normal_latency)
271 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
272 CASPAR_LOG(info) << print() << L" Disabled low-latency mode.";
276 void set_keyer(configuration::keyer_t keyer)
278 if(keyer == configuration::keyer_t::internal_keyer)
281 if(SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
282 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
283 else if(FAILED(keyer_->Enable(FALSE)))
284 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
285 else if(FAILED(keyer_->SetLevel(255)))
286 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
288 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";
290 else if(keyer == configuration::keyer_t::external_keyer)
293 if(SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
294 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
295 else if(FAILED(keyer_->Enable(TRUE)))
296 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
297 else if(FAILED(keyer_->SetLevel(255)))
298 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
300 CASPAR_LOG(info) << print() << L" Enabled external keyer.";
306 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped)))
307 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable audio output."));
309 if(FAILED(output_->SetAudioCallback(this)))
310 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not set audio callback."));
312 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
315 void enable_video(BMDDisplayMode display_mode)
317 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
318 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable video output."));
320 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
321 CASPAR_THROW_EXCEPTION(caspar_exception()
322 << msg_info(u8(print()) + " Failed to set playback completion callback.")
323 << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
326 void start_playback()
328 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
329 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Failed to schedule playback."));
332 virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*) {return E_NOINTERFACE;}
333 virtual ULONG STDMETHODCALLTYPE AddRef() {return 1;}
334 virtual ULONG STDMETHODCALLTYPE Release() {return 1;}
336 virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
339 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
343 virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result)
350 if(result == bmdOutputFrameDisplayedLate)
352 graph_->set_tag("late-frame");
353 video_scheduled_ += format_desc_.duration;
354 audio_scheduled_ += reinterpret_cast<decklink_frame*>(completed_frame)->audio_data().size()/format_desc_.audio_channels;
355 //++video_scheduled_;
356 //audio_scheduled_ += format_desc_.audio_cadence[0];
357 //++audio_scheduled_;
359 else if(result == bmdOutputFrameDropped)
360 graph_->set_tag("dropped-frame");
361 else if(result == bmdOutputFrameFlushed)
362 graph_->set_tag("flushed-frame");
364 auto frame = core::const_frame::empty();
365 video_frame_buffer_.pop(frame);
366 send_completion_.try_completion();
367 schedule_next_video(frame);
370 output_->GetBufferedVideoFrameCount(&buffered);
371 graph_->set_value("buffered-video", static_cast<double>(buffered)/format_desc_.fps);
375 lock(exception_mutex_, [&]
377 exception_ = std::current_exception();
385 virtual HRESULT STDMETHODCALLTYPE RenderAudioSamples(BOOL preroll)
394 if(++preroll_count_ >= buffer_size_)
396 output_->EndAudioPreroll();
401 schedule_next_audio(core::audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()] * format_desc_.audio_channels, 0));
406 auto frame = core::const_frame::empty();
408 while(audio_frame_buffer_.try_pop(frame))
410 send_completion_.try_completion();
411 schedule_next_audio(frame.audio_data());
416 output_->GetBufferedAudioSampleFrameCount(&buffered);
417 graph_->set_value("buffered-audio", static_cast<double>(buffered) / (format_desc_.audio_cadence[0] * format_desc_.audio_channels * 2));
421 tbb::spin_mutex::scoped_lock lock(exception_mutex_);
422 exception_ = std::current_exception();
430 void schedule_next_audio(const T& audio_data)
432 auto sample_frame_count = static_cast<int>(audio_data.size()/format_desc_.audio_channels);
434 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
436 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
437 CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
439 audio_scheduled_ += sample_frame_count;
442 void schedule_next_video(core::const_frame frame)
444 auto frame2 = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, config_.key_only));
445 if(FAILED(output_->ScheduleVideoFrame(get_raw(frame2), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
446 CASPAR_LOG(error) << print() << L" Failed to schedule video.";
448 video_scheduled_ += format_desc_.duration;
450 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
451 tick_timer_.restart();
454 std::future<bool> send(core::const_frame frame)
456 auto exception = lock(exception_mutex_, [&]
461 if(exception != nullptr)
462 std::rethrow_exception(exception);
465 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Is not running."));
467 bool audio_ready = !config_.embedded_audio;
468 bool video_ready = false;
470 auto enqueue_task = [audio_ready, video_ready, frame, this]() mutable -> boost::optional<bool>
473 audio_ready = audio_frame_buffer_.try_push(frame);
476 video_ready = video_frame_buffer_.try_push(frame);
478 if (audio_ready && video_ready)
481 return boost::optional<bool>();
485 return make_ready_future(true);
487 send_completion_.set_task(enqueue_task);
489 return send_completion_.get_future();
492 std::wstring print() const
494 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_) + L"-" +
495 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
499 struct decklink_consumer_proxy : public core::frame_consumer
501 core::monitor::subject monitor_subject_;
502 const configuration config_;
503 std::unique_ptr<decklink_consumer> consumer_;
507 decklink_consumer_proxy(const configuration& config)
509 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
511 auto ctx = core::diagnostics::call_context::for_thread();
512 executor_.begin_invoke([=]
514 core::diagnostics::call_context::for_thread() = ctx;
519 ~decklink_consumer_proxy()
530 void initialize(const core::video_format_desc& format_desc, int channel_index) override
535 consumer_.reset(new decklink_consumer(config_, format_desc, channel_index));
539 std::future<bool> send(core::const_frame frame) override
541 return consumer_->send(frame);
544 std::wstring print() const override
546 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
549 std::wstring name() const override
554 boost::property_tree::wptree info() const override
556 boost::property_tree::wptree info;
557 info.add(L"type", L"decklink");
558 info.add(L"key-only", config_.key_only);
559 info.add(L"device", config_.device_index);
560 info.add(L"low-latency", config_.latency == configuration::latency_t::low_latency);
561 info.add(L"embedded-audio", config_.embedded_audio);
562 //info.add(L"internal-key", config_.internal_key);
566 int buffer_depth() const override
568 return config_.buffer_depth();
571 int index() const override
573 return 300 + config_.device_index;
576 core::monitor::subject& monitor_output()
578 return monitor_subject_;
582 void describe_consumer(core::help_sink& sink, const core::help_repository& repo)
584 sink.short_description(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
585 sink.syntax(L"DECKLINK "
586 L"{[device_index:int]|1} "
587 L"{[keyer:INTERNAL_KEY,EXTERNAL_KEY]} "
588 L"{[low_latency:LOW_LATENCY]} "
589 L"{[embedded_audio:EMBEDDED_AUDIO]} "
590 L"{[key_only:KEY_ONLY]}");
591 sink.para()->text(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
593 ->item(L"device_index", L"The Blackmagic video card to use (See Blackmagic control panel for card order). Default is 1.")
594 ->item(L"keyer", L"If given tries to enable either internal or external keying. Not all Blackmagic cards supports this.")
595 ->item(L"low_latency", L"Tries to enable low latency if given.")
596 ->item(L"embedded_audio", L"Embeds the audio into the SDI signal if given.")
598 L" will extract only the alpha channel from the "
599 L"channel. This is useful when you have two SDI video cards, and neither has native support "
600 L"for separate fill/key output");
601 sink.para()->text(L"Examples:");
602 sink.example(L">> ADD 1 DECKLINK", L"for using the default device_index of 1.");
603 sink.example(L">> ADD 1 DECKLINK 2", L"uses device_index 2.");
604 sink.example(L">> ADD 1 DECKLINK 1 EXTERNAL_KEY EMBEDDED_AUDIO");
606 L">> ADD 1 DECKLINK 1 EMBEDDED_AUDIO\n"
607 L">> ADD 1 DECKLINK 2 KEY_ONLY", L"uses device with index 1 as fill output with audio and device with index 2 as key output.");
610 spl::shared_ptr<core::frame_consumer> create_consumer(
611 const std::vector<std::wstring>& params, core::interaction_sink*)
613 if (params.size() < 1 || !boost::iequals(params.at(0), L"DECKLINK"))
614 return core::frame_consumer::empty();
616 configuration config;
618 if (params.size() > 1)
619 config.device_index = boost::lexical_cast<int>(params.at(1));
621 if (contains_param(L"INTERNAL_KEY", params))
622 config.keyer = configuration::keyer_t::internal_keyer;
623 else if (contains_param(L"EXTERNAL_KEY", params))
624 config.keyer = configuration::keyer_t::external_keyer;
626 config.keyer = configuration::keyer_t::default_keyer;
628 if (contains_param(L"LOW_LATENCY", params))
629 config.latency = configuration::latency_t::low_latency;
631 config.embedded_audio = contains_param(L"EMBEDDED_AUDIO", params);
632 config.key_only = contains_param(L"KEY_ONLY", params);
634 return spl::make_shared<decklink_consumer_proxy>(config);
637 spl::shared_ptr<core::frame_consumer> create_preconfigured_consumer(
638 const boost::property_tree::wptree& ptree, core::interaction_sink*)
640 configuration config;
642 auto keyer = ptree.get(L"keyer", L"default");
643 if(keyer == L"external")
644 config.keyer = configuration::keyer_t::external_keyer;
645 else if(keyer == L"internal")
646 config.keyer = configuration::keyer_t::internal_keyer;
648 auto latency = ptree.get(L"latency", L"normal");
649 if(latency == L"low")
650 config.latency = configuration::latency_t::low_latency;
651 else if(latency == L"normal")
652 config.latency = configuration::latency_t::normal_latency;
654 config.key_only = ptree.get(L"key-only", config.key_only);
655 config.device_index = ptree.get(L"device", config.device_index);
656 config.embedded_audio = ptree.get(L"embedded-audio", config.embedded_audio);
657 config.base_buffer_depth = ptree.get(L"buffer-depth", config.base_buffer_depth);
659 return spl::make_shared<decklink_consumer_proxy>(config);
665 ##############################################################################
671 BMD Developer Support
672 developer@blackmagic-design.com
674 -----------------------------------------------------------------------------
676 Thanks for your inquiry. The minimum number of frames that you can preroll
677 for scheduled playback is three frames for video and four frames for audio.
678 As you mentioned if you preroll less frames then playback will not start or
679 playback will be very sporadic. From our experience with Media Express, we
680 recommended that at least seven frames are prerolled for smooth playback.
682 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
683 There can be around 3 frames worth of latency on scheduled output.
684 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
685 reduced or removed for scheduled playback. If the DisplayVideoFrameSync()
686 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
687 guarantee that the provided frame will be output as soon the previous
688 frame output has been completed.
689 ################################################################################
693 ##############################################################################
694 Async DMA Transfer without redundant copying
699 BMD Developer Support
700 developer@blackmagic-design.com
702 -----------------------------------------------------------------------------
704 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
705 and providing a pointer to your video buffer when GetBytes() is called.
706 This may help to keep copying to a minimum. Please ensure that the pixel
707 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
708 have to colourspace convert which may result in additional copying.
709 ################################################################################