2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Robert Nagy, ronag89@gmail.com
22 #include "../StdAfx.h"
24 #include "decklink_consumer.h"
26 #include "../util/util.h"
28 #include "../interop/DeckLinkAPI_h.h"
30 #include <core/frame/frame.h>
31 #include <core/mixer/audio/audio_mixer.h>
33 #include <common/executor.h>
34 #include <common/lock.h>
35 #include <common/diagnostics/graph.h>
36 #include <common/except.h>
37 #include <common/memshfl.h>
38 #include <common/array.h>
39 #include <common/future.h>
41 #include <core/consumer/frame_consumer.h>
43 #include <tbb/concurrent_queue.h>
44 #include <tbb/cache_aligned_allocator.h>
46 #include <common/assert.h>
47 #include <boost/lexical_cast.hpp>
48 #include <boost/circular_buffer.hpp>
49 #include <boost/timer.hpp>
50 #include <boost/property_tree/ptree.hpp>
52 namespace caspar { namespace decklink {
75 int base_buffer_depth;
79 , embedded_audio(true)
80 , keyer(default_keyer)
81 , latency(default_latency)
83 , base_buffer_depth(3)
87 int buffer_depth() const
89 return base_buffer_depth + (latency == low_latency ? 0 : 1) + (embedded_audio ? 1 : 0);
93 class decklink_frame : public IDeckLinkVideoFrame
95 tbb::atomic<int> ref_count_;
96 core::const_frame frame_;
97 const core::video_format_desc format_desc_;
100 std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>> data_;
102 decklink_frame(core::const_frame frame, const core::video_format_desc& format_desc, bool key_only)
104 , format_desc_(format_desc)
105 , key_only_(key_only)
112 STDMETHOD (QueryInterface(REFIID, LPVOID*))
114 return E_NOINTERFACE;
117 STDMETHOD_(ULONG, AddRef())
122 STDMETHOD_(ULONG, Release())
124 if(--ref_count_ == 0)
129 // IDecklinkVideoFrame
131 STDMETHOD_(long, GetWidth()) {return static_cast<long>(format_desc_.width);}
132 STDMETHOD_(long, GetHeight()) {return static_cast<long>(format_desc_.height);}
133 STDMETHOD_(long, GetRowBytes()) {return static_cast<long>(format_desc_.width*4);}
134 STDMETHOD_(BMDPixelFormat, GetPixelFormat()) {return bmdFormat8BitBGRA;}
135 STDMETHOD_(BMDFrameFlags, GetFlags()) {return bmdFrameFlagDefault;}
137 STDMETHOD(GetBytes(void** buffer))
141 if(static_cast<int>(frame_.image_data().size()) != format_desc_.size)
143 data_.resize(format_desc_.size, 0);
144 *buffer = data_.data();
150 data_.resize(frame_.image_data().size());
151 aligned_memshfl(data_.data(), frame_.image_data().begin(), frame_.image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
153 *buffer = data_.data();
156 *buffer = const_cast<uint8_t*>(frame_.image_data().begin());
160 CASPAR_LOG_CURRENT_EXCEPTION();
167 STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}
168 STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)) {return S_FALSE;}
172 const core::audio_buffer& audio_data()
174 return frame_.audio_data();
178 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
180 const int channel_index_;
181 const configuration config_;
183 CComPtr<IDeckLink> decklink_;
184 CComQIPtr<IDeckLinkOutput> output_;
185 CComQIPtr<IDeckLinkConfiguration> configuration_;
186 CComQIPtr<IDeckLinkKeyer> keyer_;
187 CComQIPtr<IDeckLinkAttributes> attributes_;
189 tbb::spin_mutex exception_mutex_;
190 std::exception_ptr exception_;
192 tbb::atomic<bool> is_running_;
194 const std::wstring model_name_;
195 const core::video_format_desc format_desc_;
196 const int buffer_size_;
198 long long video_scheduled_;
199 long long audio_scheduled_;
203 boost::circular_buffer<std::vector<int32_t>> audio_container_;
205 tbb::concurrent_bounded_queue<core::const_frame> video_frame_buffer_;
206 tbb::concurrent_bounded_queue<core::const_frame> audio_frame_buffer_;
208 spl::shared_ptr<diagnostics::graph> graph_;
209 boost::timer tick_timer_;
210 retry_task<bool> send_completion_;
213 decklink_consumer(const configuration& config, const core::video_format_desc& format_desc, int channel_index)
214 : channel_index_(channel_index)
216 , decklink_(get_device(config.device_index))
218 , configuration_(decklink_)
220 , attributes_(decklink_)
221 , model_name_(get_model_name(decklink_))
222 , format_desc_(format_desc)
223 , buffer_size_(config.buffer_depth()) // Minimum buffer-size 3.
224 , video_scheduled_(0)
225 , audio_scheduled_(0)
227 , audio_container_(buffer_size_+1)
231 video_frame_buffer_.set_capacity(1);
233 // Blackmagic calls RenderAudioSamples() 50 times per second
234 // regardless of video mode so we sometimes need to give them
235 // samples from 2 frames in order to keep up
236 audio_frame_buffer_.set_capacity((format_desc.fps > 50.0) ? 2 : 1);
238 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
239 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
240 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
241 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
242 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
243 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
244 graph_->set_text(print());
245 diagnostics::register_graph(graph_);
247 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));
249 if(config.embedded_audio)
252 set_latency(config.latency);
253 set_keyer(config.keyer);
255 if(config.embedded_audio)
256 output_->BeginAudioPreroll();
258 for(int n = 0; n < buffer_size_; ++n)
259 schedule_next_video(core::const_frame::empty());
261 if(!config.embedded_audio)
268 video_frame_buffer_.try_push(core::const_frame::empty());
269 audio_frame_buffer_.try_push(core::const_frame::empty());
271 if(output_ != nullptr)
273 output_->StopScheduledPlayback(0, nullptr, 0);
274 if(config_.embedded_audio)
275 output_->DisableAudioOutput();
276 output_->DisableVideoOutput();
280 void set_latency(configuration::latency_t latency)
282 if(latency == configuration::low_latency)
284 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
285 CASPAR_LOG(info) << print() << L" Enabled low-latency mode.";
287 else if(latency == configuration::normal_latency)
289 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
290 CASPAR_LOG(info) << print() << L" Disabled low-latency mode.";
294 void set_keyer(configuration::keyer_t keyer)
296 if(keyer == configuration::internal_keyer)
299 if(SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
300 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
301 else if(FAILED(keyer_->Enable(FALSE)))
302 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
303 else if(FAILED(keyer_->SetLevel(255)))
304 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
306 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";
308 else if(keyer == configuration::external_keyer)
311 if(SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
312 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
313 else if(FAILED(keyer_->Enable(TRUE)))
314 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
315 else if(FAILED(keyer_->SetLevel(255)))
316 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
318 CASPAR_LOG(info) << print() << L" Enabled external keyer.";
324 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped)))
325 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable audio output."));
327 if(FAILED(output_->SetAudioCallback(this)))
328 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not set audio callback."));
330 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
333 void enable_video(BMDDisplayMode display_mode)
335 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
336 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable video output."));
338 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
339 CASPAR_THROW_EXCEPTION(caspar_exception()
340 << msg_info(u8(print()) + " Failed to set playback completion callback.")
341 << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
344 void start_playback()
346 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
347 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Failed to schedule playback."));
350 STDMETHOD (QueryInterface(REFIID, LPVOID*)) {return E_NOINTERFACE;}
351 STDMETHOD_(ULONG, AddRef()) {return 1;}
352 STDMETHOD_(ULONG, Release()) {return 1;}
354 STDMETHOD(ScheduledPlaybackHasStopped())
357 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
361 STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))
368 if(result == bmdOutputFrameDisplayedLate)
370 graph_->set_tag("late-frame");
371 video_scheduled_ += format_desc_.duration;
372 audio_scheduled_ += reinterpret_cast<decklink_frame*>(completed_frame)->audio_data().size()/format_desc_.audio_channels;
373 //++video_scheduled_;
374 //audio_scheduled_ += format_desc_.audio_cadence[0];
375 //++audio_scheduled_;
377 else if(result == bmdOutputFrameDropped)
378 graph_->set_tag("dropped-frame");
379 else if(result == bmdOutputFrameFlushed)
380 graph_->set_tag("flushed-frame");
382 auto frame = core::const_frame::empty();
383 video_frame_buffer_.pop(frame);
384 send_completion_.try_completion();
385 schedule_next_video(frame);
387 unsigned long buffered;
388 output_->GetBufferedVideoFrameCount(&buffered);
389 graph_->set_value("buffered-video", static_cast<double>(buffered)/format_desc_.fps);
393 lock(exception_mutex_, [&]
395 exception_ = std::current_exception();
403 STDMETHOD(RenderAudioSamples(BOOL preroll))
412 if(++preroll_count_ >= buffer_size_)
414 output_->EndAudioPreroll();
419 schedule_next_audio(core::audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()] * format_desc_.audio_channels, 0));
424 auto frame = core::const_frame::empty();
426 while(audio_frame_buffer_.try_pop(frame))
428 send_completion_.try_completion();
429 schedule_next_audio(frame.audio_data());
433 unsigned long buffered;
434 output_->GetBufferedAudioSampleFrameCount(&buffered);
435 graph_->set_value("buffered-audio", static_cast<double>(buffered) / (format_desc_.audio_cadence[0] * format_desc_.audio_channels * 2));
439 tbb::spin_mutex::scoped_lock lock(exception_mutex_);
440 exception_ = std::current_exception();
448 void schedule_next_audio(const T& audio_data)
450 auto sample_frame_count = static_cast<int>(audio_data.size()/format_desc_.audio_channels);
452 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
454 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
455 CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
457 audio_scheduled_ += sample_frame_count;
460 void schedule_next_video(core::const_frame frame)
462 CComPtr<IDeckLinkVideoFrame> frame2(new decklink_frame(frame, format_desc_, config_.key_only));
463 if(FAILED(output_->ScheduleVideoFrame(frame2, video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
464 CASPAR_LOG(error) << print() << L" Failed to schedule video.";
466 video_scheduled_ += format_desc_.duration;
468 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
469 tick_timer_.restart();
472 std::future<bool> send(core::const_frame frame)
474 auto exception = lock(exception_mutex_, [&]
479 if(exception != nullptr)
480 std::rethrow_exception(exception);
483 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Is not running."));
485 bool audio_ready = !config_.embedded_audio;
486 bool video_ready = false;
488 auto enqueue_task = [audio_ready, video_ready, frame, this]() mutable -> boost::optional<bool>
491 audio_ready = audio_frame_buffer_.try_push(frame);
494 video_ready = video_frame_buffer_.try_push(frame);
496 if (audio_ready && video_ready)
499 return boost::optional<bool>();
503 return make_ready_future(true);
505 send_completion_.set_task(enqueue_task);
507 return send_completion_.get_future();
510 std::wstring print() const
512 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_) + L"-" +
513 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
517 struct decklink_consumer_proxy : public core::frame_consumer
519 core::monitor::subject monitor_subject_;
520 const configuration config_;
521 std::unique_ptr<decklink_consumer> consumer_;
525 decklink_consumer_proxy(const configuration& config)
527 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
529 executor_.begin_invoke([=]
531 ::CoInitialize(nullptr);
535 ~decklink_consumer_proxy()
546 void initialize(const core::video_format_desc& format_desc, int channel_index) override
551 consumer_.reset(new decklink_consumer(config_, format_desc, channel_index));
555 std::future<bool> send(core::const_frame frame) override
557 return consumer_->send(frame);
560 std::wstring print() const override
562 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
565 std::wstring name() const override
570 boost::property_tree::wptree info() const override
572 boost::property_tree::wptree info;
573 info.add(L"type", L"decklink");
574 info.add(L"key-only", config_.key_only);
575 info.add(L"device", config_.device_index);
576 info.add(L"low-latency", config_.low_latency);
577 info.add(L"embedded-audio", config_.embedded_audio);
578 info.add(L"low-latency", config_.low_latency);
579 //info.add(L"internal-key", config_.internal_key);
583 int buffer_depth() const override
585 return config_.buffer_depth();
588 int index() const override
590 return 300 + config_.device_index;
593 core::monitor::subject& monitor_output()
595 return monitor_subject_;
599 spl::shared_ptr<core::frame_consumer> create_consumer(const std::vector<std::wstring>& params)
601 if(params.size() < 1 || params[0] != L"DECKLINK")
602 return core::frame_consumer::empty();
604 configuration config;
606 if(params.size() > 1)
607 config.device_index = boost::lexical_cast<int>(params[1]);
609 if(std::find(params.begin(), params.end(), L"INTERNAL_KEY") != params.end())
610 config.keyer = configuration::internal_keyer;
611 else if(std::find(params.begin(), params.end(), L"EXTERNAL_KEY") != params.end())
612 config.keyer = configuration::external_keyer;
614 config.keyer = configuration::default_keyer;
616 if(std::find(params.begin(), params.end(), L"LOW_LATENCY") != params.end())
617 config.latency = configuration::low_latency;
619 config.embedded_audio = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();
620 config.key_only = std::find(params.begin(), params.end(), L"KEY_ONLY") != params.end();
622 return spl::make_shared<decklink_consumer_proxy>(config);
625 spl::shared_ptr<core::frame_consumer> create_consumer(const boost::property_tree::wptree& ptree)
627 configuration config;
629 auto keyer = ptree.get(L"keyer", L"default");
630 if(keyer == L"external")
631 config.keyer = configuration::external_keyer;
632 else if(keyer == L"internal")
633 config.keyer = configuration::internal_keyer;
635 auto latency = ptree.get(L"latency", L"normal");
636 if(latency == L"low")
637 config.latency = configuration::low_latency;
638 else if(latency == L"normal")
639 config.latency = configuration::normal_latency;
641 config.key_only = ptree.get(L"key-only", config.key_only);
642 config.device_index = ptree.get(L"device", config.device_index);
643 config.embedded_audio = ptree.get(L"embedded-audio", config.embedded_audio);
644 config.base_buffer_depth = ptree.get(L"buffer-depth", config.base_buffer_depth);
646 return spl::make_shared<decklink_consumer_proxy>(config);
652 ##############################################################################
658 BMD Developer Support
659 developer@blackmagic-design.com
661 -----------------------------------------------------------------------------
663 Thanks for your inquiry. The minimum number of frames that you can preroll
664 for scheduled playback is three frames for video and four frames for audio.
665 As you mentioned if you preroll less frames then playback will not start or
666 playback will be very sporadic. From our experience with Media Express, we
667 recommended that at least seven frames are prerolled for smooth playback.
669 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
670 There can be around 3 frames worth of latency on scheduled output.
671 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
672 reduced or removed for scheduled playback. If the DisplayVideoFrameSync()
673 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
674 guarantee that the provided frame will be output as soon the previous
675 frame output has been completed.
676 ################################################################################
680 ##############################################################################
681 Async DMA Transfer without redundant copying
686 BMD Developer Support
687 developer@blackmagic-design.com
689 -----------------------------------------------------------------------------
691 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
692 and providing a pointer to your video buffer when GetBytes() is called.
693 This may help to keep copying to a minimum. Please ensure that the pixel
694 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
695 have to colourspace convert which may result in additional copying.
696 ################################################################################