2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Robert Nagy, ronag89@gmail.com
22 #include "../StdAfx.h"
24 #include "decklink_consumer.h"
26 #include "../util/util.h"
28 #include "../interop/DeckLinkAPI_h.h"
30 #include <core/frame/frame.h>
31 #include <core/mixer/audio/audio_mixer.h>
33 #include <common/executor.h>
34 #include <common/lock.h>
35 #include <common/diagnostics/graph.h>
36 #include <common/except.h>
37 #include <common/memshfl.h>
38 #include <common/array.h>
39 #include <common/future.h>
41 #include <core/consumer/frame_consumer.h>
43 #include <tbb/concurrent_queue.h>
44 #include <tbb/cache_aligned_allocator.h>
46 #include <common/assert.h>
47 #include <boost/lexical_cast.hpp>
48 #include <boost/circular_buffer.hpp>
49 #include <boost/timer.hpp>
50 #include <boost/property_tree/ptree.hpp>
52 namespace caspar { namespace decklink {
75 int base_buffer_depth;
79 , embedded_audio(true)
80 , keyer(default_keyer)
81 , latency(default_latency)
83 , base_buffer_depth(3)
87 int buffer_depth() const
89 return base_buffer_depth + (latency == low_latency ? 0 : 1) + (embedded_audio ? 1 : 0);
93 class decklink_frame : public IDeckLinkVideoFrame
95 tbb::atomic<int> ref_count_;
96 core::const_frame frame_;
97 const core::video_format_desc format_desc_;
100 std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>> data_;
102 decklink_frame(core::const_frame frame, const core::video_format_desc& format_desc, bool key_only)
104 , format_desc_(format_desc)
105 , key_only_(key_only)
112 STDMETHOD (QueryInterface(REFIID, LPVOID*))
114 return E_NOINTERFACE;
117 STDMETHOD_(ULONG, AddRef())
122 STDMETHOD_(ULONG, Release())
124 if(--ref_count_ == 0)
129 // IDecklinkVideoFrame
131 STDMETHOD_(long, GetWidth()) {return static_cast<long>(format_desc_.width);}
132 STDMETHOD_(long, GetHeight()) {return static_cast<long>(format_desc_.height);}
133 STDMETHOD_(long, GetRowBytes()) {return static_cast<long>(format_desc_.width*4);}
134 STDMETHOD_(BMDPixelFormat, GetPixelFormat()) {return bmdFormat8BitBGRA;}
135 STDMETHOD_(BMDFrameFlags, GetFlags()) {return bmdFrameFlagDefault;}
137 STDMETHOD(GetBytes(void** buffer))
141 if(static_cast<int>(frame_.image_data().size()) != format_desc_.size)
143 data_.resize(format_desc_.size, 0);
144 *buffer = data_.data();
150 data_.resize(frame_.image_data().size());
151 aligned_memshfl(data_.data(), frame_.image_data().begin(), frame_.image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
153 *buffer = data_.data();
156 *buffer = const_cast<uint8_t*>(frame_.image_data().begin());
160 CASPAR_LOG_CURRENT_EXCEPTION();
167 STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}
168 STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)) {return S_FALSE;}
172 const core::audio_buffer& audio_data()
174 return frame_.audio_data();
178 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
180 const int channel_index_;
181 const configuration config_;
183 CComPtr<IDeckLink> decklink_;
184 CComQIPtr<IDeckLinkOutput> output_;
185 CComQIPtr<IDeckLinkConfiguration> configuration_;
186 CComQIPtr<IDeckLinkKeyer> keyer_;
187 CComQIPtr<IDeckLinkAttributes> attributes_;
189 tbb::spin_mutex exception_mutex_;
190 std::exception_ptr exception_;
192 tbb::atomic<bool> is_running_;
194 const std::wstring model_name_;
195 const core::video_format_desc format_desc_;
196 const int buffer_size_;
198 long long video_scheduled_;
199 long long audio_scheduled_;
203 boost::circular_buffer<std::vector<int32_t>> audio_container_;
205 tbb::concurrent_bounded_queue<core::const_frame> video_frame_buffer_;
206 tbb::concurrent_bounded_queue<core::const_frame> audio_frame_buffer_;
208 spl::shared_ptr<diagnostics::graph> graph_;
209 boost::timer tick_timer_;
210 retry_task<bool> send_completion_;
213 decklink_consumer(const configuration& config, const core::video_format_desc& format_desc, int channel_index)
214 : channel_index_(channel_index)
216 , decklink_(get_device(config.device_index))
218 , configuration_(decklink_)
220 , attributes_(decklink_)
221 , model_name_(get_model_name(decklink_))
222 , format_desc_(format_desc)
223 , buffer_size_(config.buffer_depth()) // Minimum buffer-size 3.
224 , video_scheduled_(0)
225 , audio_scheduled_(0)
227 , audio_container_(buffer_size_+1)
231 video_frame_buffer_.set_capacity(1);
233 // Blackmagic calls RenderAudioSamples() 50 times per second
234 // regardless of video mode so we sometimes need to give them
235 // samples from 2 frames in order to keep up
236 audio_frame_buffer_.set_capacity((format_desc.fps > 50.0) ? 2 : 1);
238 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
239 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
240 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
241 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
242 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
243 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
244 graph_->set_text(print());
245 diagnostics::register_graph(graph_);
247 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));
249 if(config.embedded_audio)
252 set_latency(config.latency);
253 set_keyer(config.keyer);
255 if(config.embedded_audio)
256 output_->BeginAudioPreroll();
258 for(int n = 0; n < buffer_size_; ++n)
259 schedule_next_video(core::const_frame::empty());
261 if(!config.embedded_audio)
268 video_frame_buffer_.try_push(core::const_frame::empty());
269 audio_frame_buffer_.try_push(core::const_frame::empty());
271 if(output_ != nullptr)
273 output_->StopScheduledPlayback(0, nullptr, 0);
274 if(config_.embedded_audio)
275 output_->DisableAudioOutput();
276 output_->DisableVideoOutput();
280 void set_latency(configuration::latency_t latency)
282 if(latency == configuration::low_latency)
284 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
285 CASPAR_LOG(info) << print() << L" Enabled low-latency mode.";
287 else if(latency == configuration::normal_latency)
289 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
290 CASPAR_LOG(info) << print() << L" Disabled low-latency mode.";
294 void set_keyer(configuration::keyer_t keyer)
296 if(keyer == configuration::internal_keyer)
299 if(SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
300 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
301 else if(FAILED(keyer_->Enable(FALSE)))
302 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
303 else if(FAILED(keyer_->SetLevel(255)))
304 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
306 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";
308 else if(keyer == configuration::external_keyer)
311 if(SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
312 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
313 else if(FAILED(keyer_->Enable(TRUE)))
314 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
315 else if(FAILED(keyer_->SetLevel(255)))
316 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
318 CASPAR_LOG(info) << print() << L" Enabled external keyer.";
324 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped)))
325 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable audio output."));
327 if(FAILED(output_->SetAudioCallback(this)))
328 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not set audio callback."));
330 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
333 void enable_video(BMDDisplayMode display_mode)
335 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
336 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable video output."));
338 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
339 CASPAR_THROW_EXCEPTION(caspar_exception()
340 << msg_info(u8(print()) + " Failed to set playback completion callback.")
341 << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
344 void start_playback()
346 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
347 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Failed to schedule playback."));
350 STDMETHOD (QueryInterface(REFIID, LPVOID*)) {return E_NOINTERFACE;}
351 STDMETHOD_(ULONG, AddRef()) {return 1;}
352 STDMETHOD_(ULONG, Release()) {return 1;}
354 STDMETHOD(ScheduledPlaybackHasStopped())
357 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
361 STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))
368 if(result == bmdOutputFrameDisplayedLate)
370 graph_->set_tag("late-frame");
371 video_scheduled_ += format_desc_.duration;
372 audio_scheduled_ += reinterpret_cast<decklink_frame*>(completed_frame)->audio_data().size()/format_desc_.audio_channels;
373 //++video_scheduled_;
374 //audio_scheduled_ += format_desc_.audio_cadence[0];
375 //++audio_scheduled_;
377 else if(result == bmdOutputFrameDropped)
378 graph_->set_tag("dropped-frame");
379 else if(result == bmdOutputFrameFlushed)
380 graph_->set_tag("flushed-frame");
382 auto frame = core::const_frame::empty();
383 video_frame_buffer_.pop(frame);
384 send_completion_.try_completion();
385 schedule_next_video(frame);
387 unsigned long buffered;
388 output_->GetBufferedVideoFrameCount(&buffered);
389 graph_->set_value("buffered-video", static_cast<double>(buffered)/format_desc_.fps);
393 lock(exception_mutex_, [&]
395 exception_ = std::current_exception();
403 STDMETHOD(RenderAudioSamples(BOOL preroll))
412 if(++preroll_count_ >= buffer_size_)
414 output_->EndAudioPreroll();
419 schedule_next_audio(core::audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()] * format_desc_.audio_channels, 0));
424 auto frame = core::const_frame::empty();
426 while(audio_frame_buffer_.try_pop(frame))
428 send_completion_.try_completion();
429 schedule_next_audio(frame.audio_data());
433 unsigned long buffered;
434 output_->GetBufferedAudioSampleFrameCount(&buffered);
435 graph_->set_value("buffered-audio", static_cast<double>(buffered)/(format_desc_.audio_cadence[0]*format_desc_.audio_channels*2));
439 tbb::spin_mutex::scoped_lock lock(exception_mutex_);
440 exception_ = std::current_exception();
448 void schedule_next_audio(const T& audio_data)
450 auto sample_frame_count = static_cast<int>(audio_data.size()/format_desc_.audio_channels);
452 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
454 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
455 CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
457 audio_scheduled_ += sample_frame_count;
460 void schedule_next_video(core::const_frame frame)
462 CComPtr<IDeckLinkVideoFrame> frame2(new decklink_frame(frame, format_desc_, config_.key_only));
463 if(FAILED(output_->ScheduleVideoFrame(frame2, video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
464 CASPAR_LOG(error) << print() << L" Failed to schedule video.";
466 video_scheduled_ += format_desc_.duration;
468 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
469 tick_timer_.restart();
472 boost::unique_future<bool> send(core::const_frame frame)
474 auto exception = lock(exception_mutex_, [&]
479 if(exception != nullptr)
480 std::rethrow_exception(exception);
483 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Is not running."));
485 bool audio_ready = !config_.embedded_audio;
486 bool video_ready = false;
488 auto enqueue_task = [audio_ready, video_ready, frame, this]() mutable -> boost::optional<bool>
491 audio_ready = audio_frame_buffer_.try_push(frame);
494 video_ready = video_frame_buffer_.try_push(frame);
496 if (audio_ready && video_ready)
499 return boost::optional<bool>();
503 return wrap_as_future(true);
505 send_completion_.set_task(enqueue_task);
507 return send_completion_.get_future();
510 std::wstring print() const
512 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_) + L"-" +
513 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
517 struct decklink_consumer_proxy : public core::frame_consumer
519 const configuration config_;
520 std::unique_ptr<decklink_consumer> consumer_;
524 decklink_consumer_proxy(const configuration& config)
526 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
528 executor_.begin_invoke([=]
530 ::CoInitialize(nullptr);
534 ~decklink_consumer_proxy()
545 void initialize(const core::video_format_desc& format_desc, int channel_index) override
550 consumer_.reset(new decklink_consumer(config_, format_desc, channel_index));
554 boost::unique_future<bool> send(core::const_frame frame) override
556 return consumer_->send(frame);
559 std::wstring print() const override
561 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
564 std::wstring name() const override
569 boost::property_tree::wptree info() const override
571 boost::property_tree::wptree info;
572 info.add(L"type", L"decklink");
573 info.add(L"key-only", config_.key_only);
574 info.add(L"device", config_.device_index);
575 info.add(L"low-latency", config_.low_latency);
576 info.add(L"embedded-audio", config_.embedded_audio);
577 info.add(L"low-latency", config_.low_latency);
578 //info.add(L"internal-key", config_.internal_key);
582 int buffer_depth() const override
584 return config_.buffer_depth();
587 int index() const override
589 return 300 + config_.device_index;
592 monitor::source& monitor_output()
594 static monitor::subject monitor_subject(""); return monitor_subject;
598 spl::shared_ptr<core::frame_consumer> create_consumer(const std::vector<std::wstring>& params)
600 if(params.size() < 1 || params[0] != L"DECKLINK")
601 return core::frame_consumer::empty();
603 configuration config;
605 if(params.size() > 1)
606 config.device_index = boost::lexical_cast<int>(params[1]);
608 if(std::find(params.begin(), params.end(), L"INTERNAL_KEY") != params.end())
609 config.keyer = configuration::internal_keyer;
610 else if(std::find(params.begin(), params.end(), L"EXTERNAL_KEY") != params.end())
611 config.keyer = configuration::external_keyer;
613 config.keyer = configuration::default_keyer;
615 if(std::find(params.begin(), params.end(), L"LOW_LATENCY") != params.end())
616 config.latency = configuration::low_latency;
618 config.embedded_audio = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();
619 config.key_only = std::find(params.begin(), params.end(), L"KEY_ONLY") != params.end();
621 return spl::make_shared<decklink_consumer_proxy>(config);
624 spl::shared_ptr<core::frame_consumer> create_consumer(const boost::property_tree::wptree& ptree)
626 configuration config;
628 auto keyer = ptree.get(L"keyer", L"default");
629 if(keyer == L"external")
630 config.keyer = configuration::external_keyer;
631 else if(keyer == L"internal")
632 config.keyer = configuration::internal_keyer;
634 auto latency = ptree.get(L"latency", L"normal");
635 if(latency == L"low")
636 config.latency = configuration::low_latency;
637 else if(latency == L"normal")
638 config.latency = configuration::normal_latency;
640 config.key_only = ptree.get(L"key-only", config.key_only);
641 config.device_index = ptree.get(L"device", config.device_index);
642 config.embedded_audio = ptree.get(L"embedded-audio", config.embedded_audio);
643 config.base_buffer_depth = ptree.get(L"buffer-depth", config.base_buffer_depth);
645 return spl::make_shared<decklink_consumer_proxy>(config);
651 ##############################################################################
657 BMD Developer Support
658 developer@blackmagic-design.com
660 -----------------------------------------------------------------------------
662 Thanks for your inquiry. The minimum number of frames that you can preroll
663 for scheduled playback is three frames for video and four frames for audio.
664 As you mentioned if you preroll less frames then playback will not start or
665 playback will be very sporadic. From our experience with Media Express, we
666 recommended that at least seven frames are prerolled for smooth playback.
668 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
669 There can be around 3 frames worth of latency on scheduled output.
670 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
671 reduced or removed for scheduled playback. If the DisplayVideoFrameSync()
672 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
673 guarantee that the provided frame will be output as soon the previous
674 frame output has been completed.
675 ################################################################################
679 ##############################################################################
680 Async DMA Transfer without redundant copying
685 BMD Developer Support
686 developer@blackmagic-design.com
688 -----------------------------------------------------------------------------
690 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
691 and providing a pointer to your video buffer when GetBytes() is called.
692 This may help to keep copying to a minimum. Please ensure that the pixel
693 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
694 have to colourspace convert which may result in additional copying.
695 ################################################################################