2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Robert Nagy, ronag89@gmail.com
22 #include "../StdAfx.h"
24 #include "decklink_consumer.h"
26 #include "../util/util.h"
28 #include "../decklink_api.h"
30 #include <core/frame/frame.h>
31 #include <core/mixer/audio/audio_mixer.h>
33 #include <common/executor.h>
34 #include <common/lock.h>
35 #include <common/diagnostics/graph.h>
36 #include <common/except.h>
37 #include <common/memshfl.h>
38 #include <common/array.h>
39 #include <common/future.h>
40 #include <common/cache_aligned_vector.h>
42 #include <core/consumer/frame_consumer.h>
43 #include <core/diagnostics/call_context.h>
45 #include <tbb/concurrent_queue.h>
47 #include <common/assert.h>
48 #include <boost/lexical_cast.hpp>
49 #include <boost/circular_buffer.hpp>
50 #include <boost/timer.hpp>
51 #include <boost/property_tree/ptree.hpp>
53 namespace caspar { namespace decklink {
72 bool embedded_audio = true;
73 keyer_t keyer = keyer_t::default_keyer;
74 latency_t latency = latency_t::default_latency;
75 bool key_only = false;
76 int base_buffer_depth = 3;
78 int buffer_depth() const
80 return base_buffer_depth + (latency == latency_t::low_latency ? 0 : 1) + (embedded_audio ? 1 : 0);
84 class decklink_frame : public IDeckLinkVideoFrame
86 tbb::atomic<int> ref_count_;
87 core::const_frame frame_;
88 const core::video_format_desc format_desc_;
91 cache_aligned_vector<uint8_t> data_;
93 decklink_frame(core::const_frame frame, const core::video_format_desc& format_desc, bool key_only)
95 , format_desc_(format_desc)
103 virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)
105 return E_NOINTERFACE;
108 virtual ULONG STDMETHODCALLTYPE AddRef()
113 virtual ULONG STDMETHODCALLTYPE Release()
115 if(--ref_count_ == 0)
120 // IDecklinkVideoFrame
122 virtual long STDMETHODCALLTYPE GetWidth() {return static_cast<long>(format_desc_.width);}
123 virtual long STDMETHODCALLTYPE GetHeight() {return static_cast<long>(format_desc_.height);}
124 virtual long STDMETHODCALLTYPE GetRowBytes() {return static_cast<long>(format_desc_.width*4);}
125 virtual BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat() {return bmdFormat8BitBGRA;}
126 virtual BMDFrameFlags STDMETHODCALLTYPE GetFlags() {return bmdFrameFlagDefault;}
128 virtual HRESULT STDMETHODCALLTYPE GetBytes(void** buffer)
132 if(static_cast<int>(frame_.image_data().size()) != format_desc_.size)
134 data_.resize(format_desc_.size, 0);
135 *buffer = data_.data();
141 data_.resize(frame_.image_data().size());
142 aligned_memshfl(data_.data(), frame_.image_data().begin(), frame_.image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
144 *buffer = data_.data();
147 *buffer = const_cast<uint8_t*>(frame_.image_data().begin());
151 CASPAR_LOG_CURRENT_EXCEPTION();
158 virtual HRESULT STDMETHODCALLTYPE GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode) {return S_FALSE;}
159 virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary) {return S_FALSE;}
163 const core::audio_buffer& audio_data()
165 return frame_.audio_data();
169 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
171 const int channel_index_;
172 const configuration config_;
174 com_ptr<IDeckLink> decklink_ = get_device(config_.device_index);
175 com_iface_ptr<IDeckLinkOutput> output_ = iface_cast<IDeckLinkOutput>(decklink_);
176 com_iface_ptr<IDeckLinkConfiguration> configuration_ = iface_cast<IDeckLinkConfiguration>(decklink_);
177 com_iface_ptr<IDeckLinkKeyer> keyer_ = iface_cast<IDeckLinkKeyer>(decklink_);
178 com_iface_ptr<IDeckLinkAttributes> attributes_ = iface_cast<IDeckLinkAttributes>(decklink_);
180 tbb::spin_mutex exception_mutex_;
181 std::exception_ptr exception_;
183 tbb::atomic<bool> is_running_;
185 const std::wstring model_name_ = get_model_name(decklink_);
186 const core::video_format_desc format_desc_;
187 const int buffer_size_ = config_.buffer_depth(); // Minimum buffer-size 3.
189 long long video_scheduled_ = 0;
190 long long audio_scheduled_ = 0;
192 int preroll_count_ = 0;
194 boost::circular_buffer<std::vector<int32_t>> audio_container_ { buffer_size_ + 1 };
196 tbb::concurrent_bounded_queue<core::const_frame> video_frame_buffer_;
197 tbb::concurrent_bounded_queue<core::const_frame> audio_frame_buffer_;
199 spl::shared_ptr<diagnostics::graph> graph_;
200 boost::timer tick_timer_;
201 retry_task<bool> send_completion_;
204 decklink_consumer(const configuration& config, const core::video_format_desc& format_desc, int channel_index)
205 : channel_index_(channel_index)
207 , format_desc_(format_desc)
211 video_frame_buffer_.set_capacity(1);
213 // Blackmagic calls RenderAudioSamples() 50 times per second
214 // regardless of video mode so we sometimes need to give them
215 // samples from 2 frames in order to keep up
216 audio_frame_buffer_.set_capacity((format_desc.fps > 50.0) ? 2 : 1);
218 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
219 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
220 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
221 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
222 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
223 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
224 graph_->set_text(print());
225 diagnostics::register_graph(graph_);
227 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));
229 if(config.embedded_audio)
232 set_latency(config.latency);
233 set_keyer(config.keyer);
235 if(config.embedded_audio)
236 output_->BeginAudioPreroll();
238 for(int n = 0; n < buffer_size_; ++n)
239 schedule_next_video(core::const_frame::empty());
241 if(!config.embedded_audio)
248 video_frame_buffer_.try_push(core::const_frame::empty());
249 audio_frame_buffer_.try_push(core::const_frame::empty());
251 if(output_ != nullptr)
253 output_->StopScheduledPlayback(0, nullptr, 0);
254 if(config_.embedded_audio)
255 output_->DisableAudioOutput();
256 output_->DisableVideoOutput();
260 void set_latency(configuration::latency_t latency)
262 if(latency == configuration::latency_t::low_latency)
264 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
265 CASPAR_LOG(info) << print() << L" Enabled low-latency mode.";
267 else if(latency == configuration::latency_t::normal_latency)
269 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
270 CASPAR_LOG(info) << print() << L" Disabled low-latency mode.";
274 void set_keyer(configuration::keyer_t keyer)
276 if(keyer == configuration::keyer_t::internal_keyer)
279 if(SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
280 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
281 else if(FAILED(keyer_->Enable(FALSE)))
282 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
283 else if(FAILED(keyer_->SetLevel(255)))
284 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
286 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";
288 else if(keyer == configuration::keyer_t::external_keyer)
291 if(SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
292 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
293 else if(FAILED(keyer_->Enable(TRUE)))
294 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
295 else if(FAILED(keyer_->SetLevel(255)))
296 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
298 CASPAR_LOG(info) << print() << L" Enabled external keyer.";
304 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped)))
305 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable audio output."));
307 if(FAILED(output_->SetAudioCallback(this)))
308 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not set audio callback."));
310 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
313 void enable_video(BMDDisplayMode display_mode)
315 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
316 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable video output."));
318 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
319 CASPAR_THROW_EXCEPTION(caspar_exception()
320 << msg_info(u8(print()) + " Failed to set playback completion callback.")
321 << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
324 void start_playback()
326 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
327 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Failed to schedule playback."));
330 virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*) {return E_NOINTERFACE;}
331 virtual ULONG STDMETHODCALLTYPE AddRef() {return 1;}
332 virtual ULONG STDMETHODCALLTYPE Release() {return 1;}
334 virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
337 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
341 virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result)
348 if(result == bmdOutputFrameDisplayedLate)
350 graph_->set_tag("late-frame");
351 video_scheduled_ += format_desc_.duration;
352 audio_scheduled_ += reinterpret_cast<decklink_frame*>(completed_frame)->audio_data().size()/format_desc_.audio_channels;
353 //++video_scheduled_;
354 //audio_scheduled_ += format_desc_.audio_cadence[0];
355 //++audio_scheduled_;
357 else if(result == bmdOutputFrameDropped)
358 graph_->set_tag("dropped-frame");
359 else if(result == bmdOutputFrameFlushed)
360 graph_->set_tag("flushed-frame");
362 auto frame = core::const_frame::empty();
363 video_frame_buffer_.pop(frame);
364 send_completion_.try_completion();
365 schedule_next_video(frame);
368 output_->GetBufferedVideoFrameCount(&buffered);
369 graph_->set_value("buffered-video", static_cast<double>(buffered)/format_desc_.fps);
373 lock(exception_mutex_, [&]
375 exception_ = std::current_exception();
383 virtual HRESULT STDMETHODCALLTYPE RenderAudioSamples(BOOL preroll)
392 if(++preroll_count_ >= buffer_size_)
394 output_->EndAudioPreroll();
399 schedule_next_audio(core::audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()] * format_desc_.audio_channels, 0));
404 auto frame = core::const_frame::empty();
406 while(audio_frame_buffer_.try_pop(frame))
408 send_completion_.try_completion();
409 schedule_next_audio(frame.audio_data());
414 output_->GetBufferedAudioSampleFrameCount(&buffered);
415 graph_->set_value("buffered-audio", static_cast<double>(buffered) / (format_desc_.audio_cadence[0] * format_desc_.audio_channels * 2));
419 tbb::spin_mutex::scoped_lock lock(exception_mutex_);
420 exception_ = std::current_exception();
428 void schedule_next_audio(const T& audio_data)
430 auto sample_frame_count = static_cast<int>(audio_data.size()/format_desc_.audio_channels);
432 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
434 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
435 CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
437 audio_scheduled_ += sample_frame_count;
440 void schedule_next_video(core::const_frame frame)
442 auto frame2 = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, config_.key_only));
443 if(FAILED(output_->ScheduleVideoFrame(get_raw(frame2), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
444 CASPAR_LOG(error) << print() << L" Failed to schedule video.";
446 video_scheduled_ += format_desc_.duration;
448 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
449 tick_timer_.restart();
452 std::future<bool> send(core::const_frame frame)
454 auto exception = lock(exception_mutex_, [&]
459 if(exception != nullptr)
460 std::rethrow_exception(exception);
463 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Is not running."));
465 bool audio_ready = !config_.embedded_audio;
466 bool video_ready = false;
468 auto enqueue_task = [audio_ready, video_ready, frame, this]() mutable -> boost::optional<bool>
471 audio_ready = audio_frame_buffer_.try_push(frame);
474 video_ready = video_frame_buffer_.try_push(frame);
476 if (audio_ready && video_ready)
479 return boost::optional<bool>();
483 return make_ready_future(true);
485 send_completion_.set_task(enqueue_task);
487 return send_completion_.get_future();
490 std::wstring print() const
492 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_) + L"-" +
493 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
497 struct decklink_consumer_proxy : public core::frame_consumer
499 core::monitor::subject monitor_subject_;
500 const configuration config_;
501 std::unique_ptr<decklink_consumer> consumer_;
505 decklink_consumer_proxy(const configuration& config)
507 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
509 auto ctx = core::diagnostics::call_context::for_thread();
510 executor_.begin_invoke([=]
512 core::diagnostics::call_context::for_thread() = ctx;
517 ~decklink_consumer_proxy()
528 void initialize(const core::video_format_desc& format_desc, int channel_index) override
533 consumer_.reset(new decklink_consumer(config_, format_desc, channel_index));
537 std::future<bool> send(core::const_frame frame) override
539 return consumer_->send(frame);
542 std::wstring print() const override
544 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
547 std::wstring name() const override
552 boost::property_tree::wptree info() const override
554 boost::property_tree::wptree info;
555 info.add(L"type", L"decklink");
556 info.add(L"key-only", config_.key_only);
557 info.add(L"device", config_.device_index);
558 info.add(L"low-latency", config_.latency == configuration::latency_t::low_latency);
559 info.add(L"embedded-audio", config_.embedded_audio);
560 //info.add(L"internal-key", config_.internal_key);
564 int buffer_depth() const override
566 return config_.buffer_depth();
569 int index() const override
571 return 300 + config_.device_index;
574 core::monitor::subject& monitor_output()
576 return monitor_subject_;
580 spl::shared_ptr<core::frame_consumer> create_consumer(
581 const std::vector<std::wstring>& params, core::interaction_sink*)
583 if(params.size() < 1 || params[0] != L"DECKLINK")
584 return core::frame_consumer::empty();
586 configuration config;
588 if(params.size() > 1)
589 config.device_index = boost::lexical_cast<int>(params[1]);
591 if(std::find(params.begin(), params.end(), L"INTERNAL_KEY") != params.end())
592 config.keyer = configuration::keyer_t::internal_keyer;
593 else if(std::find(params.begin(), params.end(), L"EXTERNAL_KEY") != params.end())
594 config.keyer = configuration::keyer_t::external_keyer;
596 config.keyer = configuration::keyer_t::default_keyer;
598 if(std::find(params.begin(), params.end(), L"LOW_LATENCY") != params.end())
599 config.latency = configuration::latency_t::low_latency;
601 config.embedded_audio = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();
602 config.key_only = std::find(params.begin(), params.end(), L"KEY_ONLY") != params.end();
604 return spl::make_shared<decklink_consumer_proxy>(config);
607 spl::shared_ptr<core::frame_consumer> create_preconfigured_consumer(
608 const boost::property_tree::wptree& ptree, core::interaction_sink*)
610 configuration config;
612 auto keyer = ptree.get(L"keyer", L"default");
613 if(keyer == L"external")
614 config.keyer = configuration::keyer_t::external_keyer;
615 else if(keyer == L"internal")
616 config.keyer = configuration::keyer_t::internal_keyer;
618 auto latency = ptree.get(L"latency", L"normal");
619 if(latency == L"low")
620 config.latency = configuration::latency_t::low_latency;
621 else if(latency == L"normal")
622 config.latency = configuration::latency_t::normal_latency;
624 config.key_only = ptree.get(L"key-only", config.key_only);
625 config.device_index = ptree.get(L"device", config.device_index);
626 config.embedded_audio = ptree.get(L"embedded-audio", config.embedded_audio);
627 config.base_buffer_depth = ptree.get(L"buffer-depth", config.base_buffer_depth);
629 return spl::make_shared<decklink_consumer_proxy>(config);
635 ##############################################################################
641 BMD Developer Support
642 developer@blackmagic-design.com
644 -----------------------------------------------------------------------------
646 Thanks for your inquiry. The minimum number of frames that you can preroll
647 for scheduled playback is three frames for video and four frames for audio.
648 As you mentioned if you preroll less frames then playback will not start or
649 playback will be very sporadic. From our experience with Media Express, we
650 recommended that at least seven frames are prerolled for smooth playback.
652 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
653 There can be around 3 frames worth of latency on scheduled output.
654 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
655 reduced or removed for scheduled playback. If the DisplayVideoFrameSync()
656 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
657 guarantee that the provided frame will be output as soon the previous
658 frame output has been completed.
659 ################################################################################
663 ##############################################################################
664 Async DMA Transfer without redundant copying
669 BMD Developer Support
670 developer@blackmagic-design.com
672 -----------------------------------------------------------------------------
674 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
675 and providing a pointer to your video buffer when GetBytes() is called.
676 This may help to keep copying to a minimum. Please ensure that the pixel
677 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
678 have to colourspace convert which may result in additional copying.
679 ################################################################################