2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
\r
4 * This file is part of CasparCG (www.casparcg.com).
\r
6 * CasparCG is free software: you can redistribute it and/or modify
\r
7 * it under the terms of the GNU General Public License as published by
\r
8 * the Free Software Foundation, either version 3 of the License, or
\r
9 * (at your option) any later version.
\r
11 * CasparCG is distributed in the hope that it will be useful,
\r
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
\r
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
\r
14 * GNU General Public License for more details.
\r
16 * You should have received a copy of the GNU General Public License
\r
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
\r
19 * Author: Robert Nagy, ronag89@gmail.com
\r
22 #include "../StdAfx.h"
\r
24 #include "decklink_consumer.h"
\r
26 #include "../util/util.h"
\r
28 #include "../interop/DeckLinkAPI_h.h"
\r
30 #include <core/frame/frame.h>
\r
31 #include <core/mixer/audio/audio_mixer.h>
\r
33 #include <common/concurrency/executor.h>
\r
34 #include <common/concurrency/lock.h>
\r
35 #include <common/diagnostics/graph.h>
\r
36 #include <common/except.h>
\r
37 #include <common/memory/memshfl.h>
\r
38 #include <common/memory/array.h>
\r
40 #include <core/consumer/frame_consumer.h>
\r
42 #include <tbb/concurrent_queue.h>
\r
43 #include <tbb/cache_aligned_allocator.h>
\r
45 #include <common/assert.h>
\r
46 #include <boost/lexical_cast.hpp>
\r
47 #include <boost/circular_buffer.hpp>
\r
48 #include <boost/timer.hpp>
\r
49 #include <boost/property_tree/ptree.hpp>
\r
51 namespace caspar { namespace decklink {
\r
53 struct configuration
\r
70 bool embedded_audio;
\r
74 int base_buffer_depth;
\r
78 , embedded_audio(false)
\r
79 , keyer(default_keyer)
\r
80 , latency(default_latency)
\r
82 , base_buffer_depth(3)
\r
86 int buffer_depth() const
\r
88 return base_buffer_depth + (latency == low_latency ? 0 : 1) + (embedded_audio ? 1 : 0);
\r
92 class decklink_frame : public IDeckLinkVideoFrame
\r
94 tbb::atomic<int> ref_count_;
\r
95 core::const_frame frame_;
\r
96 const core::video_format_desc format_desc_;
\r
98 const bool key_only_;
\r
99 std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>> data_;
\r
101 decklink_frame(core::const_frame frame, const core::video_format_desc& format_desc, bool key_only)
\r
103 , format_desc_(format_desc)
\r
104 , key_only_(key_only)
\r
111 STDMETHOD (QueryInterface(REFIID, LPVOID*))
\r
113 return E_NOINTERFACE;
\r
116 STDMETHOD_(ULONG, AddRef())
\r
118 return ++ref_count_;
\r
121 STDMETHOD_(ULONG, Release())
\r
123 if(--ref_count_ == 0)
\r
128 // IDecklinkVideoFrame
\r
130 STDMETHOD_(long, GetWidth()) {return static_cast<long>(format_desc_.width);}
\r
131 STDMETHOD_(long, GetHeight()) {return static_cast<long>(format_desc_.height);}
\r
132 STDMETHOD_(long, GetRowBytes()) {return static_cast<long>(format_desc_.width*4);}
\r
133 STDMETHOD_(BMDPixelFormat, GetPixelFormat()) {return bmdFormat8BitBGRA;}
\r
134 STDMETHOD_(BMDFrameFlags, GetFlags()) {return bmdFrameFlagDefault;}
\r
136 STDMETHOD(GetBytes(void** buffer))
\r
140 if(static_cast<int>(frame_.image_data().size()) != format_desc_.size)
\r
142 data_.resize(format_desc_.size, 0);
\r
143 *buffer = data_.data();
\r
149 data_.resize(frame_.image_data().size());
\r
150 aligned_memshfl(data_.data(), frame_.image_data().begin(), frame_.image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
\r
152 *buffer = data_.data();
\r
155 *buffer = const_cast<uint8_t*>(frame_.image_data().begin());
\r
159 CASPAR_LOG_CURRENT_EXCEPTION();
\r
166 STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}
\r
167 STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)) {return S_FALSE;}
\r
171 const core::audio_buffer& audio_data()
\r
173 return frame_.audio_data();
\r
177 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
\r
179 const int channel_index_;
\r
180 const configuration config_;
\r
182 CComPtr<IDeckLink> decklink_;
\r
183 CComQIPtr<IDeckLinkOutput> output_;
\r
184 CComQIPtr<IDeckLinkConfiguration> configuration_;
\r
185 CComQIPtr<IDeckLinkKeyer> keyer_;
\r
186 CComQIPtr<IDeckLinkAttributes> attributes_;
\r
188 tbb::spin_mutex exception_mutex_;
\r
189 std::exception_ptr exception_;
\r
191 tbb::atomic<bool> is_running_;
\r
193 const std::wstring model_name_;
\r
194 const core::video_format_desc format_desc_;
\r
195 const int buffer_size_;
\r
197 long long video_scheduled_;
\r
198 long long audio_scheduled_;
\r
200 int preroll_count_;
\r
202 boost::circular_buffer<std::vector<int32_t>> audio_container_;
\r
204 tbb::concurrent_bounded_queue<core::const_frame> video_frame_buffer_;
\r
205 tbb::concurrent_bounded_queue<core::const_frame> audio_frame_buffer_;
\r
207 spl::shared_ptr<diagnostics::graph> graph_;
\r
208 boost::timer tick_timer_;
\r
211 decklink_consumer(const configuration& config, const core::video_format_desc& format_desc, int channel_index)
\r
212 : channel_index_(channel_index)
\r
214 , decklink_(get_device(config.device_index))
\r
215 , output_(decklink_)
\r
216 , configuration_(decklink_)
\r
217 , keyer_(decklink_)
\r
218 , attributes_(decklink_)
\r
219 , model_name_(get_model_name(decklink_))
\r
220 , format_desc_(format_desc)
\r
221 , buffer_size_(config.buffer_depth()) // Minimum buffer-size 3.
\r
222 , video_scheduled_(0)
\r
223 , audio_scheduled_(0)
\r
224 , preroll_count_(0)
\r
225 , audio_container_(buffer_size_+1)
\r
227 is_running_ = true;
\r
229 video_frame_buffer_.set_capacity(1);
\r
230 audio_frame_buffer_.set_capacity(1);
\r
232 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
\r
233 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
\r
234 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
\r
235 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
\r
236 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
\r
237 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
\r
238 graph_->set_text(print());
\r
239 diagnostics::register_graph(graph_);
\r
241 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));
\r
243 if(config.embedded_audio)
\r
246 set_latency(config.latency);
\r
247 set_keyer(config.keyer);
\r
249 if(config.embedded_audio)
\r
250 output_->BeginAudioPreroll();
\r
252 for(int n = 0; n < buffer_size_; ++n)
\r
253 schedule_next_video(core::const_frame::empty());
\r
255 if(!config.embedded_audio)
\r
259 ~decklink_consumer()
\r
261 is_running_ = false;
\r
262 video_frame_buffer_.try_push(core::const_frame::empty());
\r
263 audio_frame_buffer_.try_push(core::const_frame::empty());
\r
265 if(output_ != nullptr)
\r
267 output_->StopScheduledPlayback(0, nullptr, 0);
\r
268 if(config_.embedded_audio)
\r
269 output_->DisableAudioOutput();
\r
270 output_->DisableVideoOutput();
\r
274 void set_latency(configuration::latency_t latency)
\r
276 if(latency == configuration::low_latency)
\r
278 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
\r
279 CASPAR_LOG(info) << print() << L" Enabled low-latency mode.";
\r
281 else if(latency == configuration::normal_latency)
\r
283 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
\r
284 CASPAR_LOG(info) << print() << L" Disabled low-latency mode.";
\r
288 void set_keyer(configuration::keyer_t keyer)
\r
290 if(keyer == configuration::internal_keyer)
\r
293 if(SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
\r
294 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
\r
295 else if(FAILED(keyer_->Enable(FALSE)))
\r
296 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
\r
297 else if(FAILED(keyer_->SetLevel(255)))
\r
298 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
300 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";
\r
302 else if(keyer == configuration::external_keyer)
\r
305 if(SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
\r
306 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
\r
307 else if(FAILED(keyer_->Enable(TRUE)))
\r
308 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
\r
309 else if(FAILED(keyer_->SetLevel(255)))
\r
310 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
312 CASPAR_LOG(info) << print() << L" Enabled external keyer.";
\r
316 void enable_audio()
\r
318 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped)))
\r
319 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable audio output."));
\r
321 if(FAILED(output_->SetAudioCallback(this)))
\r
322 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not set audio callback."));
\r
324 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
\r
327 void enable_video(BMDDisplayMode display_mode)
\r
329 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
\r
330 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable video output."));
\r
332 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
\r
333 BOOST_THROW_EXCEPTION(caspar_exception()
\r
334 << msg_info(u8(print()) + " Failed to set playback completion callback.")
\r
335 << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
\r
338 void start_playback()
\r
340 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
\r
341 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Failed to schedule playback."));
\r
344 STDMETHOD (QueryInterface(REFIID, LPVOID*)) {return E_NOINTERFACE;}
\r
345 STDMETHOD_(ULONG, AddRef()) {return 1;}
\r
346 STDMETHOD_(ULONG, Release()) {return 1;}
\r
348 STDMETHOD(ScheduledPlaybackHasStopped())
\r
350 is_running_ = false;
\r
351 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
\r
355 STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))
\r
362 if(result == bmdOutputFrameDisplayedLate)
\r
364 graph_->set_tag("late-frame");
\r
365 video_scheduled_ += format_desc_.duration;
\r
366 audio_scheduled_ += reinterpret_cast<decklink_frame*>(completed_frame)->audio_data().size()/format_desc_.audio_channels;
\r
367 //++video_scheduled_;
\r
368 //audio_scheduled_ += format_desc_.audio_cadence[0];
\r
369 //++audio_scheduled_;
\r
371 else if(result == bmdOutputFrameDropped)
\r
372 graph_->set_tag("dropped-frame");
\r
373 else if(result == bmdOutputFrameFlushed)
\r
374 graph_->set_tag("flushed-frame");
\r
376 auto frame = core::const_frame::empty();
\r
377 video_frame_buffer_.pop(frame);
\r
378 schedule_next_video(frame);
\r
380 unsigned long buffered;
\r
381 output_->GetBufferedVideoFrameCount(&buffered);
\r
382 graph_->set_value("buffered-video", static_cast<double>(buffered)/format_desc_.fps);
\r
386 lock(exception_mutex_, [&]
\r
388 exception_ = std::current_exception();
\r
396 STDMETHOD(RenderAudioSamples(BOOL preroll))
\r
405 if(++preroll_count_ >= buffer_size_)
\r
407 output_->EndAudioPreroll();
\r
411 schedule_next_audio(core::audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()], 0));
\r
415 auto frame = core::const_frame::empty();
\r
416 audio_frame_buffer_.pop(frame);
\r
417 schedule_next_audio(frame.audio_data());
\r
420 unsigned long buffered;
\r
421 output_->GetBufferedAudioSampleFrameCount(&buffered);
\r
422 graph_->set_value("buffered-audio", static_cast<double>(buffered)/(format_desc_.audio_cadence[0]*2));
\r
426 tbb::spin_mutex::scoped_lock lock(exception_mutex_);
\r
427 exception_ = std::current_exception();
\r
434 template<typename T>
\r
435 void schedule_next_audio(const T& audio_data)
\r
437 auto sample_frame_count = static_cast<int>(audio_data.size()/format_desc_.audio_channels);
\r
439 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
\r
441 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
\r
442 CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
\r
444 audio_scheduled_ += sample_frame_count;
\r
447 void schedule_next_video(core::const_frame frame)
\r
449 CComPtr<IDeckLinkVideoFrame> frame2(new decklink_frame(frame, format_desc_, config_.key_only));
\r
450 if(FAILED(output_->ScheduleVideoFrame(frame2, video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
\r
451 CASPAR_LOG(error) << print() << L" Failed to schedule video.";
\r
453 video_scheduled_ += format_desc_.duration;
\r
455 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
\r
456 tick_timer_.restart();
\r
459 void send(core::const_frame frame)
\r
461 auto exception = lock(exception_mutex_, [&]
\r
466 if(exception != nullptr)
\r
467 std::rethrow_exception(exception);
\r
470 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Is not running."));
\r
472 if(config_.embedded_audio)
\r
473 audio_frame_buffer_.push(frame);
\r
474 video_frame_buffer_.push(frame);
\r
477 std::wstring print() const
\r
479 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_) + L"-" +
\r
480 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
\r
484 struct decklink_consumer_proxy : public core::frame_consumer
\r
486 const configuration config_;
\r
487 std::unique_ptr<decklink_consumer> consumer_;
\r
488 executor executor_;
\r
491 decklink_consumer_proxy(const configuration& config)
\r
493 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
\r
495 executor_.begin_invoke([=]
\r
497 ::CoInitialize(nullptr);
\r
501 ~decklink_consumer_proxy()
\r
503 executor_.invoke([=]
\r
505 ::CoUninitialize();
\r
511 virtual void initialize(const core::video_format_desc& format_desc, int channel_index) override
\r
513 executor_.invoke([=]
\r
516 consumer_.reset(new decklink_consumer(config_, format_desc, channel_index));
\r
520 virtual bool send(core::const_frame frame) override
\r
522 consumer_->send(frame);
\r
526 virtual std::wstring print() const override
\r
528 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
\r
531 virtual std::wstring name() const override
\r
533 return L"decklink";
\r
536 virtual boost::property_tree::wptree info() const override
\r
538 boost::property_tree::wptree info;
\r
539 info.add(L"type", L"decklink");
\r
540 info.add(L"key-only", config_.key_only);
\r
541 info.add(L"device", config_.device_index);
\r
542 info.add(L"low-latency", config_.low_latency);
\r
543 info.add(L"embedded-audio", config_.embedded_audio);
\r
544 info.add(L"low-latency", config_.low_latency);
\r
545 //info.add(L"internal-key", config_.internal_key);
\r
549 virtual int buffer_depth() const override
\r
551 return config_.buffer_depth();
\r
554 virtual int index() const override
\r
556 return 300 + config_.device_index;
\r
560 spl::shared_ptr<core::frame_consumer> create_consumer(const std::vector<std::wstring>& params)
\r
562 if(params.size() < 1 || params[0] != L"DECKLINK")
\r
563 return core::frame_consumer::empty();
\r
565 configuration config;
\r
567 if(params.size() > 1)
\r
568 config.device_index = boost::lexical_cast<int>(params[1]);
\r
570 if(std::find(params.begin(), params.end(), L"INTERNAL_KEY") != params.end())
\r
571 config.keyer = configuration::internal_keyer;
\r
572 else if(std::find(params.begin(), params.end(), L"EXTERNAL_KEY") != params.end())
\r
573 config.keyer = configuration::external_keyer;
\r
575 config.keyer = configuration::default_keyer;
\r
577 if(std::find(params.begin(), params.end(), L"LOW_LATENCY") != params.end())
\r
578 config.latency = configuration::low_latency;
\r
580 config.embedded_audio = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();
\r
581 config.key_only = std::find(params.begin(), params.end(), L"KEY_ONLY") != params.end();
\r
583 return spl::make_shared<decklink_consumer_proxy>(config);
\r
586 spl::shared_ptr<core::frame_consumer> create_consumer(const boost::property_tree::wptree& ptree)
\r
588 configuration config;
\r
590 auto keyer = ptree.get(L"keyer", L"external");
\r
591 if(keyer == L"external")
\r
592 config.keyer = configuration::external_keyer;
\r
593 else if(keyer == L"internal")
\r
594 config.keyer = configuration::internal_keyer;
\r
596 auto latency = ptree.get(L"latency", L"normal");
\r
597 if(latency == L"low")
\r
598 config.latency = configuration::low_latency;
\r
599 else if(latency == L"normal")
\r
600 config.latency = configuration::normal_latency;
\r
602 config.key_only = ptree.get(L"key-only", config.key_only);
\r
603 config.device_index = ptree.get(L"device", config.device_index);
\r
604 config.embedded_audio = ptree.get(L"embedded-audio", config.embedded_audio);
\r
605 config.base_buffer_depth = ptree.get(L"buffer-depth", config.base_buffer_depth);
\r
607 return spl::make_shared<decklink_consumer_proxy>(config);
\r
613 ##############################################################################
\r
619 BMD Developer Support
\r
620 developer@blackmagic-design.com
\r
622 -----------------------------------------------------------------------------
\r
624 Thanks for your inquiry. The minimum number of frames that you can preroll
\r
625 for scheduled playback is three frames for video and four frames for audio.
\r
626 As you mentioned if you preroll less frames then playback will not start or
\r
627 playback will be very sporadic. From our experience with Media Express, we
\r
628 recommended that at least seven frames are prerolled for smooth playback.
\r
630 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
\r
631 There can be around 3 frames worth of latency on scheduled output.
\r
632 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
\r
633 reduced or removed for scheduled playback. If the DisplayVideoFrameSync()
\r
634 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
\r
635 guarantee that the provided frame will be output as soon the previous
\r
636 frame output has been completed.
\r
637 ################################################################################
\r
641 ##############################################################################
\r
642 Async DMA Transfer without redundant copying
\r
647 BMD Developer Support
\r
648 developer@blackmagic-design.com
\r
650 -----------------------------------------------------------------------------
\r
652 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
\r
653 and providing a pointer to your video buffer when GetBytes() is called.
\r
654 This may help to keep copying to a minimum. Please ensure that the pixel
\r
655 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
\r
656 have to colourspace convert which may result in additional copying.
\r
657 ################################################################################
\r