2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
\r
4 * This file is part of CasparCG (www.casparcg.com).
\r
6 * CasparCG is free software: you can redistribute it and/or modify
\r
7 * it under the terms of the GNU General Public License as published by
\r
8 * the Free Software Foundation, either version 3 of the License, or
\r
9 * (at your option) any later version.
\r
11 * CasparCG is distributed in the hope that it will be useful,
\r
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
\r
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
\r
14 * GNU General Public License for more details.
\r
16 * You should have received a copy of the GNU General Public License
\r
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
\r
19 * Author: Robert Nagy, ronag89@gmail.com
\r
22 #include "../StdAfx.h"
\r
24 #include "decklink_consumer.h"
\r
26 #include "../util/util.h"
\r
28 #include "../interop/DeckLinkAPI_h.h"
\r
30 #include <core/frame/data_frame.h>
\r
31 #include <core/mixer/audio/audio_mixer.h>
\r
33 #include <common/concurrency/executor.h>
\r
34 #include <common/concurrency/lock.h>
\r
35 #include <common/diagnostics/graph.h>
\r
36 #include <common/except.h>
\r
37 #include <common/memory/memshfl.h>
\r
39 #include <core/consumer/frame_consumer.h>
\r
41 #include <tbb/concurrent_queue.h>
\r
42 #include <tbb/cache_aligned_allocator.h>
\r
44 #include <common/assert.h>
\r
45 #include <boost/lexical_cast.hpp>
\r
46 #include <boost/circular_buffer.hpp>
\r
47 #include <boost/timer.hpp>
\r
48 #include <boost/property_tree/ptree.hpp>
\r
50 namespace caspar { namespace decklink {
\r
52 struct configuration
\r
69 bool embedded_audio;
\r
73 int base_buffer_depth;
\r
77 , embedded_audio(false)
\r
78 , keyer(default_keyer)
\r
79 , latency(default_latency)
\r
81 , base_buffer_depth(3)
\r
85 int buffer_depth() const
\r
87 return base_buffer_depth + (latency == low_latency ? 0 : 1) + (embedded_audio ? 1 : 0);
\r
91 class decklink_frame : public IDeckLinkVideoFrame
\r
93 tbb::atomic<int> ref_count_;
\r
94 std::shared_ptr<const core::data_frame> frame_;
\r
95 const core::video_format_desc format_desc_;
\r
97 const bool key_only_;
\r
98 std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>> data_;
\r
100 decklink_frame(const spl::shared_ptr<const core::data_frame>& frame, const core::video_format_desc& format_desc, bool key_only)
\r
102 , format_desc_(format_desc)
\r
103 , key_only_(key_only)
\r
110 STDMETHOD (QueryInterface(REFIID, LPVOID*))
\r
112 return E_NOINTERFACE;
\r
115 STDMETHOD_(ULONG, AddRef())
\r
117 return ++ref_count_;
\r
120 STDMETHOD_(ULONG, Release())
\r
122 if(--ref_count_ == 0)
\r
127 // IDecklinkVideoFrame
\r
129 STDMETHOD_(long, GetWidth()) {return format_desc_.width;}
\r
130 STDMETHOD_(long, GetHeight()) {return format_desc_.height;}
\r
131 STDMETHOD_(long, GetRowBytes()) {return format_desc_.width*4;}
\r
132 STDMETHOD_(BMDPixelFormat, GetPixelFormat()) {return bmdFormat8BitBGRA;}
\r
133 STDMETHOD_(BMDFrameFlags, GetFlags()) {return bmdFrameFlagDefault;}
\r
135 STDMETHOD(GetBytes(void** buffer))
\r
139 if(static_cast<int>(frame_->image_data().size()) != format_desc_.size)
\r
141 data_.resize(format_desc_.size, 0);
\r
142 *buffer = data_.data();
\r
148 data_.resize(frame_->image_data().size());
\r
149 aligned_memshfl(data_.data(), frame_->image_data().begin(), frame_->image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
\r
151 *buffer = data_.data();
\r
154 *buffer = const_cast<uint8_t*>(frame_->image_data().begin());
\r
158 CASPAR_LOG_CURRENT_EXCEPTION();
\r
165 STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}
\r
166 STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)) {return S_FALSE;}
\r
170 const core::audio_buffer& audio_data()
\r
172 return frame_->audio_data();
\r
176 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
\r
178 const int channel_index_;
\r
179 const configuration config_;
\r
181 CComPtr<IDeckLink> decklink_;
\r
182 CComQIPtr<IDeckLinkOutput> output_;
\r
183 CComQIPtr<IDeckLinkConfiguration> configuration_;
\r
184 CComQIPtr<IDeckLinkKeyer> keyer_;
\r
185 CComQIPtr<IDeckLinkAttributes> attributes_;
\r
187 tbb::spin_mutex exception_mutex_;
\r
188 std::exception_ptr exception_;
\r
190 tbb::atomic<bool> is_running_;
\r
192 const std::wstring model_name_;
\r
193 const core::video_format_desc format_desc_;
\r
194 const int buffer_size_;
\r
196 long long video_scheduled_;
\r
197 long long audio_scheduled_;
\r
199 int preroll_count_;
\r
201 boost::circular_buffer<std::vector<int32_t>> audio_container_;
\r
203 tbb::concurrent_bounded_queue<std::shared_ptr<const core::data_frame>> video_frame_buffer_;
\r
204 tbb::concurrent_bounded_queue<std::shared_ptr<const core::data_frame>> audio_frame_buffer_;
\r
206 spl::shared_ptr<diagnostics::graph> graph_;
\r
207 boost::timer tick_timer_;
\r
210 decklink_consumer(const configuration& config, const core::video_format_desc& format_desc, int channel_index)
\r
211 : channel_index_(channel_index)
\r
213 , decklink_(get_device(config.device_index))
\r
214 , output_(decklink_)
\r
215 , configuration_(decklink_)
\r
216 , keyer_(decklink_)
\r
217 , attributes_(decklink_)
\r
218 , model_name_(get_model_name(decklink_))
\r
219 , format_desc_(format_desc)
\r
220 , buffer_size_(config.buffer_depth()) // Minimum buffer-size 3.
\r
221 , video_scheduled_(0)
\r
222 , audio_scheduled_(0)
\r
223 , preroll_count_(0)
\r
224 , audio_container_(buffer_size_+1)
\r
226 is_running_ = true;
\r
228 video_frame_buffer_.set_capacity(1);
\r
229 audio_frame_buffer_.set_capacity(1);
\r
231 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
\r
232 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
\r
233 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
\r
234 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
\r
235 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
\r
236 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
\r
237 graph_->set_text(print());
\r
238 diagnostics::register_graph(graph_);
\r
240 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));
\r
242 if(config.embedded_audio)
\r
245 set_latency(config.latency);
\r
246 set_keyer(config.keyer);
\r
248 if(config.embedded_audio)
\r
249 output_->BeginAudioPreroll();
\r
251 for(int n = 0; n < buffer_size_; ++n)
\r
252 schedule_next_video(core::data_frame::empty());
\r
254 if(!config.embedded_audio)
\r
258 ~decklink_consumer()
\r
260 is_running_ = false;
\r
261 video_frame_buffer_.try_push(core::data_frame::empty());
\r
262 audio_frame_buffer_.try_push(core::data_frame::empty());
\r
264 if(output_ != nullptr)
\r
266 output_->StopScheduledPlayback(0, nullptr, 0);
\r
267 if(config_.embedded_audio)
\r
268 output_->DisableAudioOutput();
\r
269 output_->DisableVideoOutput();
\r
273 void set_latency(configuration::latency_t latency)
\r
275 if(latency == configuration::low_latency)
\r
277 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
\r
278 CASPAR_LOG(info) << print() << L" Enabled low-latency mode.";
\r
280 else if(latency == configuration::normal_latency)
\r
282 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
\r
283 CASPAR_LOG(info) << print() << L" Disabled low-latency mode.";
\r
287 void set_keyer(configuration::keyer_t keyer)
\r
289 if(keyer == configuration::internal_keyer)
\r
292 if(SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
\r
293 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
\r
294 else if(FAILED(keyer_->Enable(FALSE)))
\r
295 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
\r
296 else if(FAILED(keyer_->SetLevel(255)))
\r
297 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
299 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";
\r
301 else if(keyer == configuration::external_keyer)
\r
304 if(SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
\r
305 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
\r
306 else if(FAILED(keyer_->Enable(TRUE)))
\r
307 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
\r
308 else if(FAILED(keyer_->SetLevel(255)))
\r
309 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
311 CASPAR_LOG(info) << print() << L" Enabled external keyer.";
\r
315 void enable_audio()
\r
317 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped)))
\r
318 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable audio output."));
\r
320 if(FAILED(output_->SetAudioCallback(this)))
\r
321 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not set audio callback."));
\r
323 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
\r
326 void enable_video(BMDDisplayMode display_mode)
\r
328 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
\r
329 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable video output."));
\r
331 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
\r
332 BOOST_THROW_EXCEPTION(caspar_exception()
\r
333 << msg_info(u8(print()) + " Failed to set playback completion callback.")
\r
334 << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
\r
337 void start_playback()
\r
339 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
\r
340 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Failed to schedule playback."));
\r
343 STDMETHOD (QueryInterface(REFIID, LPVOID*)) {return E_NOINTERFACE;}
\r
344 STDMETHOD_(ULONG, AddRef()) {return 1;}
\r
345 STDMETHOD_(ULONG, Release()) {return 1;}
\r
347 STDMETHOD(ScheduledPlaybackHasStopped())
\r
349 is_running_ = false;
\r
350 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
\r
354 STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))
\r
361 if(result == bmdOutputFrameDisplayedLate)
\r
363 graph_->set_tag("late-frame");
\r
364 video_scheduled_ += format_desc_.duration;
\r
365 audio_scheduled_ += reinterpret_cast<decklink_frame*>(completed_frame)->audio_data().size()/format_desc_.audio_channels;
\r
366 //++video_scheduled_;
\r
367 //audio_scheduled_ += format_desc_.audio_cadence[0];
\r
368 //++audio_scheduled_;
\r
370 else if(result == bmdOutputFrameDropped)
\r
371 graph_->set_tag("dropped-frame");
\r
372 else if(result == bmdOutputFrameFlushed)
\r
373 graph_->set_tag("flushed-frame");
\r
375 std::shared_ptr<const core::data_frame> frame;
\r
376 video_frame_buffer_.pop(frame);
\r
377 schedule_next_video(spl::make_shared_ptr(frame));
\r
379 unsigned long buffered;
\r
380 output_->GetBufferedVideoFrameCount(&buffered);
\r
381 graph_->set_value("buffered-video", static_cast<double>(buffered)/format_desc_.fps);
\r
385 lock(exception_mutex_, [&]
\r
387 exception_ = std::current_exception();
\r
395 STDMETHOD(RenderAudioSamples(BOOL preroll))
\r
404 if(++preroll_count_ >= buffer_size_)
\r
406 output_->EndAudioPreroll();
\r
410 schedule_next_audio(core::audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()], 0));
\r
414 std::shared_ptr<const core::data_frame> frame;
\r
415 audio_frame_buffer_.pop(frame);
\r
416 schedule_next_audio(frame->audio_data());
\r
419 unsigned long buffered;
\r
420 output_->GetBufferedAudioSampleFrameCount(&buffered);
\r
421 graph_->set_value("buffered-audio", static_cast<double>(buffered)/(format_desc_.audio_cadence[0]*2));
\r
425 tbb::spin_mutex::scoped_lock lock(exception_mutex_);
\r
426 exception_ = std::current_exception();
\r
433 template<typename T>
\r
434 void schedule_next_audio(const T& audio_data)
\r
436 auto sample_frame_count = static_cast<int>(audio_data.size()/format_desc_.audio_channels);
\r
438 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
\r
440 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
\r
441 CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
\r
443 audio_scheduled_ += sample_frame_count;
\r
446 void schedule_next_video(const spl::shared_ptr<const core::data_frame>& frame)
\r
448 CComPtr<IDeckLinkVideoFrame> frame2(new decklink_frame(frame, format_desc_, config_.key_only));
\r
449 if(FAILED(output_->ScheduleVideoFrame(frame2, video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
\r
450 CASPAR_LOG(error) << print() << L" Failed to schedule video.";
\r
452 video_scheduled_ += format_desc_.duration;
\r
454 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
\r
455 tick_timer_.restart();
\r
458 void send(const spl::shared_ptr<const core::data_frame>& frame)
\r
460 auto exception = lock(exception_mutex_, [&]
\r
465 if(exception != nullptr)
\r
466 std::rethrow_exception(exception);
\r
469 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Is not running."));
\r
471 if(config_.embedded_audio)
\r
472 audio_frame_buffer_.push(frame);
\r
473 video_frame_buffer_.push(frame);
\r
476 std::wstring print() const
\r
478 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_) + L"-" +
\r
479 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
\r
483 struct decklink_consumer_proxy : public core::frame_consumer
\r
485 const configuration config_;
\r
486 std::unique_ptr<decklink_consumer> consumer_;
\r
487 std::vector<int> audio_cadence_;
\r
488 executor executor_;
\r
491 decklink_consumer_proxy(const configuration& config)
\r
493 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
\r
495 executor_.begin_invoke([=]
\r
497 ::CoInitialize(nullptr);
\r
501 ~decklink_consumer_proxy()
\r
503 executor_.invoke([=]
\r
505 ::CoUninitialize();
\r
511 virtual void initialize(const core::video_format_desc& format_desc, int channel_index) override
\r
513 executor_.invoke([=]
\r
516 consumer_.reset(new decklink_consumer(config_, format_desc, channel_index));
\r
517 audio_cadence_ = format_desc.audio_cadence;
\r
521 virtual bool send(const spl::shared_ptr<const core::data_frame>& frame) override
\r
523 CASPAR_VERIFY(audio_cadence_.front() == static_cast<int>(frame->audio_data().size()));
\r
524 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1);
\r
526 consumer_->send(frame);
\r
530 virtual std::wstring print() const override
\r
532 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
\r
535 virtual std::wstring name() const override
\r
537 return L"decklink";
\r
540 virtual boost::property_tree::wptree info() const override
\r
542 boost::property_tree::wptree info;
\r
543 info.add(L"type", L"decklink");
\r
544 info.add(L"key-only", config_.key_only);
\r
545 info.add(L"device", config_.device_index);
\r
546 info.add(L"low-latency", config_.low_latency);
\r
547 info.add(L"embedded-audio", config_.embedded_audio);
\r
548 info.add(L"low-latency", config_.low_latency);
\r
549 //info.add(L"internal-key", config_.internal_key);
\r
553 virtual int buffer_depth() const override
\r
555 return config_.buffer_depth();
\r
558 virtual int index() const override
\r
560 return 300 + config_.device_index;
\r
564 spl::shared_ptr<core::frame_consumer> create_consumer(const std::vector<std::wstring>& params)
\r
566 if(params.size() < 1 || params[0] != L"DECKLINK")
\r
567 return core::frame_consumer::empty();
\r
569 configuration config;
\r
571 if(params.size() > 1)
\r
572 config.device_index = boost::lexical_cast<int>(params[1]);
\r
574 if(std::find(params.begin(), params.end(), L"INTERNAL_KEY") != params.end())
\r
575 config.keyer = configuration::internal_keyer;
\r
576 else if(std::find(params.begin(), params.end(), L"EXTERNAL_KEY") != params.end())
\r
577 config.keyer = configuration::external_keyer;
\r
579 config.keyer = configuration::default_keyer;
\r
581 if(std::find(params.begin(), params.end(), L"LOW_LATENCY") != params.end())
\r
582 config.latency = configuration::low_latency;
\r
584 config.embedded_audio = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();
\r
585 config.key_only = std::find(params.begin(), params.end(), L"KEY_ONLY") != params.end();
\r
587 return spl::make_shared<decklink_consumer_proxy>(config);
\r
590 spl::shared_ptr<core::frame_consumer> create_consumer(const boost::property_tree::wptree& ptree)
\r
592 configuration config;
\r
594 auto keyer = ptree.get(L"keyer", L"external");
\r
595 if(keyer == L"external")
\r
596 config.keyer = configuration::external_keyer;
\r
597 else if(keyer == L"internal")
\r
598 config.keyer = configuration::internal_keyer;
\r
600 auto latency = ptree.get(L"latency", L"normal");
\r
601 if(latency == L"low")
\r
602 config.latency = configuration::low_latency;
\r
603 else if(latency == L"normal")
\r
604 config.latency = configuration::normal_latency;
\r
606 config.key_only = ptree.get(L"key-only", config.key_only);
\r
607 config.device_index = ptree.get(L"device", config.device_index);
\r
608 config.embedded_audio = ptree.get(L"embedded-audio", config.embedded_audio);
\r
609 config.base_buffer_depth = ptree.get(L"buffer-depth", config.base_buffer_depth);
\r
611 return spl::make_shared<decklink_consumer_proxy>(config);
\r
617 ##############################################################################
\r
623 BMD Developer Support
\r
624 developer@blackmagic-design.com
\r
626 -----------------------------------------------------------------------------
\r
628 Thanks for your inquiry. The minimum number of frames that you can preroll
\r
629 for scheduled playback is three frames for video and four frames for audio.
\r
630 As you mentioned if you preroll less frames then playback will not start or
\r
631 playback will be very sporadic. From our experience with Media Express, we
\r
632 recommended that at least seven frames are prerolled for smooth playback.
\r
634 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
\r
635 There can be around 3 frames worth of latency on scheduled output.
\r
636 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
\r
637 reduced or removed for scheduled playback. If the DisplayVideoFrameSync()
\r
638 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
\r
639 guarantee that the provided frame will be output as soon the previous
\r
640 frame output has been completed.
\r
641 ################################################################################
\r
645 ##############################################################################
\r
646 Async DMA Transfer without redundant copying
\r
651 BMD Developer Support
\r
652 developer@blackmagic-design.com
\r
654 -----------------------------------------------------------------------------
\r
656 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
\r
657 and providing a pointer to your video buffer when GetBytes() is called.
\r
658 This may help to keep copying to a minimum. Please ensure that the pixel
\r
659 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
\r
660 have to colourspace convert which may result in additional copying.
\r
661 ################################################################################
\r