2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
\r
4 * This file is part of CasparCG (www.casparcg.com).
\r
6 * CasparCG is free software: you can redistribute it and/or modify
\r
7 * it under the terms of the GNU General Public License as published by
\r
8 * the Free Software Foundation, either version 3 of the License, or
\r
9 * (at your option) any later version.
\r
11 * CasparCG is distributed in the hope that it will be useful,
\r
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
\r
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
\r
14 * GNU General Public License for more details.
\r
16 * You should have received a copy of the GNU General Public License
\r
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
\r
19 * Author: Robert Nagy, ronag89@gmail.com
\r
22 #include "../StdAfx.h"
\r
24 #include "decklink_consumer.h"
\r
26 #include "../util/util.h"
\r
28 #include "../interop/DeckLinkAPI_h.h"
\r
30 #include <core/frame/data_frame.h>
\r
31 #include <core/mixer/audio/audio_mixer.h>
\r
33 #include <common/concurrency/executor.h>
\r
34 #include <common/concurrency/lock.h>
\r
35 #include <common/diagnostics/graph.h>
\r
36 #include <common/except.h>
\r
37 #include <common/memory/memshfl.h>
\r
39 #include <core/consumer/frame_consumer.h>
\r
41 #include <tbb/concurrent_queue.h>
\r
42 #include <tbb/cache_aligned_allocator.h>
\r
44 #include <common/assert.h>
\r
45 #include <boost/lexical_cast.hpp>
\r
46 #include <boost/circular_buffer.hpp>
\r
47 #include <boost/timer.hpp>
\r
48 #include <boost/property_tree/ptree.hpp>
\r
50 namespace caspar { namespace decklink {
\r
52 struct configuration
\r
69 bool embedded_audio;
\r
73 int base_buffer_depth;
\r
77 , embedded_audio(false)
\r
78 , keyer(default_keyer)
\r
79 , latency(default_latency)
\r
81 , base_buffer_depth(3)
\r
85 int buffer_depth() const
\r
87 return base_buffer_depth + (latency == low_latency ? 0 : 1) + (embedded_audio ? 1 : 0);
\r
91 class decklink_frame : public IDeckLinkVideoFrame
\r
93 tbb::atomic<int> ref_count_;
\r
94 std::shared_ptr<const core::data_frame> frame_;
\r
95 const core::video_format_desc format_desc_;
\r
97 const bool key_only_;
\r
98 std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>> data_;
\r
100 decklink_frame(const safe_ptr<const core::data_frame>& frame, const core::video_format_desc& format_desc, bool key_only)
\r
102 , format_desc_(format_desc)
\r
103 , key_only_(key_only)
\r
110 STDMETHOD (QueryInterface(REFIID, LPVOID*))
\r
112 return E_NOINTERFACE;
\r
115 STDMETHOD_(ULONG, AddRef())
\r
117 return ++ref_count_;
\r
120 STDMETHOD_(ULONG, Release())
\r
122 if(--ref_count_ == 0)
\r
127 // IDecklinkVideoFrame
\r
129 STDMETHOD_(long, GetWidth()) {return format_desc_.width;}
\r
130 STDMETHOD_(long, GetHeight()) {return format_desc_.height;}
\r
131 STDMETHOD_(long, GetRowBytes()) {return format_desc_.width*4;}
\r
132 STDMETHOD_(BMDPixelFormat, GetPixelFormat()) {return bmdFormat8BitBGRA;}
\r
133 STDMETHOD_(BMDFrameFlags, GetFlags()) {return bmdFrameFlagDefault;}
\r
135 STDMETHOD(GetBytes(void** buffer))
\r
139 if(static_cast<int>(frame_->image_data().size()) != format_desc_.size)
\r
141 data_.resize(format_desc_.size, 0);
\r
142 *buffer = data_.data();
\r
148 data_.resize(frame_->image_data().size());
\r
149 aligned_memshfl(data_.data(), frame_->image_data().begin(), frame_->image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
\r
151 *buffer = data_.data();
\r
154 *buffer = const_cast<uint8_t*>(frame_->image_data().begin());
\r
158 CASPAR_LOG_CURRENT_EXCEPTION();
\r
165 STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}
\r
166 STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)) {return S_FALSE;}
\r
170 const core::audio_buffer& audio_data()
\r
172 return frame_->audio_data();
\r
176 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
\r
178 const int channel_index_;
\r
179 const configuration config_;
\r
181 CComPtr<IDeckLink> decklink_;
\r
182 CComQIPtr<IDeckLinkOutput> output_;
\r
183 CComQIPtr<IDeckLinkConfiguration> configuration_;
\r
184 CComQIPtr<IDeckLinkKeyer> keyer_;
\r
185 CComQIPtr<IDeckLinkAttributes> attributes_;
\r
187 tbb::spin_mutex exception_mutex_;
\r
188 std::exception_ptr exception_;
\r
190 tbb::atomic<bool> is_running_;
\r
192 const std::wstring model_name_;
\r
193 const core::video_format_desc format_desc_;
\r
194 const int buffer_size_;
\r
196 long long video_scheduled_;
\r
197 long long audio_scheduled_;
\r
199 int preroll_count_;
\r
201 boost::circular_buffer<std::vector<int32_t>> audio_container_;
\r
203 tbb::concurrent_bounded_queue<std::shared_ptr<const core::data_frame>> video_frame_buffer_;
\r
204 tbb::concurrent_bounded_queue<std::shared_ptr<const core::data_frame>> audio_frame_buffer_;
\r
206 safe_ptr<diagnostics::graph> graph_;
\r
207 boost::timer tick_timer_;
\r
210 decklink_consumer(const configuration& config, const core::video_format_desc& format_desc, int channel_index)
\r
211 : channel_index_(channel_index)
\r
213 , decklink_(get_device(config.device_index))
\r
214 , output_(decklink_)
\r
215 , configuration_(decklink_)
\r
216 , keyer_(decklink_)
\r
217 , attributes_(decklink_)
\r
218 , model_name_(get_model_name(decklink_))
\r
219 , format_desc_(format_desc)
\r
220 , buffer_size_(config.buffer_depth()) // Minimum buffer-size 3.
\r
221 , video_scheduled_(0)
\r
222 , audio_scheduled_(0)
\r
223 , preroll_count_(0)
\r
224 , audio_container_(buffer_size_+1)
\r
226 is_running_ = true;
\r
228 video_frame_buffer_.set_capacity(1);
\r
229 audio_frame_buffer_.set_capacity(1);
\r
231 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
\r
232 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
\r
233 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
\r
234 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
\r
235 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
\r
236 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
\r
237 graph_->set_text(print());
\r
238 diagnostics::register_graph(graph_);
\r
240 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));
\r
242 if(config.embedded_audio)
\r
245 set_latency(config.latency);
\r
246 set_keyer(config.keyer);
\r
248 if(config.embedded_audio)
\r
249 output_->BeginAudioPreroll();
\r
251 for(int n = 0; n < buffer_size_; ++n)
\r
252 schedule_next_video(core::data_frame::empty());
\r
254 if(!config.embedded_audio)
\r
258 ~decklink_consumer()
\r
260 is_running_ = false;
\r
261 video_frame_buffer_.try_push(core::data_frame::empty());
\r
262 audio_frame_buffer_.try_push(core::data_frame::empty());
\r
264 if(output_ != nullptr)
\r
266 output_->StopScheduledPlayback(0, nullptr, 0);
\r
267 if(config_.embedded_audio)
\r
268 output_->DisableAudioOutput();
\r
269 output_->DisableVideoOutput();
\r
273 void set_latency(configuration::latency_t latency)
\r
275 if(latency == configuration::low_latency)
\r
277 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
\r
278 CASPAR_LOG(info) << print() << L" Enabled low-latency mode.";
\r
280 else if(latency == configuration::normal_latency)
\r
282 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
\r
283 CASPAR_LOG(info) << print() << L" Disabled low-latency mode.";
\r
287 void set_keyer(configuration::keyer_t keyer)
\r
289 if(keyer == configuration::internal_keyer)
\r
292 if(SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
\r
293 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
\r
294 else if(FAILED(keyer_->Enable(FALSE)))
\r
295 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
\r
296 else if(FAILED(keyer_->SetLevel(255)))
\r
297 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
299 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";
\r
301 else if(keyer == configuration::external_keyer)
\r
304 if(SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
\r
305 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
\r
306 else if(FAILED(keyer_->Enable(TRUE)))
\r
307 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
\r
308 else if(FAILED(keyer_->SetLevel(255)))
\r
309 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
311 CASPAR_LOG(info) << print() << L" Enabled external keyer.";
\r
315 void enable_audio()
\r
317 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped)))
\r
318 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable audio output."));
\r
320 if(FAILED(output_->SetAudioCallback(this)))
\r
321 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not set audio callback."));
\r
323 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
\r
326 void enable_video(BMDDisplayMode display_mode)
\r
328 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
\r
329 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable video output."));
\r
331 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
\r
332 BOOST_THROW_EXCEPTION(caspar_exception()
\r
333 << msg_info(u8(print()) + " Failed to set playback completion callback.")
\r
334 << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
\r
337 void start_playback()
\r
339 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
\r
340 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Failed to schedule playback."));
\r
343 STDMETHOD (QueryInterface(REFIID, LPVOID*)) {return E_NOINTERFACE;}
\r
344 STDMETHOD_(ULONG, AddRef()) {return 1;}
\r
345 STDMETHOD_(ULONG, Release()) {return 1;}
\r
347 STDMETHOD(ScheduledPlaybackHasStopped())
\r
349 is_running_ = false;
\r
350 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
\r
354 STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))
\r
361 if(result == bmdOutputFrameDisplayedLate)
\r
363 graph_->set_tag("late-frame");
\r
364 video_scheduled_ += format_desc_.duration;
\r
365 audio_scheduled_ += reinterpret_cast<decklink_frame*>(completed_frame)->audio_data().size()/format_desc_.audio_channels;
\r
366 //++video_scheduled_;
\r
367 //audio_scheduled_ += format_desc_.audio_cadence[0];
\r
368 //++audio_scheduled_;
\r
370 else if(result == bmdOutputFrameDropped)
\r
371 graph_->set_tag("dropped-frame");
\r
372 else if(result == bmdOutputFrameFlushed)
\r
373 graph_->set_tag("flushed-frame");
\r
375 std::shared_ptr<const core::data_frame> frame;
\r
376 video_frame_buffer_.pop(frame);
\r
377 schedule_next_video(make_safe_ptr(frame));
\r
379 unsigned long buffered;
\r
380 output_->GetBufferedVideoFrameCount(&buffered);
\r
381 graph_->set_value("buffered-video", static_cast<double>(buffered)/format_desc_.fps);
\r
385 lock(exception_mutex_, [&]
\r
387 exception_ = std::current_exception();
\r
395 STDMETHOD(RenderAudioSamples(BOOL preroll))
\r
404 if(++preroll_count_ >= buffer_size_)
\r
406 output_->EndAudioPreroll();
\r
410 schedule_next_audio(core::audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()], 0));
\r
414 std::shared_ptr<const core::data_frame> frame;
\r
415 audio_frame_buffer_.pop(frame);
\r
416 schedule_next_audio(frame->audio_data());
\r
419 unsigned long buffered;
\r
420 output_->GetBufferedAudioSampleFrameCount(&buffered);
\r
421 graph_->set_value("buffered-audio", static_cast<double>(buffered)/(format_desc_.audio_cadence[0]*2));
\r
425 tbb::spin_mutex::scoped_lock lock(exception_mutex_);
\r
426 exception_ = std::current_exception();
\r
433 template<typename T>
\r
434 void schedule_next_audio(const T& audio_data)
\r
436 auto sample_frame_count = static_cast<int>(audio_data.size()/format_desc_.audio_channels);
\r
438 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
\r
440 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
\r
441 CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
\r
443 audio_scheduled_ += sample_frame_count;
\r
446 void schedule_next_video(const safe_ptr<const core::data_frame>& frame)
\r
448 CComPtr<IDeckLinkVideoFrame> frame2(new decklink_frame(frame, format_desc_, config_.key_only));
\r
449 if(FAILED(output_->ScheduleVideoFrame(frame2, video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
\r
450 CASPAR_LOG(error) << print() << L" Failed to schedule video.";
\r
452 video_scheduled_ += format_desc_.duration;
\r
454 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
\r
455 tick_timer_.restart();
\r
458 void send(const safe_ptr<const core::data_frame>& frame)
\r
460 auto exception = lock(exception_mutex_, [&]
\r
465 if(exception != nullptr)
\r
466 std::rethrow_exception(exception);
\r
469 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Is not running."));
\r
471 if(config_.embedded_audio)
\r
472 audio_frame_buffer_.push(frame);
\r
473 video_frame_buffer_.push(frame);
\r
476 std::wstring print() const
\r
478 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_) + L"-" +
\r
479 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
\r
483 struct decklink_consumer_proxy : public core::frame_consumer
\r
485 const configuration config_;
\r
486 std::unique_ptr<decklink_consumer> consumer_;
\r
487 std::vector<int> audio_cadence_;
\r
488 executor executor_;
\r
491 decklink_consumer_proxy(const configuration& config)
\r
493 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
\r
495 executor_.begin_invoke([=]
\r
497 ::CoInitialize(nullptr);
\r
501 ~decklink_consumer_proxy()
\r
503 executor_.invoke([=]
\r
507 auto str = print();
\r
509 CASPAR_LOG(info) << str << L" Successfully Uninitialized.";
\r
511 ::CoUninitialize();
\r
517 virtual void initialize(const core::video_format_desc& format_desc, int channel_index) override
\r
519 executor_.invoke([=]
\r
521 consumer_.reset(new decklink_consumer(config_, format_desc, channel_index));
\r
522 audio_cadence_ = format_desc.audio_cadence;
\r
524 CASPAR_LOG(info) << print() << L" Successfully Initialized.";
\r
528 virtual bool send(const safe_ptr<const core::data_frame>& frame) override
\r
530 CASPAR_VERIFY(audio_cadence_.front() == static_cast<int>(frame->audio_data().size()));
\r
531 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1);
\r
533 consumer_->send(frame);
\r
537 virtual std::wstring print() const override
\r
539 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
\r
542 virtual boost::property_tree::wptree info() const override
\r
544 boost::property_tree::wptree info;
\r
545 info.add(L"type", L"decklink-consumer");
\r
546 info.add(L"key-only", config_.key_only);
\r
547 info.add(L"device", config_.device_index);
\r
548 info.add(L"low-latency", config_.low_latency);
\r
549 info.add(L"embedded-audio", config_.embedded_audio);
\r
550 info.add(L"low-latency", config_.low_latency);
\r
551 //info.add(L"internal-key", config_.internal_key);
\r
555 virtual int buffer_depth() const override
\r
557 return config_.buffer_depth();
\r
560 virtual int index() const override
\r
562 return 300 + config_.device_index;
\r
566 safe_ptr<core::frame_consumer> create_consumer(const std::vector<std::wstring>& params)
\r
568 if(params.size() < 1 || params[0] != L"DECKLINK")
\r
569 return core::frame_consumer::empty();
\r
571 configuration config;
\r
573 if(params.size() > 1)
\r
574 config.device_index = boost::lexical_cast<int>(params[1]);
\r
576 if(std::find(params.begin(), params.end(), L"INTERNAL_KEY") != params.end())
\r
577 config.keyer = configuration::internal_keyer;
\r
578 else if(std::find(params.begin(), params.end(), L"EXTERNAL_KEY") != params.end())
\r
579 config.keyer = configuration::external_keyer;
\r
581 config.keyer = configuration::default_keyer;
\r
583 if(std::find(params.begin(), params.end(), L"LOW_LATENCY") != params.end())
\r
584 config.latency = configuration::low_latency;
\r
586 config.embedded_audio = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();
\r
587 config.key_only = std::find(params.begin(), params.end(), L"KEY_ONLY") != params.end();
\r
589 return make_safe<decklink_consumer_proxy>(config);
\r
592 safe_ptr<core::frame_consumer> create_consumer(const boost::property_tree::wptree& ptree)
\r
594 configuration config;
\r
596 auto keyer = ptree.get(L"keyer", L"external");
\r
597 if(keyer == L"external")
\r
598 config.keyer = configuration::external_keyer;
\r
599 else if(keyer == L"internal")
\r
600 config.keyer = configuration::internal_keyer;
\r
602 auto latency = ptree.get(L"latency", L"normal");
\r
603 if(latency == L"low")
\r
604 config.latency = configuration::low_latency;
\r
605 else if(latency == L"normal")
\r
606 config.latency = configuration::normal_latency;
\r
608 config.key_only = ptree.get(L"key-only", config.key_only);
\r
609 config.device_index = ptree.get(L"device", config.device_index);
\r
610 config.embedded_audio = ptree.get(L"embedded-audio", config.embedded_audio);
\r
611 config.base_buffer_depth = ptree.get(L"buffer-depth", config.base_buffer_depth);
\r
613 return make_safe<decklink_consumer_proxy>(config);
\r
619 ##############################################################################
\r
625 BMD Developer Support
\r
626 developer@blackmagic-design.com
\r
628 -----------------------------------------------------------------------------
\r
630 Thanks for your inquiry. The minimum number of frames that you can preroll
\r
631 for scheduled playback is three frames for video and four frames for audio.
\r
632 As you mentioned if you preroll less frames then playback will not start or
\r
633 playback will be very sporadic. From our experience with Media Express, we
\r
634 recommended that at least seven frames are prerolled for smooth playback.
\r
636 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
\r
637 There can be around 3 frames worth of latency on scheduled output.
\r
638 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
\r
639 reduced or removed for scheduled playback. If the DisplayVideoFrameSync()
\r
640 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
\r
641 guarantee that the provided frame will be output as soon the previous
\r
642 frame output has been completed.
\r
643 ################################################################################
\r
647 ##############################################################################
\r
648 Async DMA Transfer without redundant copying
\r
653 BMD Developer Support
\r
654 developer@blackmagic-design.com
\r
656 -----------------------------------------------------------------------------
\r
658 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
\r
659 and providing a pointer to your video buffer when GetBytes() is called.
\r
660 This may help to keep copying to a minimum. Please ensure that the pixel
\r
661 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
\r
662 have to colourspace convert which may result in additional copying.
\r
663 ################################################################################
\r