2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
\r
4 * This file is part of CasparCG (www.casparcg.com).
\r
6 * CasparCG is free software: you can redistribute it and/or modify
\r
7 * it under the terms of the GNU General Public License as published by
\r
8 * the Free Software Foundation, either version 3 of the License, or
\r
9 * (at your option) any later version.
\r
11 * CasparCG is distributed in the hope that it will be useful,
\r
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
\r
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
\r
14 * GNU General Public License for more details.
\r
16 * You should have received a copy of the GNU General Public License
\r
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
\r
19 * Author: Robert Nagy, ronag89@gmail.com
\r
22 #include "../StdAfx.h"
\r
24 #include "decklink_consumer.h"
\r
26 #include "../util/util.h"
\r
28 #include "../interop/DeckLinkAPI_h.h"
\r
30 #include <core/frame/data_frame.h>
\r
31 #include <core/mixer/audio/audio_mixer.h>
\r
33 #include <common/concurrency/executor.h>
\r
34 #include <common/concurrency/lock.h>
\r
35 #include <common/diagnostics/graph.h>
\r
36 #include <common/except.h>
\r
37 #include <common/memory/memshfl.h>
\r
39 #include <core/consumer/frame_consumer.h>
\r
41 #include <tbb/concurrent_queue.h>
\r
42 #include <tbb/cache_aligned_allocator.h>
\r
44 #include <common/assert.h>
\r
45 #include <boost/lexical_cast.hpp>
\r
46 #include <boost/circular_buffer.hpp>
\r
47 #include <boost/timer.hpp>
\r
48 #include <boost/property_tree/ptree.hpp>
\r
50 namespace caspar { namespace decklink {
\r
52 struct configuration
\r
69 bool embedded_audio;
\r
73 int base_buffer_depth;
\r
77 , embedded_audio(false)
\r
78 , keyer(default_keyer)
\r
79 , latency(default_latency)
\r
81 , base_buffer_depth(3)
\r
85 int buffer_depth() const
\r
87 return base_buffer_depth + (latency == low_latency ? 0 : 1) + (embedded_audio ? 1 : 0);
\r
91 class decklink_frame : public IDeckLinkVideoFrame
\r
93 tbb::atomic<int> ref_count_;
\r
94 std::shared_ptr<const core::data_frame> frame_;
\r
95 const core::video_format_desc format_desc_;
\r
97 const bool key_only_;
\r
98 std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>> data_;
\r
100 decklink_frame(const spl::shared_ptr<const core::data_frame>& frame, const core::video_format_desc& format_desc, bool key_only)
\r
102 , format_desc_(format_desc)
\r
103 , key_only_(key_only)
\r
110 STDMETHOD (QueryInterface(REFIID, LPVOID*))
\r
112 return E_NOINTERFACE;
\r
115 STDMETHOD_(ULONG, AddRef())
\r
117 return ++ref_count_;
\r
120 STDMETHOD_(ULONG, Release())
\r
122 if(--ref_count_ == 0)
\r
127 // IDecklinkVideoFrame
\r
129 STDMETHOD_(long, GetWidth()) {return format_desc_.width;}
\r
130 STDMETHOD_(long, GetHeight()) {return format_desc_.height;}
\r
131 STDMETHOD_(long, GetRowBytes()) {return format_desc_.width*4;}
\r
132 STDMETHOD_(BMDPixelFormat, GetPixelFormat()) {return bmdFormat8BitBGRA;}
\r
133 STDMETHOD_(BMDFrameFlags, GetFlags()) {return bmdFrameFlagDefault;}
\r
135 STDMETHOD(GetBytes(void** buffer))
\r
139 if(static_cast<int>(frame_->image_data().size()) != format_desc_.size)
\r
141 data_.resize(format_desc_.size, 0);
\r
142 *buffer = data_.data();
\r
148 data_.resize(frame_->image_data().size());
\r
149 aligned_memshfl(data_.data(), frame_->image_data().begin(), frame_->image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
\r
151 *buffer = data_.data();
\r
154 *buffer = const_cast<uint8_t*>(frame_->image_data().begin());
\r
158 CASPAR_LOG_CURRENT_EXCEPTION();
\r
165 STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}
\r
166 STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)) {return S_FALSE;}
\r
170 const core::audio_buffer& audio_data()
\r
172 return frame_->audio_data();
\r
176 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
\r
178 const int channel_index_;
\r
179 const configuration config_;
\r
181 CComPtr<IDeckLink> decklink_;
\r
182 CComQIPtr<IDeckLinkOutput> output_;
\r
183 CComQIPtr<IDeckLinkConfiguration> configuration_;
\r
184 CComQIPtr<IDeckLinkKeyer> keyer_;
\r
185 CComQIPtr<IDeckLinkAttributes> attributes_;
\r
187 tbb::spin_mutex exception_mutex_;
\r
188 std::exception_ptr exception_;
\r
190 tbb::atomic<bool> is_running_;
\r
192 const std::wstring model_name_;
\r
193 const core::video_format_desc format_desc_;
\r
194 const int buffer_size_;
\r
196 long long video_scheduled_;
\r
197 long long audio_scheduled_;
\r
199 int preroll_count_;
\r
201 boost::circular_buffer<std::vector<int32_t>> audio_container_;
\r
203 tbb::concurrent_bounded_queue<std::shared_ptr<const core::data_frame>> video_frame_buffer_;
\r
204 tbb::concurrent_bounded_queue<std::shared_ptr<const core::data_frame>> audio_frame_buffer_;
\r
206 spl::shared_ptr<diagnostics::graph> graph_;
\r
207 boost::timer tick_timer_;
\r
210 decklink_consumer(const configuration& config, const core::video_format_desc& format_desc, int channel_index)
\r
211 : channel_index_(channel_index)
\r
213 , decklink_(get_device(config.device_index))
\r
214 , output_(decklink_)
\r
215 , configuration_(decklink_)
\r
216 , keyer_(decklink_)
\r
217 , attributes_(decklink_)
\r
218 , model_name_(get_model_name(decklink_))
\r
219 , format_desc_(format_desc)
\r
220 , buffer_size_(config.buffer_depth()) // Minimum buffer-size 3.
\r
221 , video_scheduled_(0)
\r
222 , audio_scheduled_(0)
\r
223 , preroll_count_(0)
\r
224 , audio_container_(buffer_size_+1)
\r
226 is_running_ = true;
\r
228 video_frame_buffer_.set_capacity(1);
\r
229 audio_frame_buffer_.set_capacity(1);
\r
231 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
\r
232 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
\r
233 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
\r
234 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
\r
235 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
\r
236 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
\r
237 graph_->set_text(print());
\r
238 diagnostics::register_graph(graph_);
\r
240 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));
\r
242 if(config.embedded_audio)
\r
245 set_latency(config.latency);
\r
246 set_keyer(config.keyer);
\r
248 if(config.embedded_audio)
\r
249 output_->BeginAudioPreroll();
\r
251 for(int n = 0; n < buffer_size_; ++n)
\r
252 schedule_next_video(core::data_frame::empty());
\r
254 if(!config.embedded_audio)
\r
258 ~decklink_consumer()
\r
260 is_running_ = false;
\r
261 video_frame_buffer_.try_push(core::data_frame::empty());
\r
262 audio_frame_buffer_.try_push(core::data_frame::empty());
\r
264 if(output_ != nullptr)
\r
266 output_->StopScheduledPlayback(0, nullptr, 0);
\r
267 if(config_.embedded_audio)
\r
268 output_->DisableAudioOutput();
\r
269 output_->DisableVideoOutput();
\r
273 void set_latency(configuration::latency_t latency)
\r
275 if(latency == configuration::low_latency)
\r
277 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
\r
278 CASPAR_LOG(info) << print() << L" Enabled low-latency mode.";
\r
280 else if(latency == configuration::normal_latency)
\r
282 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
\r
283 CASPAR_LOG(info) << print() << L" Disabled low-latency mode.";
\r
287 void set_keyer(configuration::keyer_t keyer)
\r
289 if(keyer == configuration::internal_keyer)
\r
292 if(SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
\r
293 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
\r
294 else if(FAILED(keyer_->Enable(FALSE)))
\r
295 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
\r
296 else if(FAILED(keyer_->SetLevel(255)))
\r
297 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
299 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";
\r
301 else if(keyer == configuration::external_keyer)
\r
304 if(SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
\r
305 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
\r
306 else if(FAILED(keyer_->Enable(TRUE)))
\r
307 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
\r
308 else if(FAILED(keyer_->SetLevel(255)))
\r
309 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
311 CASPAR_LOG(info) << print() << L" Enabled external keyer.";
\r
315 void enable_audio()
\r
317 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped)))
\r
318 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable audio output."));
\r
320 if(FAILED(output_->SetAudioCallback(this)))
\r
321 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not set audio callback."));
\r
323 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
\r
326 void enable_video(BMDDisplayMode display_mode)
\r
328 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
\r
329 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable video output."));
\r
331 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
\r
332 BOOST_THROW_EXCEPTION(caspar_exception()
\r
333 << msg_info(u8(print()) + " Failed to set playback completion callback.")
\r
334 << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
\r
337 void start_playback()
\r
339 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
\r
340 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Failed to schedule playback."));
\r
343 STDMETHOD (QueryInterface(REFIID, LPVOID*)) {return E_NOINTERFACE;}
\r
344 STDMETHOD_(ULONG, AddRef()) {return 1;}
\r
345 STDMETHOD_(ULONG, Release()) {return 1;}
\r
347 STDMETHOD(ScheduledPlaybackHasStopped())
\r
349 is_running_ = false;
\r
350 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
\r
354 STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))
\r
361 if(result == bmdOutputFrameDisplayedLate)
\r
363 graph_->set_tag("late-frame");
\r
364 video_scheduled_ += format_desc_.duration;
\r
365 audio_scheduled_ += reinterpret_cast<decklink_frame*>(completed_frame)->audio_data().size()/format_desc_.audio_channels;
\r
366 //++video_scheduled_;
\r
367 //audio_scheduled_ += format_desc_.audio_cadence[0];
\r
368 //++audio_scheduled_;
\r
370 else if(result == bmdOutputFrameDropped)
\r
371 graph_->set_tag("dropped-frame");
\r
372 else if(result == bmdOutputFrameFlushed)
\r
373 graph_->set_tag("flushed-frame");
\r
375 std::shared_ptr<const core::data_frame> frame;
\r
376 video_frame_buffer_.pop(frame);
\r
377 schedule_next_video(spl::make_shared_ptr(frame));
\r
379 unsigned long buffered;
\r
380 output_->GetBufferedVideoFrameCount(&buffered);
\r
381 graph_->set_value("buffered-video", static_cast<double>(buffered)/format_desc_.fps);
\r
385 lock(exception_mutex_, [&]
\r
387 exception_ = std::current_exception();
\r
395 STDMETHOD(RenderAudioSamples(BOOL preroll))
\r
404 if(++preroll_count_ >= buffer_size_)
\r
406 output_->EndAudioPreroll();
\r
410 schedule_next_audio(core::audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()], 0));
\r
414 std::shared_ptr<const core::data_frame> frame;
\r
415 audio_frame_buffer_.pop(frame);
\r
416 schedule_next_audio(frame->audio_data());
\r
419 unsigned long buffered;
\r
420 output_->GetBufferedAudioSampleFrameCount(&buffered);
\r
421 graph_->set_value("buffered-audio", static_cast<double>(buffered)/(format_desc_.audio_cadence[0]*2));
\r
425 tbb::spin_mutex::scoped_lock lock(exception_mutex_);
\r
426 exception_ = std::current_exception();
\r
433 template<typename T>
\r
434 void schedule_next_audio(const T& audio_data)
\r
436 auto sample_frame_count = static_cast<int>(audio_data.size()/format_desc_.audio_channels);
\r
438 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
\r
440 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
\r
441 CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
\r
443 audio_scheduled_ += sample_frame_count;
\r
446 void schedule_next_video(const spl::shared_ptr<const core::data_frame>& frame)
\r
448 CComPtr<IDeckLinkVideoFrame> frame2(new decklink_frame(frame, format_desc_, config_.key_only));
\r
449 if(FAILED(output_->ScheduleVideoFrame(frame2, video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
\r
450 CASPAR_LOG(error) << print() << L" Failed to schedule video.";
\r
452 video_scheduled_ += format_desc_.duration;
\r
454 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
\r
455 tick_timer_.restart();
\r
458 void send(const spl::shared_ptr<const core::data_frame>& frame)
\r
460 auto exception = lock(exception_mutex_, [&]
\r
465 if(exception != nullptr)
\r
466 std::rethrow_exception(exception);
\r
469 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Is not running."));
\r
471 if(config_.embedded_audio)
\r
472 audio_frame_buffer_.push(frame);
\r
473 video_frame_buffer_.push(frame);
\r
476 std::wstring print() const
\r
478 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_) + L"-" +
\r
479 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
\r
483 struct decklink_consumer_proxy : public core::frame_consumer
\r
485 const configuration config_;
\r
486 std::unique_ptr<decklink_consumer> consumer_;
\r
487 std::vector<int> audio_cadence_;
\r
488 executor executor_;
\r
491 decklink_consumer_proxy(const configuration& config)
\r
493 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
\r
495 executor_.begin_invoke([=]
\r
497 ::CoInitialize(nullptr);
\r
501 ~decklink_consumer_proxy()
\r
503 executor_.invoke([=]
\r
505 ::CoUninitialize();
\r
511 virtual void initialize(const core::video_format_desc& format_desc, int channel_index) override
\r
513 executor_.invoke([=]
\r
516 consumer_.reset(new decklink_consumer(config_, format_desc, channel_index));
\r
517 audio_cadence_ = format_desc.audio_cadence;
\r
521 virtual bool send(const spl::shared_ptr<const core::data_frame>& frame) override
\r
523 CASPAR_VERIFY(audio_cadence_.front() == static_cast<int>(frame->audio_data().size()));
\r
524 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1);
\r
526 consumer_->send(frame);
\r
530 virtual std::wstring print() const override
\r
532 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
\r
535 virtual boost::property_tree::wptree info() const override
\r
537 boost::property_tree::wptree info;
\r
538 info.add(L"type", L"decklink-consumer");
\r
539 info.add(L"key-only", config_.key_only);
\r
540 info.add(L"device", config_.device_index);
\r
541 info.add(L"low-latency", config_.low_latency);
\r
542 info.add(L"embedded-audio", config_.embedded_audio);
\r
543 info.add(L"low-latency", config_.low_latency);
\r
544 //info.add(L"internal-key", config_.internal_key);
\r
548 virtual int buffer_depth() const override
\r
550 return config_.buffer_depth();
\r
553 virtual int index() const override
\r
555 return 300 + config_.device_index;
\r
559 spl::shared_ptr<core::frame_consumer> create_consumer(const std::vector<std::wstring>& params)
\r
561 if(params.size() < 1 || params[0] != L"DECKLINK")
\r
562 return core::frame_consumer::empty();
\r
564 configuration config;
\r
566 if(params.size() > 1)
\r
567 config.device_index = boost::lexical_cast<int>(params[1]);
\r
569 if(std::find(params.begin(), params.end(), L"INTERNAL_KEY") != params.end())
\r
570 config.keyer = configuration::internal_keyer;
\r
571 else if(std::find(params.begin(), params.end(), L"EXTERNAL_KEY") != params.end())
\r
572 config.keyer = configuration::external_keyer;
\r
574 config.keyer = configuration::default_keyer;
\r
576 if(std::find(params.begin(), params.end(), L"LOW_LATENCY") != params.end())
\r
577 config.latency = configuration::low_latency;
\r
579 config.embedded_audio = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();
\r
580 config.key_only = std::find(params.begin(), params.end(), L"KEY_ONLY") != params.end();
\r
582 return spl::make_shared<decklink_consumer_proxy>(config);
\r
585 spl::shared_ptr<core::frame_consumer> create_consumer(const boost::property_tree::wptree& ptree)
\r
587 configuration config;
\r
589 auto keyer = ptree.get(L"keyer", L"external");
\r
590 if(keyer == L"external")
\r
591 config.keyer = configuration::external_keyer;
\r
592 else if(keyer == L"internal")
\r
593 config.keyer = configuration::internal_keyer;
\r
595 auto latency = ptree.get(L"latency", L"normal");
\r
596 if(latency == L"low")
\r
597 config.latency = configuration::low_latency;
\r
598 else if(latency == L"normal")
\r
599 config.latency = configuration::normal_latency;
\r
601 config.key_only = ptree.get(L"key-only", config.key_only);
\r
602 config.device_index = ptree.get(L"device", config.device_index);
\r
603 config.embedded_audio = ptree.get(L"embedded-audio", config.embedded_audio);
\r
604 config.base_buffer_depth = ptree.get(L"buffer-depth", config.base_buffer_depth);
\r
606 return spl::make_shared<decklink_consumer_proxy>(config);
\r
612 ##############################################################################
\r
618 BMD Developer Support
\r
619 developer@blackmagic-design.com
\r
621 -----------------------------------------------------------------------------
\r
623 Thanks for your inquiry. The minimum number of frames that you can preroll
\r
624 for scheduled playback is three frames for video and four frames for audio.
\r
625 As you mentioned if you preroll less frames then playback will not start or
\r
626 playback will be very sporadic. From our experience with Media Express, we
\r
627 recommended that at least seven frames are prerolled for smooth playback.
\r
629 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
\r
630 There can be around 3 frames worth of latency on scheduled output.
\r
631 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
\r
632 reduced or removed for scheduled playback. If the DisplayVideoFrameSync()
\r
633 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
\r
634 guarantee that the provided frame will be output as soon the previous
\r
635 frame output has been completed.
\r
636 ################################################################################
\r
640 ##############################################################################
\r
641 Async DMA Transfer without redundant copying
\r
646 BMD Developer Support
\r
647 developer@blackmagic-design.com
\r
649 -----------------------------------------------------------------------------
\r
651 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
\r
652 and providing a pointer to your video buffer when GetBytes() is called.
\r
653 This may help to keep copying to a minimum. Please ensure that the pixel
\r
654 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
\r
655 have to colourspace convert which may result in additional copying.
\r
656 ################################################################################
\r