2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
\r
4 * This file is part of CasparCG (www.casparcg.com).
\r
6 * CasparCG is free software: you can redistribute it and/or modify
\r
7 * it under the terms of the GNU General Public License as published by
\r
8 * the Free Software Foundation, either version 3 of the License, or
\r
9 * (at your option) any later version.
\r
11 * CasparCG is distributed in the hope that it will be useful,
\r
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
\r
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
\r
14 * GNU General Public License for more details.
\r
16 * You should have received a copy of the GNU General Public License
\r
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
\r
19 * Author: Robert Nagy, ronag89@gmail.com
\r
22 #include "../StdAfx.h"
\r
24 #include "decklink_consumer.h"
\r
26 #include "../util/util.h"
\r
28 #include "../interop/DeckLinkAPI_h.h"
\r
30 #include <core/mixer/read_frame.h>
\r
32 #include <common/concurrency/executor.h>
\r
33 #include <common/concurrency/lock.h>
\r
34 #include <common/diagnostics/graph.h>
\r
35 #include <common/exception/exceptions.h>
\r
36 #include <common/memory/memcpy.h>
\r
37 #include <common/memory/memclr.h>
\r
38 #include <common/memory/memshfl.h>
\r
40 #include <core/consumer/frame_consumer.h>
\r
42 #include <tbb/concurrent_queue.h>
\r
43 #include <tbb/cache_aligned_allocator.h>
\r
45 #include <boost/circular_buffer.hpp>
\r
46 #include <boost/timer.hpp>
\r
47 #include <boost/property_tree/ptree.hpp>
\r
49 namespace caspar { namespace decklink {
\r
51 struct configuration
\r
53 size_t device_index;
\r
54 bool embedded_audio;
\r
58 size_t base_buffer_depth;
\r
59 size_t buffer_depth;
\r
63 , embedded_audio(false)
\r
64 , internal_key(false)
\r
67 , base_buffer_depth(3)
\r
68 , buffer_depth(base_buffer_depth + (low_latency ? 0 : 1) + (embedded_audio ? 1 : 0)){}
\r
71 class decklink_frame : public IDeckLinkVideoFrame
\r
73 tbb::atomic<int> ref_count_;
\r
74 std::shared_ptr<core::read_frame> frame_;
\r
75 const core::video_format_desc format_desc_;
\r
77 const bool key_only_;
\r
78 std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>> data_;
\r
80 decklink_frame(const safe_ptr<core::read_frame>& frame, const core::video_format_desc& format_desc, bool key_only)
\r
82 , format_desc_(format_desc)
\r
83 , key_only_(key_only)
\r
90 STDMETHOD (QueryInterface(REFIID, LPVOID*))
\r
92 return E_NOINTERFACE;
\r
95 STDMETHOD_(ULONG, AddRef())
\r
97 return ++ref_count_;
\r
100 STDMETHOD_(ULONG, Release())
\r
102 if(--ref_count_ == 0)
\r
107 // IDecklinkVideoFrame
\r
109 STDMETHOD_(long, GetWidth()) {return format_desc_.width;}
\r
110 STDMETHOD_(long, GetHeight()) {return format_desc_.height;}
\r
111 STDMETHOD_(long, GetRowBytes()) {return format_desc_.width*4;}
\r
112 STDMETHOD_(BMDPixelFormat, GetPixelFormat()) {return bmdFormat8BitBGRA;}
\r
113 STDMETHOD_(BMDFrameFlags, GetFlags()) {return bmdFrameFlagDefault;}
\r
115 STDMETHOD(GetBytes(void** buffer))
\r
119 if(static_cast<size_t>(frame_->image_data().size()) != format_desc_.size)
\r
121 data_.resize(format_desc_.size, 0);
\r
122 *buffer = data_.data();
\r
128 data_.resize(frame_->image_data().size());
\r
129 fast_memshfl(data_.data(), frame_->image_data().begin(), frame_->image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
\r
132 *buffer = data_.data();
\r
135 *buffer = const_cast<uint8_t*>(frame_->image_data().begin());
\r
139 CASPAR_LOG_CURRENT_EXCEPTION();
\r
146 STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}
\r
147 STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)) {return S_FALSE;}
\r
151 const boost::iterator_range<const int32_t*> audio_data()
\r
153 return frame_->audio_data();
\r
157 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
\r
159 const int channel_index_;
\r
160 const configuration config_;
\r
162 CComPtr<IDeckLink> decklink_;
\r
163 CComQIPtr<IDeckLinkOutput> output_;
\r
164 CComQIPtr<IDeckLinkConfiguration> configuration_;
\r
165 CComQIPtr<IDeckLinkKeyer> keyer_;
\r
167 tbb::spin_mutex exception_mutex_;
\r
168 std::exception_ptr exception_;
\r
170 tbb::atomic<bool> is_running_;
\r
172 const std::wstring model_name_;
\r
173 const core::video_format_desc format_desc_;
\r
174 const size_t buffer_size_;
\r
176 long long video_scheduled_;
\r
177 long long audio_scheduled_;
\r
179 size_t preroll_count_;
\r
181 boost::circular_buffer<std::vector<int32_t>> audio_container_;
\r
183 tbb::concurrent_bounded_queue<std::shared_ptr<core::read_frame>> video_frame_buffer_;
\r
184 tbb::concurrent_bounded_queue<std::shared_ptr<core::read_frame>> audio_frame_buffer_;
\r
186 safe_ptr<diagnostics::graph> graph_;
\r
187 boost::timer tick_timer_;
\r
190 decklink_consumer(const configuration& config, const core::video_format_desc& format_desc, int channel_index)
\r
191 : channel_index_(channel_index)
\r
193 , decklink_(get_device(config.device_index))
\r
194 , output_(decklink_)
\r
195 , configuration_(decklink_)
\r
196 , keyer_(decklink_)
\r
197 , model_name_(get_model_name(decklink_))
\r
198 , format_desc_(format_desc)
\r
199 , buffer_size_(config.buffer_depth) // Minimum buffer-size 3.
\r
200 , video_scheduled_(0)
\r
201 , audio_scheduled_(0)
\r
202 , preroll_count_(0)
\r
203 , audio_container_(buffer_size_+1)
\r
205 is_running_ = true;
\r
207 video_frame_buffer_.set_capacity(1);
\r
208 audio_frame_buffer_.set_capacity(1);
\r
210 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
\r
211 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
\r
212 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
\r
213 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
\r
214 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
\r
215 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
\r
216 graph_->set_text(print());
\r
217 diagnostics::register_graph(graph_);
\r
219 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));
\r
221 if(config.embedded_audio)
\r
224 set_latency(config.low_latency);
\r
225 set_keyer(config.internal_key);
\r
227 if(config.embedded_audio)
\r
228 output_->BeginAudioPreroll();
\r
230 for(size_t n = 0; n < buffer_size_; ++n)
\r
231 schedule_next_video(make_safe<core::read_frame>());
\r
233 if(!config.embedded_audio)
\r
237 ~decklink_consumer()
\r
239 is_running_ = false;
\r
240 video_frame_buffer_.try_push(std::make_shared<core::read_frame>());
\r
241 audio_frame_buffer_.try_push(std::make_shared<core::read_frame>());
\r
243 if(output_ != nullptr)
\r
245 output_->StopScheduledPlayback(0, nullptr, 0);
\r
246 if(config_.embedded_audio)
\r
247 output_->DisableAudioOutput();
\r
248 output_->DisableVideoOutput();
\r
252 void set_latency(bool low_latency)
\r
256 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
\r
257 CASPAR_LOG(info) << print() << L" Enabled normal-latency mode";
\r
261 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
\r
262 CASPAR_LOG(info) << print() << L" Enabled low-latency mode";
\r
266 void set_keyer(bool internal_key)
\r
270 if(FAILED(keyer_->Enable(FALSE)))
\r
271 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
\r
272 else if(FAILED(keyer_->SetLevel(255)))
\r
273 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
275 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";
\r
279 if(FAILED(keyer_->Enable(TRUE)))
\r
280 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
\r
281 else if(FAILED(keyer_->SetLevel(255)))
\r
282 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
284 CASPAR_LOG(info) << print() << L" Enabled external keyer.";
\r
288 void enable_audio()
\r
290 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped)))
\r
291 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not enable audio output."));
\r
293 if(FAILED(output_->SetAudioCallback(this)))
\r
294 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not set audio callback."));
\r
296 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
\r
299 void enable_video(BMDDisplayMode display_mode)
\r
301 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
\r
302 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not enable video output."));
\r
304 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
\r
305 BOOST_THROW_EXCEPTION(caspar_exception()
\r
306 << msg_info(narrow(print()) + " Failed to set playback completion callback.")
\r
307 << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
\r
310 void start_playback()
\r
312 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
\r
313 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Failed to schedule playback."));
\r
316 STDMETHOD (QueryInterface(REFIID, LPVOID*)) {return E_NOINTERFACE;}
\r
317 STDMETHOD_(ULONG, AddRef()) {return 1;}
\r
318 STDMETHOD_(ULONG, Release()) {return 1;}
\r
320 STDMETHOD(ScheduledPlaybackHasStopped())
\r
322 is_running_ = false;
\r
323 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
\r
327 STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))
\r
334 if(result == bmdOutputFrameDisplayedLate)
\r
336 graph_->set_tag("late-frame");
\r
337 video_scheduled_ += format_desc_.duration;
\r
338 audio_scheduled_ += reinterpret_cast<decklink_frame*>(completed_frame)->audio_data().size()/format_desc_.audio_channels;
\r
339 //++video_scheduled_;
\r
340 //audio_scheduled_ += format_desc_.audio_cadence[0];
\r
341 //++audio_scheduled_;
\r
343 else if(result == bmdOutputFrameDropped)
\r
344 graph_->set_tag("dropped-frame");
\r
345 else if(result == bmdOutputFrameFlushed)
\r
346 graph_->set_tag("flushed-frame");
\r
348 std::shared_ptr<core::read_frame> frame;
\r
349 video_frame_buffer_.pop(frame);
\r
350 schedule_next_video(make_safe_ptr(frame));
\r
352 unsigned long buffered;
\r
353 output_->GetBufferedVideoFrameCount(&buffered);
\r
354 graph_->set_value("buffered-video", static_cast<double>(buffered)/format_desc_.fps);
\r
358 lock(exception_mutex_, [this]
\r
360 exception_ = std::current_exception();
\r
368 STDMETHOD(RenderAudioSamples(BOOL preroll))
\r
377 if(++preroll_count_ >= buffer_size_)
\r
379 output_->EndAudioPreroll();
\r
383 schedule_next_audio(core::audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()], 0));
\r
387 std::shared_ptr<core::read_frame> frame;
\r
388 audio_frame_buffer_.pop(frame);
\r
389 schedule_next_audio(frame->audio_data());
\r
392 unsigned long buffered;
\r
393 output_->GetBufferedAudioSampleFrameCount(&buffered);
\r
394 graph_->set_value("buffered-audio", static_cast<double>(buffered)/(format_desc_.audio_cadence[0]*2));
\r
398 lock(exception_mutex_, [this]
\r
400 exception_ = std::current_exception();
\r
408 template<typename T>
\r
409 void schedule_next_audio(const T& audio_data)
\r
411 const int sample_frame_count = audio_data.size()/format_desc_.audio_channels;
\r
413 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
\r
415 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
\r
416 CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
\r
418 audio_scheduled_ += sample_frame_count;
\r
421 void schedule_next_video(const safe_ptr<core::read_frame>& frame)
\r
423 CComPtr<IDeckLinkVideoFrame> frame2(new decklink_frame(frame, format_desc_, config_.key_only));
\r
424 if(FAILED(output_->ScheduleVideoFrame(frame2, video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
\r
425 CASPAR_LOG(error) << print() << L" Failed to schedule video.";
\r
427 video_scheduled_ += format_desc_.duration;
\r
429 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
\r
430 tick_timer_.restart();
\r
433 void send(const safe_ptr<core::read_frame>& frame)
\r
435 auto exception = lock(exception_mutex_, [this]
\r
440 if(exception != nullptr)
\r
441 std::rethrow_exception(exception);
\r
444 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Is not running."));
\r
446 if(config_.embedded_audio)
\r
447 audio_frame_buffer_.push(frame);
\r
448 video_frame_buffer_.push(frame);
\r
451 std::wstring print() const
\r
453 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_) + L"-" +
\r
454 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
\r
458 struct decklink_consumer_proxy : public core::frame_consumer
\r
460 const configuration config_;
\r
461 std::unique_ptr<decklink_consumer> consumer_;
\r
462 std::vector<size_t> audio_cadence_;
\r
463 executor executor_;
\r
466 decklink_consumer_proxy(const configuration& config)
\r
468 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
\r
470 executor_.begin_invoke([=]
\r
472 ::CoInitialize(nullptr);
\r
476 ~decklink_consumer_proxy()
\r
478 executor_.invoke([=]
\r
482 auto str = print();
\r
484 CASPAR_LOG(info) << str << L" Successfully Uninitialized.";
\r
488 executor_.invoke([=]
\r
490 ::CoUninitialize();
\r
496 virtual void initialize(const core::video_format_desc& format_desc, int channel_index) override
\r
498 executor_.invoke([=]
\r
500 consumer_.reset(new decklink_consumer(config_, format_desc, channel_index));
\r
501 audio_cadence_ = format_desc.audio_cadence;
\r
503 CASPAR_LOG(info) << print() << L" Successfully Initialized.";
\r
507 virtual bool send(const safe_ptr<core::read_frame>& frame) override
\r
509 CASPAR_VERIFY(audio_cadence_.front() == static_cast<size_t>(frame->audio_data().size()));
\r
510 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1);
\r
512 consumer_->send(frame);
\r
516 virtual std::wstring print() const override
\r
518 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
\r
521 virtual boost::property_tree::wptree info() const override
\r
523 boost::property_tree::wptree info;
\r
524 info.add(L"type", L"decklink-consumer");
\r
525 info.add(L"key-only", config_.key_only);
\r
526 info.add(L"device", config_.device_index);
\r
527 info.add(L"low-latency", config_.low_latency);
\r
528 info.add(L"embedded-audio", config_.embedded_audio);
\r
529 info.add(L"low-latency", config_.low_latency);
\r
530 info.add(L"internal-key", config_.internal_key);
\r
534 virtual size_t buffer_depth() const override
\r
536 return config_.buffer_depth;
\r
539 virtual int index() const override
\r
541 return 300 + config_.device_index;
\r
545 safe_ptr<core::frame_consumer> create_consumer(const std::vector<std::wstring>& params)
\r
547 if(params.size() < 1 || params[0] != L"DECKLINK")
\r
548 return core::frame_consumer::empty();
\r
550 configuration config;
\r
552 if(params.size() > 1)
\r
553 config.device_index = lexical_cast_or_default<int>(params[1], config.device_index);
\r
555 config.internal_key = std::find(params.begin(), params.end(), L"INTERNAL_KEY") != params.end();
\r
556 config.low_latency = std::find(params.begin(), params.end(), L"LOW_LATENCY") != params.end();
\r
557 config.embedded_audio = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();
\r
558 config.key_only = std::find(params.begin(), params.end(), L"KEY_ONLY") != params.end();
\r
560 return make_safe<decklink_consumer_proxy>(config);
\r
563 safe_ptr<core::frame_consumer> create_consumer(const boost::property_tree::wptree& ptree)
\r
565 configuration config;
\r
567 config.internal_key = ptree.get(L"internal-key", config.internal_key);
\r
568 config.low_latency = ptree.get(L"low-latency", config.low_latency);
\r
569 config.key_only = ptree.get(L"key-only", config.key_only);
\r
570 config.device_index = ptree.get(L"device", config.device_index);
\r
571 config.embedded_audio = ptree.get(L"embedded-audio", config.embedded_audio);
\r
572 config.base_buffer_depth = ptree.get(L"buffer-depth", config.base_buffer_depth);
\r
574 return make_safe<decklink_consumer_proxy>(config);
\r
580 ##############################################################################
\r
586 BMD Developer Support
\r
587 developer@blackmagic-design.com
\r
589 -----------------------------------------------------------------------------
\r
591 Thanks for your inquiry. The minimum number of frames that you can preroll
\r
592 for scheduled playback is three frames for video and four frames for audio.
\r
593 As you mentioned if you preroll less frames then playback will not start or
\r
594 playback will be very sporadic. From our experience with Media Express, we
\r
595 recommended that at least seven frames are prerolled for smooth playback.
\r
597 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
\r
598 There can be around 3 frames worth of latency on scheduled output.
\r
599 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
\r
600 reduced or removed for scheduled playback. If the DisplayVideoFrameSync()
\r
601 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
\r
602 guarantee that the provided frame will be output as soon the previous
\r
603 frame output has been completed.
\r
604 ################################################################################
\r
608 ##############################################################################
\r
609 Async DMA Transfer without redundant copying
\r
614 BMD Developer Support
\r
615 developer@blackmagic-design.com
\r
617 -----------------------------------------------------------------------------
\r
619 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
\r
620 and providing a pointer to your video buffer when GetBytes() is called.
\r
621 This may help to keep copying to a minimum. Please ensure that the pixel
\r
622 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
\r
623 have to colourspace convert which may result in additional copying.
\r
624 ################################################################################
\r