2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
\r
4 * This file is part of CasparCG (www.casparcg.com).
\r
6 * CasparCG is free software: you can redistribute it and/or modify
\r
7 * it under the terms of the GNU General Public License as published by
\r
8 * the Free Software Foundation, either version 3 of the License, or
\r
9 * (at your option) any later version.
\r
11 * CasparCG is distributed in the hope that it will be useful,
\r
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
\r
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
\r
14 * GNU General Public License for more details.
\r
16 * You should have received a copy of the GNU General Public License
\r
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
\r
19 * Author: Robert Nagy, ronag89@gmail.com
\r
22 #include "../StdAfx.h"
\r
24 #include "decklink_consumer.h"
\r
26 #include "../util/util.h"
\r
28 #include "../interop/DeckLinkAPI_h.h"
\r
30 #include <core/frame.h>
\r
31 #include <core/mixer/audio/audio_mixer.h>
\r
33 #include <common/concurrency/executor.h>
\r
34 #include <common/concurrency/lock.h>
\r
35 #include <common/diagnostics/graph.h>
\r
36 #include <common/exception/exceptions.h>
\r
37 #include <common/memory/memshfl.h>
\r
39 #include <core/consumer/frame_consumer.h>
\r
41 #include <tbb/concurrent_queue.h>
\r
42 #include <tbb/cache_aligned_allocator.h>
\r
44 #include <common/assert.h>
\r
45 #include <boost/lexical_cast.hpp>
\r
46 #include <boost/circular_buffer.hpp>
\r
47 #include <boost/timer.hpp>
\r
48 #include <boost/property_tree/ptree.hpp>
\r
50 namespace caspar { namespace decklink {
\r
52 struct configuration
\r
55 bool embedded_audio;
\r
59 int base_buffer_depth;
\r
63 , embedded_audio(false)
\r
64 , internal_key(false)
\r
67 , base_buffer_depth(3)
\r
71 int buffer_depth() const
\r
73 return base_buffer_depth + (low_latency ? 0 : 1) + (embedded_audio ? 1 : 0);
\r
77 class decklink_frame : public IDeckLinkVideoFrame
\r
79 tbb::atomic<int> ref_count_;
\r
80 std::shared_ptr<const core::frame> frame_;
\r
81 const core::video_format_desc format_desc_;
\r
83 const bool key_only_;
\r
84 std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>> data_;
\r
86 decklink_frame(const safe_ptr<const core::frame>& frame, const core::video_format_desc& format_desc, bool key_only)
\r
88 , format_desc_(format_desc)
\r
89 , key_only_(key_only)
\r
96 STDMETHOD (QueryInterface(REFIID, LPVOID*))
\r
98 return E_NOINTERFACE;
\r
101 STDMETHOD_(ULONG, AddRef())
\r
103 return ++ref_count_;
\r
106 STDMETHOD_(ULONG, Release())
\r
108 if(--ref_count_ == 0)
\r
113 // IDecklinkVideoFrame
\r
115 STDMETHOD_(long, GetWidth()) {return format_desc_.width;}
\r
116 STDMETHOD_(long, GetHeight()) {return format_desc_.height;}
\r
117 STDMETHOD_(long, GetRowBytes()) {return format_desc_.width*4;}
\r
118 STDMETHOD_(BMDPixelFormat, GetPixelFormat()) {return bmdFormat8BitBGRA;}
\r
119 STDMETHOD_(BMDFrameFlags, GetFlags()) {return bmdFrameFlagDefault;}
\r
121 STDMETHOD(GetBytes(void** buffer))
\r
125 if(static_cast<int>(frame_->image_data().size()) != format_desc_.size)
\r
127 data_.resize(format_desc_.size, 0);
\r
128 *buffer = data_.data();
\r
134 data_.resize(frame_->image_data().size());
\r
135 aligned_memshfl(data_.data(), frame_->image_data().begin(), frame_->image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
\r
137 *buffer = data_.data();
\r
140 *buffer = const_cast<uint8_t*>(frame_->image_data().begin());
\r
144 CASPAR_LOG_CURRENT_EXCEPTION();
\r
151 STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}
\r
152 STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)) {return S_FALSE;}
\r
156 const boost::iterator_range<const int32_t*> audio_data()
\r
158 return frame_->audio_data();
\r
162 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
\r
164 const int channel_index_;
\r
165 const configuration config_;
\r
167 CComPtr<IDeckLink> decklink_;
\r
168 CComQIPtr<IDeckLinkOutput> output_;
\r
169 CComQIPtr<IDeckLinkConfiguration> configuration_;
\r
170 CComQIPtr<IDeckLinkKeyer> keyer_;
\r
172 tbb::spin_mutex exception_mutex_;
\r
173 std::exception_ptr exception_;
\r
175 tbb::atomic<bool> is_running_;
\r
177 const std::wstring model_name_;
\r
178 const core::video_format_desc format_desc_;
\r
179 const int buffer_size_;
\r
181 long long video_scheduled_;
\r
182 long long audio_scheduled_;
\r
184 int preroll_count_;
\r
186 boost::circular_buffer<std::vector<int32_t>> audio_container_;
\r
188 tbb::concurrent_bounded_queue<std::shared_ptr<const core::frame>> video_frame_buffer_;
\r
189 tbb::concurrent_bounded_queue<std::shared_ptr<const core::frame>> audio_frame_buffer_;
\r
191 safe_ptr<diagnostics::graph> graph_;
\r
192 boost::timer tick_timer_;
\r
195 decklink_consumer(const configuration& config, const core::video_format_desc& format_desc, int channel_index)
\r
196 : channel_index_(channel_index)
\r
198 , decklink_(get_device(config.device_index))
\r
199 , output_(decklink_)
\r
200 , configuration_(decklink_)
\r
201 , keyer_(decklink_)
\r
202 , model_name_(get_model_name(decklink_))
\r
203 , format_desc_(format_desc)
\r
204 , buffer_size_(config.buffer_depth()) // Minimum buffer-size 3.
\r
205 , video_scheduled_(0)
\r
206 , audio_scheduled_(0)
\r
207 , preroll_count_(0)
\r
208 , audio_container_(buffer_size_+1)
\r
210 is_running_ = true;
\r
212 video_frame_buffer_.set_capacity(1);
\r
213 audio_frame_buffer_.set_capacity(1);
\r
215 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
\r
216 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
\r
217 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
\r
218 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
\r
219 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
\r
220 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
\r
221 graph_->set_text(print());
\r
222 diagnostics::register_graph(graph_);
\r
224 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));
\r
226 if(config.embedded_audio)
\r
229 set_latency(config.low_latency);
\r
230 set_keyer(config.internal_key);
\r
232 if(config.embedded_audio)
\r
233 output_->BeginAudioPreroll();
\r
235 for(int n = 0; n < buffer_size_; ++n)
\r
236 schedule_next_video(core::frame::empty());
\r
238 if(!config.embedded_audio)
\r
242 ~decklink_consumer()
\r
244 is_running_ = false;
\r
245 video_frame_buffer_.try_push(core::frame::empty());
\r
246 audio_frame_buffer_.try_push(core::frame::empty());
\r
248 if(output_ != nullptr)
\r
250 output_->StopScheduledPlayback(0, nullptr, 0);
\r
251 if(config_.embedded_audio)
\r
252 output_->DisableAudioOutput();
\r
253 output_->DisableVideoOutput();
\r
257 void set_latency(bool low_latency)
\r
261 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
\r
262 CASPAR_LOG(info) << print() << L" Enabled normal-latency mode";
\r
266 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
\r
267 CASPAR_LOG(info) << print() << L" Enabled low-latency mode";
\r
271 void set_keyer(bool internal_key)
\r
275 if(FAILED(keyer_->Enable(FALSE)))
\r
276 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
\r
277 else if(FAILED(keyer_->SetLevel(255)))
\r
278 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
280 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";
\r
284 if(FAILED(keyer_->Enable(TRUE)))
\r
285 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
\r
286 else if(FAILED(keyer_->SetLevel(255)))
\r
287 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
289 CASPAR_LOG(info) << print() << L" Enabled external keyer.";
\r
293 void enable_audio()
\r
295 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped)))
\r
296 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable audio output."));
\r
298 if(FAILED(output_->SetAudioCallback(this)))
\r
299 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not set audio callback."));
\r
301 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
\r
304 void enable_video(BMDDisplayMode display_mode)
\r
306 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
\r
307 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable video output."));
\r
309 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
\r
310 BOOST_THROW_EXCEPTION(caspar_exception()
\r
311 << msg_info(u8(print()) + " Failed to set playback completion callback.")
\r
312 << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
\r
315 void start_playback()
\r
317 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
\r
318 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Failed to schedule playback."));
\r
321 STDMETHOD (QueryInterface(REFIID, LPVOID*)) {return E_NOINTERFACE;}
\r
322 STDMETHOD_(ULONG, AddRef()) {return 1;}
\r
323 STDMETHOD_(ULONG, Release()) {return 1;}
\r
325 STDMETHOD(ScheduledPlaybackHasStopped())
\r
327 is_running_ = false;
\r
328 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
\r
332 STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))
\r
339 if(result == bmdOutputFrameDisplayedLate)
\r
341 graph_->set_tag("late-frame");
\r
342 video_scheduled_ += format_desc_.duration;
\r
343 audio_scheduled_ += reinterpret_cast<decklink_frame*>(completed_frame)->audio_data().size()/format_desc_.audio_channels;
\r
344 //++video_scheduled_;
\r
345 //audio_scheduled_ += format_desc_.audio_cadence[0];
\r
346 //++audio_scheduled_;
\r
348 else if(result == bmdOutputFrameDropped)
\r
349 graph_->set_tag("dropped-frame");
\r
350 else if(result == bmdOutputFrameFlushed)
\r
351 graph_->set_tag("flushed-frame");
\r
353 std::shared_ptr<const core::frame> frame;
\r
354 video_frame_buffer_.pop(frame);
\r
355 schedule_next_video(make_safe_ptr(frame));
\r
357 unsigned long buffered;
\r
358 output_->GetBufferedVideoFrameCount(&buffered);
\r
359 graph_->set_value("buffered-video", static_cast<double>(buffered)/format_desc_.fps);
\r
363 lock(exception_mutex_, [&]
\r
365 exception_ = std::current_exception();
\r
373 STDMETHOD(RenderAudioSamples(BOOL preroll))
\r
382 if(++preroll_count_ >= buffer_size_)
\r
384 output_->EndAudioPreroll();
\r
388 schedule_next_audio(core::audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()], 0));
\r
392 std::shared_ptr<const core::frame> frame;
\r
393 audio_frame_buffer_.pop(frame);
\r
394 schedule_next_audio(frame->audio_data());
\r
397 unsigned long buffered;
\r
398 output_->GetBufferedAudioSampleFrameCount(&buffered);
\r
399 graph_->set_value("buffered-audio", static_cast<double>(buffered)/(format_desc_.audio_cadence[0]*2));
\r
403 tbb::spin_mutex::scoped_lock lock(exception_mutex_);
\r
404 exception_ = std::current_exception();
\r
411 template<typename T>
\r
412 void schedule_next_audio(const T& audio_data)
\r
414 auto sample_frame_count = static_cast<int>(audio_data.size()/format_desc_.audio_channels);
\r
416 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
\r
418 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
\r
419 CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
\r
421 audio_scheduled_ += sample_frame_count;
\r
424 void schedule_next_video(const safe_ptr<const core::frame>& frame)
\r
426 CComPtr<IDeckLinkVideoFrame> frame2(new decklink_frame(frame, format_desc_, config_.key_only));
\r
427 if(FAILED(output_->ScheduleVideoFrame(frame2, video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
\r
428 CASPAR_LOG(error) << print() << L" Failed to schedule video.";
\r
430 video_scheduled_ += format_desc_.duration;
\r
432 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
\r
433 tick_timer_.restart();
\r
436 void send(const safe_ptr<const core::frame>& frame)
\r
438 auto exception = lock(exception_mutex_, [&]
\r
443 if(exception != nullptr)
\r
444 std::rethrow_exception(exception);
\r
447 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Is not running."));
\r
449 if(config_.embedded_audio)
\r
450 audio_frame_buffer_.push(frame);
\r
451 video_frame_buffer_.push(frame);
\r
454 std::wstring print() const
\r
456 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_) + L"-" +
\r
457 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
\r
461 struct decklink_consumer_proxy : public core::frame_consumer
\r
463 const configuration config_;
\r
464 std::unique_ptr<decklink_consumer> consumer_;
\r
465 std::vector<int> audio_cadence_;
\r
466 executor executor_;
\r
469 decklink_consumer_proxy(const configuration& config)
\r
471 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
\r
473 executor_.begin_invoke([=]
\r
475 ::CoInitialize(nullptr);
\r
479 ~decklink_consumer_proxy()
\r
481 executor_.invoke([=]
\r
485 auto str = print();
\r
487 CASPAR_LOG(info) << str << L" Successfully Uninitialized.";
\r
489 ::CoUninitialize();
\r
495 virtual void initialize(const core::video_format_desc& format_desc, int channel_index) override
\r
497 executor_.invoke([=]
\r
499 consumer_.reset(new decklink_consumer(config_, format_desc, channel_index));
\r
500 audio_cadence_ = format_desc.audio_cadence;
\r
502 CASPAR_LOG(info) << print() << L" Successfully Initialized.";
\r
506 virtual bool send(const safe_ptr<const core::frame>& frame) override
\r
508 CASPAR_VERIFY(audio_cadence_.front() == static_cast<int>(frame->audio_data().size()));
\r
509 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1);
\r
511 consumer_->send(frame);
\r
515 virtual std::wstring print() const override
\r
517 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
\r
520 virtual boost::property_tree::wptree info() const override
\r
522 boost::property_tree::wptree info;
\r
523 info.add(L"type", L"decklink-consumer");
\r
524 info.add(L"key-only", config_.key_only);
\r
525 info.add(L"device", config_.device_index);
\r
526 info.add(L"low-latency", config_.low_latency);
\r
527 info.add(L"embedded-audio", config_.embedded_audio);
\r
528 info.add(L"low-latency", config_.low_latency);
\r
529 info.add(L"internal-key", config_.internal_key);
\r
533 virtual int buffer_depth() const override
\r
535 return config_.buffer_depth();
\r
538 virtual int index() const override
\r
540 return 300 + config_.device_index;
\r
544 safe_ptr<core::frame_consumer> create_consumer(const std::vector<std::wstring>& params)
\r
546 if(params.size() < 1 || params[0] != L"DECKLINK")
\r
547 return core::frame_consumer::empty();
\r
549 configuration config;
\r
551 if(params.size() > 1)
\r
552 config.device_index = boost::lexical_cast<int>(params[1]);
\r
554 config.internal_key = std::find(params.begin(), params.end(), L"INTERNAL_KEY") != params.end();
\r
555 config.low_latency = std::find(params.begin(), params.end(), L"LOW_LATENCY") != params.end();
\r
556 config.embedded_audio = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();
\r
557 config.key_only = std::find(params.begin(), params.end(), L"KEY_ONLY") != params.end();
\r
559 return make_safe<decklink_consumer_proxy>(config);
\r
562 safe_ptr<core::frame_consumer> create_consumer(const boost::property_tree::wptree& ptree)
\r
564 configuration config;
\r
566 config.internal_key = ptree.get(L"internal-key", config.internal_key);
\r
567 config.low_latency = ptree.get(L"low-latency", config.low_latency);
\r
568 config.key_only = ptree.get(L"key-only", config.key_only);
\r
569 config.device_index = ptree.get(L"device", config.device_index);
\r
570 config.embedded_audio = ptree.get(L"embedded-audio", config.embedded_audio);
\r
571 config.base_buffer_depth = ptree.get(L"buffer-depth", config.base_buffer_depth);
\r
573 return make_safe<decklink_consumer_proxy>(config);
\r
579 ##############################################################################
\r
585 BMD Developer Support
\r
586 developer@blackmagic-design.com
\r
588 -----------------------------------------------------------------------------
\r
590 Thanks for your inquiry. The minimum number of frames that you can preroll
\r
591 for scheduled playback is three frames for video and four frames for audio.
\r
592 As you mentioned if you preroll less frames then playback will not start or
\r
593 playback will be very sporadic. From our experience with Media Express, we
\r
594 recommended that at least seven frames are prerolled for smooth playback.
\r
596 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
\r
597 There can be around 3 frames worth of latency on scheduled output.
\r
598 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
\r
599 reduced or removed for scheduled playback. If the DisplayVideoFrameSync()
\r
600 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
\r
601 guarantee that the provided frame will be output as soon the previous
\r
602 frame output has been completed.
\r
603 ################################################################################
\r
607 ##############################################################################
\r
608 Async DMA Transfer without redundant copying
\r
613 BMD Developer Support
\r
614 developer@blackmagic-design.com
\r
616 -----------------------------------------------------------------------------
\r
618 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
\r
619 and providing a pointer to your video buffer when GetBytes() is called.
\r
620 This may help to keep copying to a minimum. Please ensure that the pixel
\r
621 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
\r
622 have to colourspace convert which may result in additional copying.
\r
623 ################################################################################
\r