2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
\r
4 * This file is part of CasparCG (www.casparcg.com).
\r
6 * CasparCG is free software: you can redistribute it and/or modify
\r
7 * it under the terms of the GNU General Public License as published by
\r
8 * the Free Software Foundation, either version 3 of the License, or
\r
9 * (at your option) any later version.
\r
11 * CasparCG is distributed in the hope that it will be useful,
\r
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
\r
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
\r
14 * GNU General Public License for more details.
\r
16 * You should have received a copy of the GNU General Public License
\r
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
\r
19 * Author: Robert Nagy, ronag89@gmail.com
\r
22 #include "../StdAfx.h"
\r
24 #include "decklink_consumer.h"
\r
26 #include "../util/util.h"
\r
28 #include "../interop/DeckLinkAPI_h.h"
\r
30 #include <core/mixer/read_frame.h>
\r
32 #include <common/concurrency/executor.h>
\r
33 #include <common/concurrency/lock.h>
\r
34 #include <common/diagnostics/graph.h>
\r
35 #include <common/exception/exceptions.h>
\r
36 #include <common/memory/memshfl.h>
\r
38 #include <core/consumer/frame_consumer.h>
\r
40 #include <tbb/concurrent_queue.h>
\r
41 #include <tbb/cache_aligned_allocator.h>
\r
43 #include <common/assert.h>
\r
44 #include <boost/lexical_cast.hpp>
\r
45 #include <boost/circular_buffer.hpp>
\r
46 #include <boost/timer.hpp>
\r
47 #include <boost/property_tree/ptree.hpp>
\r
49 namespace caspar { namespace decklink {
\r
51 struct configuration
\r
54 bool embedded_audio;
\r
58 int base_buffer_depth;
\r
62 , embedded_audio(false)
\r
63 , internal_key(false)
\r
66 , base_buffer_depth(3)
\r
70 int buffer_depth() const
\r
72 return base_buffer_depth + (low_latency ? 0 : 1) + (embedded_audio ? 1 : 0);
\r
76 class decklink_frame : public IDeckLinkVideoFrame
\r
78 tbb::atomic<int> ref_count_;
\r
79 std::shared_ptr<core::read_frame> frame_;
\r
80 const core::video_format_desc format_desc_;
\r
82 const bool key_only_;
\r
83 std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>> data_;
\r
85 decklink_frame(const safe_ptr<core::read_frame>& frame, const core::video_format_desc& format_desc, bool key_only)
\r
87 , format_desc_(format_desc)
\r
88 , key_only_(key_only)
\r
95 STDMETHOD (QueryInterface(REFIID, LPVOID*))
\r
97 return E_NOINTERFACE;
\r
100 STDMETHOD_(ULONG, AddRef())
\r
102 return ++ref_count_;
\r
105 STDMETHOD_(ULONG, Release())
\r
107 if(--ref_count_ == 0)
\r
112 // IDecklinkVideoFrame
\r
114 STDMETHOD_(long, GetWidth()) {return format_desc_.width;}
\r
115 STDMETHOD_(long, GetHeight()) {return format_desc_.height;}
\r
116 STDMETHOD_(long, GetRowBytes()) {return format_desc_.width*4;}
\r
117 STDMETHOD_(BMDPixelFormat, GetPixelFormat()) {return bmdFormat8BitBGRA;}
\r
118 STDMETHOD_(BMDFrameFlags, GetFlags()) {return bmdFrameFlagDefault;}
\r
120 STDMETHOD(GetBytes(void** buffer))
\r
124 if(static_cast<int>(frame_->image_data().size()) != format_desc_.size)
\r
126 data_.resize(format_desc_.size, 0);
\r
127 *buffer = data_.data();
\r
133 data_.resize(frame_->image_data().size());
\r
134 aligned_memshfl(data_.data(), frame_->image_data().begin(), frame_->image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
\r
136 *buffer = data_.data();
\r
139 *buffer = const_cast<uint8_t*>(frame_->image_data().begin());
\r
143 CASPAR_LOG_CURRENT_EXCEPTION();
\r
150 STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}
\r
151 STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)) {return S_FALSE;}
\r
155 const boost::iterator_range<const int32_t*> audio_data()
\r
157 return frame_->audio_data();
\r
161 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
\r
163 const int channel_index_;
\r
164 const configuration config_;
\r
166 CComPtr<IDeckLink> decklink_;
\r
167 CComQIPtr<IDeckLinkOutput> output_;
\r
168 CComQIPtr<IDeckLinkConfiguration> configuration_;
\r
169 CComQIPtr<IDeckLinkKeyer> keyer_;
\r
171 tbb::spin_mutex exception_mutex_;
\r
172 std::exception_ptr exception_;
\r
174 tbb::atomic<bool> is_running_;
\r
176 const std::wstring model_name_;
\r
177 const core::video_format_desc format_desc_;
\r
178 const int buffer_size_;
\r
180 long long video_scheduled_;
\r
181 long long audio_scheduled_;
\r
183 int preroll_count_;
\r
185 boost::circular_buffer<std::vector<int32_t>> audio_container_;
\r
187 tbb::concurrent_bounded_queue<std::shared_ptr<core::read_frame>> video_frame_buffer_;
\r
188 tbb::concurrent_bounded_queue<std::shared_ptr<core::read_frame>> audio_frame_buffer_;
\r
190 safe_ptr<diagnostics::graph> graph_;
\r
191 boost::timer tick_timer_;
\r
194 decklink_consumer(const configuration& config, const core::video_format_desc& format_desc, int channel_index)
\r
195 : channel_index_(channel_index)
\r
197 , decklink_(get_device(config.device_index))
\r
198 , output_(decklink_)
\r
199 , configuration_(decklink_)
\r
200 , keyer_(decklink_)
\r
201 , model_name_(get_model_name(decklink_))
\r
202 , format_desc_(format_desc)
\r
203 , buffer_size_(config.buffer_depth()) // Minimum buffer-size 3.
\r
204 , video_scheduled_(0)
\r
205 , audio_scheduled_(0)
\r
206 , preroll_count_(0)
\r
207 , audio_container_(buffer_size_+1)
\r
209 is_running_ = true;
\r
211 video_frame_buffer_.set_capacity(1);
\r
212 audio_frame_buffer_.set_capacity(1);
\r
214 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
\r
215 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
\r
216 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
\r
217 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
\r
218 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
\r
219 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
\r
220 graph_->set_text(print());
\r
221 diagnostics::register_graph(graph_);
\r
223 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));
\r
225 if(config.embedded_audio)
\r
228 set_latency(config.low_latency);
\r
229 set_keyer(config.internal_key);
\r
231 if(config.embedded_audio)
\r
232 output_->BeginAudioPreroll();
\r
234 for(int n = 0; n < buffer_size_; ++n)
\r
235 schedule_next_video(make_safe<core::read_frame>());
\r
237 if(!config.embedded_audio)
\r
241 ~decklink_consumer()
\r
243 is_running_ = false;
\r
244 video_frame_buffer_.try_push(std::make_shared<core::read_frame>());
\r
245 audio_frame_buffer_.try_push(std::make_shared<core::read_frame>());
\r
247 if(output_ != nullptr)
\r
249 output_->StopScheduledPlayback(0, nullptr, 0);
\r
250 if(config_.embedded_audio)
\r
251 output_->DisableAudioOutput();
\r
252 output_->DisableVideoOutput();
\r
256 void set_latency(bool low_latency)
\r
260 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
\r
261 CASPAR_LOG(info) << print() << L" Enabled normal-latency mode";
\r
265 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
\r
266 CASPAR_LOG(info) << print() << L" Enabled low-latency mode";
\r
270 void set_keyer(bool internal_key)
\r
274 if(FAILED(keyer_->Enable(FALSE)))
\r
275 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
\r
276 else if(FAILED(keyer_->SetLevel(255)))
\r
277 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
279 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";
\r
283 if(FAILED(keyer_->Enable(TRUE)))
\r
284 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
\r
285 else if(FAILED(keyer_->SetLevel(255)))
\r
286 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
288 CASPAR_LOG(info) << print() << L" Enabled external keyer.";
\r
292 void enable_audio()
\r
294 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped)))
\r
295 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable audio output."));
\r
297 if(FAILED(output_->SetAudioCallback(this)))
\r
298 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not set audio callback."));
\r
300 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
\r
303 void enable_video(BMDDisplayMode display_mode)
\r
305 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
\r
306 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable video output."));
\r
308 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
\r
309 BOOST_THROW_EXCEPTION(caspar_exception()
\r
310 << msg_info(u8(print()) + " Failed to set playback completion callback.")
\r
311 << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
\r
314 void start_playback()
\r
316 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
\r
317 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Failed to schedule playback."));
\r
320 STDMETHOD (QueryInterface(REFIID, LPVOID*)) {return E_NOINTERFACE;}
\r
321 STDMETHOD_(ULONG, AddRef()) {return 1;}
\r
322 STDMETHOD_(ULONG, Release()) {return 1;}
\r
324 STDMETHOD(ScheduledPlaybackHasStopped())
\r
326 is_running_ = false;
\r
327 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
\r
331 STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))
\r
338 if(result == bmdOutputFrameDisplayedLate)
\r
340 graph_->set_tag("late-frame");
\r
341 video_scheduled_ += format_desc_.duration;
\r
342 audio_scheduled_ += reinterpret_cast<decklink_frame*>(completed_frame)->audio_data().size()/format_desc_.audio_channels;
\r
343 //++video_scheduled_;
\r
344 //audio_scheduled_ += format_desc_.audio_cadence[0];
\r
345 //++audio_scheduled_;
\r
347 else if(result == bmdOutputFrameDropped)
\r
348 graph_->set_tag("dropped-frame");
\r
349 else if(result == bmdOutputFrameFlushed)
\r
350 graph_->set_tag("flushed-frame");
\r
352 std::shared_ptr<core::read_frame> frame;
\r
353 video_frame_buffer_.pop(frame);
\r
354 schedule_next_video(make_safe_ptr(frame));
\r
356 unsigned long buffered;
\r
357 output_->GetBufferedVideoFrameCount(&buffered);
\r
358 graph_->set_value("buffered-video", static_cast<double>(buffered)/format_desc_.fps);
\r
362 lock(exception_mutex_, [&]
\r
364 exception_ = std::current_exception();
\r
372 STDMETHOD(RenderAudioSamples(BOOL preroll))
\r
381 if(++preroll_count_ >= buffer_size_)
\r
383 output_->EndAudioPreroll();
\r
387 schedule_next_audio(core::audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()], 0));
\r
391 std::shared_ptr<core::read_frame> frame;
\r
392 audio_frame_buffer_.pop(frame);
\r
393 schedule_next_audio(frame->audio_data());
\r
396 unsigned long buffered;
\r
397 output_->GetBufferedAudioSampleFrameCount(&buffered);
\r
398 graph_->set_value("buffered-audio", static_cast<double>(buffered)/(format_desc_.audio_cadence[0]*2));
\r
402 tbb::spin_mutex::scoped_lock lock(exception_mutex_);
\r
403 exception_ = std::current_exception();
\r
410 template<typename T>
\r
411 void schedule_next_audio(const T& audio_data)
\r
413 auto sample_frame_count = static_cast<int>(audio_data.size()/format_desc_.audio_channels);
\r
415 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
\r
417 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
\r
418 CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
\r
420 audio_scheduled_ += sample_frame_count;
\r
423 void schedule_next_video(const safe_ptr<core::read_frame>& frame)
\r
425 CComPtr<IDeckLinkVideoFrame> frame2(new decklink_frame(frame, format_desc_, config_.key_only));
\r
426 if(FAILED(output_->ScheduleVideoFrame(frame2, video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
\r
427 CASPAR_LOG(error) << print() << L" Failed to schedule video.";
\r
429 video_scheduled_ += format_desc_.duration;
\r
431 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
\r
432 tick_timer_.restart();
\r
435 void send(const safe_ptr<core::read_frame>& frame)
\r
437 auto exception = lock(exception_mutex_, [&]
\r
442 if(exception != nullptr)
\r
443 std::rethrow_exception(exception);
\r
446 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Is not running."));
\r
448 if(config_.embedded_audio)
\r
449 audio_frame_buffer_.push(frame);
\r
450 video_frame_buffer_.push(frame);
\r
453 std::wstring print() const
\r
455 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_) + L"-" +
\r
456 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
\r
460 struct decklink_consumer_proxy : public core::frame_consumer
\r
462 const configuration config_;
\r
463 std::unique_ptr<decklink_consumer> consumer_;
\r
464 std::vector<int> audio_cadence_;
\r
465 executor executor_;
\r
468 decklink_consumer_proxy(const configuration& config)
\r
470 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
\r
472 executor_.begin_invoke([=]
\r
474 ::CoInitialize(nullptr);
\r
478 ~decklink_consumer_proxy()
\r
480 executor_.invoke([=]
\r
484 auto str = print();
\r
486 CASPAR_LOG(info) << str << L" Successfully Uninitialized.";
\r
488 ::CoUninitialize();
\r
494 virtual void initialize(const core::video_format_desc& format_desc, int channel_index) override
\r
496 executor_.invoke([=]
\r
498 consumer_.reset(new decklink_consumer(config_, format_desc, channel_index));
\r
499 audio_cadence_ = format_desc.audio_cadence;
\r
501 CASPAR_LOG(info) << print() << L" Successfully Initialized.";
\r
505 virtual bool send(const safe_ptr<core::read_frame>& frame) override
\r
507 CASPAR_VERIFY(audio_cadence_.front() == static_cast<int>(frame->audio_data().size()));
\r
508 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1);
\r
510 consumer_->send(frame);
\r
514 virtual std::wstring print() const override
\r
516 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
\r
519 virtual boost::property_tree::wptree info() const override
\r
521 boost::property_tree::wptree info;
\r
522 info.add(L"type", L"decklink-consumer");
\r
523 info.add(L"key-only", config_.key_only);
\r
524 info.add(L"device", config_.device_index);
\r
525 info.add(L"low-latency", config_.low_latency);
\r
526 info.add(L"embedded-audio", config_.embedded_audio);
\r
527 info.add(L"low-latency", config_.low_latency);
\r
528 info.add(L"internal-key", config_.internal_key);
\r
532 virtual int buffer_depth() const override
\r
534 return config_.buffer_depth();
\r
537 virtual int index() const override
\r
539 return 300 + config_.device_index;
\r
543 safe_ptr<core::frame_consumer> create_consumer(const std::vector<std::wstring>& params)
\r
545 if(params.size() < 1 || params[0] != L"DECKLINK")
\r
546 return core::frame_consumer::empty();
\r
548 configuration config;
\r
550 if(params.size() > 1)
\r
551 config.device_index = boost::lexical_cast<int>(params[1]);
\r
553 config.internal_key = std::find(params.begin(), params.end(), L"INTERNAL_KEY") != params.end();
\r
554 config.low_latency = std::find(params.begin(), params.end(), L"LOW_LATENCY") != params.end();
\r
555 config.embedded_audio = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();
\r
556 config.key_only = std::find(params.begin(), params.end(), L"KEY_ONLY") != params.end();
\r
558 return make_safe<decklink_consumer_proxy>(config);
\r
561 safe_ptr<core::frame_consumer> create_consumer(const boost::property_tree::wptree& ptree)
\r
563 configuration config;
\r
565 config.internal_key = ptree.get(L"internal-key", config.internal_key);
\r
566 config.low_latency = ptree.get(L"low-latency", config.low_latency);
\r
567 config.key_only = ptree.get(L"key-only", config.key_only);
\r
568 config.device_index = ptree.get(L"device", config.device_index);
\r
569 config.embedded_audio = ptree.get(L"embedded-audio", config.embedded_audio);
\r
570 config.base_buffer_depth = ptree.get(L"buffer-depth", config.base_buffer_depth);
\r
572 return make_safe<decklink_consumer_proxy>(config);
\r
578 ##############################################################################
\r
584 BMD Developer Support
\r
585 developer@blackmagic-design.com
\r
587 -----------------------------------------------------------------------------
\r
589 Thanks for your inquiry. The minimum number of frames that you can preroll
\r
590 for scheduled playback is three frames for video and four frames for audio.
\r
591 As you mentioned if you preroll less frames then playback will not start or
\r
592 playback will be very sporadic. From our experience with Media Express, we
\r
593 recommended that at least seven frames are prerolled for smooth playback.
\r
595 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
\r
596 There can be around 3 frames worth of latency on scheduled output.
\r
597 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
\r
598 reduced or removed for scheduled playback. If the DisplayVideoFrameSync()
\r
599 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
\r
600 guarantee that the provided frame will be output as soon the previous
\r
601 frame output has been completed.
\r
602 ################################################################################
\r
606 ##############################################################################
\r
607 Async DMA Transfer without redundant copying
\r
612 BMD Developer Support
\r
613 developer@blackmagic-design.com
\r
615 -----------------------------------------------------------------------------
\r
617 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
\r
618 and providing a pointer to your video buffer when GetBytes() is called.
\r
619 This may help to keep copying to a minimum. Please ensure that the pixel
\r
620 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
\r
621 have to colourspace convert which may result in additional copying.
\r
622 ################################################################################
\r