2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
\r
4 * This file is part of CasparCG (www.casparcg.com).
\r
6 * CasparCG is free software: you can redistribute it and/or modify
\r
7 * it under the terms of the GNU General Public License as published by
\r
8 * the Free Software Foundation, either version 3 of the License, or
\r
9 * (at your option) any later version.
\r
11 * CasparCG is distributed in the hope that it will be useful,
\r
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
\r
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
\r
14 * GNU General Public License for more details.
\r
16 * You should have received a copy of the GNU General Public License
\r
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
\r
19 * Author: Robert Nagy, ronag89@gmail.com
\r
22 #include "../StdAfx.h"
\r
24 #include "decklink_consumer.h"
\r
26 #include "../util/util.h"
\r
28 #include "../interop/DeckLinkAPI_h.h"
\r
30 #include <core/mixer/read_frame.h>
\r
32 #include <common/concurrency/executor.h>
\r
33 #include <common/concurrency/lock.h>
\r
34 #include <common/diagnostics/graph.h>
\r
35 #include <common/exception/exceptions.h>
\r
36 #include <common/memory/memshfl.h>
\r
38 #include <core/consumer/frame_consumer.h>
\r
40 #include <tbb/concurrent_queue.h>
\r
41 #include <tbb/cache_aligned_allocator.h>
\r
43 #include <common/assert.h>
\r
44 #include <boost/lexical_cast.hpp>
\r
45 #include <boost/circular_buffer.hpp>
\r
46 #include <boost/timer.hpp>
\r
47 #include <boost/property_tree/ptree.hpp>
\r
49 namespace caspar { namespace decklink {
\r
51 struct configuration
\r
54 bool embedded_audio;
\r
58 int base_buffer_depth;
\r
63 , embedded_audio(false)
\r
64 , internal_key(false)
\r
67 , base_buffer_depth(3)
\r
68 , buffer_depth(base_buffer_depth + (low_latency ? 0 : 1) + (embedded_audio ? 1 : 0)){}
\r
71 class decklink_frame : public IDeckLinkVideoFrame
\r
73 tbb::atomic<int> ref_count_;
\r
74 std::shared_ptr<core::read_frame> frame_;
\r
75 const core::video_format_desc format_desc_;
\r
77 const bool key_only_;
\r
78 std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>> data_;
\r
80 decklink_frame(const safe_ptr<core::read_frame>& frame, const core::video_format_desc& format_desc, bool key_only)
\r
82 , format_desc_(format_desc)
\r
83 , key_only_(key_only)
\r
90 STDMETHOD (QueryInterface(REFIID, LPVOID*))
\r
92 return E_NOINTERFACE;
\r
95 STDMETHOD_(ULONG, AddRef())
\r
97 return ++ref_count_;
\r
100 STDMETHOD_(ULONG, Release())
\r
102 if(--ref_count_ == 0)
\r
107 // IDecklinkVideoFrame
\r
109 STDMETHOD_(long, GetWidth()) {return format_desc_.width;}
\r
110 STDMETHOD_(long, GetHeight()) {return format_desc_.height;}
\r
111 STDMETHOD_(long, GetRowBytes()) {return format_desc_.width*4;}
\r
112 STDMETHOD_(BMDPixelFormat, GetPixelFormat()) {return bmdFormat8BitBGRA;}
\r
113 STDMETHOD_(BMDFrameFlags, GetFlags()) {return bmdFrameFlagDefault;}
\r
115 STDMETHOD(GetBytes(void** buffer))
\r
119 if(static_cast<int>(frame_->image_data().size()) != format_desc_.size)
\r
121 data_.resize(format_desc_.size, 0);
\r
122 *buffer = data_.data();
\r
128 data_.resize(frame_->image_data().size());
\r
129 aligned_memshfl(data_.data(), frame_->image_data().begin(), frame_->image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
\r
131 *buffer = data_.data();
\r
134 *buffer = const_cast<uint8_t*>(frame_->image_data().begin());
\r
138 CASPAR_LOG_CURRENT_EXCEPTION();
\r
145 STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}
\r
146 STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)) {return S_FALSE;}
\r
150 const boost::iterator_range<const int32_t*> audio_data()
\r
152 return frame_->audio_data();
\r
156 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
\r
158 const int channel_index_;
\r
159 const configuration config_;
\r
161 CComPtr<IDeckLink> decklink_;
\r
162 CComQIPtr<IDeckLinkOutput> output_;
\r
163 CComQIPtr<IDeckLinkConfiguration> configuration_;
\r
164 CComQIPtr<IDeckLinkKeyer> keyer_;
\r
166 tbb::spin_mutex exception_mutex_;
\r
167 std::exception_ptr exception_;
\r
169 tbb::atomic<bool> is_running_;
\r
171 const std::wstring model_name_;
\r
172 const core::video_format_desc format_desc_;
\r
173 const int buffer_size_;
\r
175 long long video_scheduled_;
\r
176 long long audio_scheduled_;
\r
178 int preroll_count_;
\r
180 boost::circular_buffer<std::vector<int32_t>> audio_container_;
\r
182 tbb::concurrent_bounded_queue<std::shared_ptr<core::read_frame>> video_frame_buffer_;
\r
183 tbb::concurrent_bounded_queue<std::shared_ptr<core::read_frame>> audio_frame_buffer_;
\r
185 safe_ptr<diagnostics::graph> graph_;
\r
186 boost::timer tick_timer_;
\r
189 decklink_consumer(const configuration& config, const core::video_format_desc& format_desc, int channel_index)
\r
190 : channel_index_(channel_index)
\r
192 , decklink_(get_device(config.device_index))
\r
193 , output_(decklink_)
\r
194 , configuration_(decklink_)
\r
195 , keyer_(decklink_)
\r
196 , model_name_(get_model_name(decklink_))
\r
197 , format_desc_(format_desc)
\r
198 , buffer_size_(config.buffer_depth) // Minimum buffer-size 3.
\r
199 , video_scheduled_(0)
\r
200 , audio_scheduled_(0)
\r
201 , preroll_count_(0)
\r
202 , audio_container_(buffer_size_+1)
\r
204 is_running_ = true;
\r
206 video_frame_buffer_.set_capacity(1);
\r
207 audio_frame_buffer_.set_capacity(1);
\r
209 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
\r
210 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
\r
211 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
\r
212 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
\r
213 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
\r
214 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
\r
215 graph_->set_text(print());
\r
216 diagnostics::register_graph(graph_);
\r
218 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));
\r
220 if(config.embedded_audio)
\r
223 set_latency(config.low_latency);
\r
224 set_keyer(config.internal_key);
\r
226 if(config.embedded_audio)
\r
227 output_->BeginAudioPreroll();
\r
229 for(int n = 0; n < buffer_size_; ++n)
\r
230 schedule_next_video(make_safe<core::read_frame>());
\r
232 if(!config.embedded_audio)
\r
236 ~decklink_consumer()
\r
238 is_running_ = false;
\r
239 video_frame_buffer_.try_push(std::make_shared<core::read_frame>());
\r
240 audio_frame_buffer_.try_push(std::make_shared<core::read_frame>());
\r
242 if(output_ != nullptr)
\r
244 output_->StopScheduledPlayback(0, nullptr, 0);
\r
245 if(config_.embedded_audio)
\r
246 output_->DisableAudioOutput();
\r
247 output_->DisableVideoOutput();
\r
251 void set_latency(bool low_latency)
\r
255 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
\r
256 CASPAR_LOG(info) << print() << L" Enabled normal-latency mode";
\r
260 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
\r
261 CASPAR_LOG(info) << print() << L" Enabled low-latency mode";
\r
265 void set_keyer(bool internal_key)
\r
269 if(FAILED(keyer_->Enable(FALSE)))
\r
270 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
\r
271 else if(FAILED(keyer_->SetLevel(255)))
\r
272 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
274 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";
\r
278 if(FAILED(keyer_->Enable(TRUE)))
\r
279 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
\r
280 else if(FAILED(keyer_->SetLevel(255)))
\r
281 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
283 CASPAR_LOG(info) << print() << L" Enabled external keyer.";
\r
287 void enable_audio()
\r
289 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped)))
\r
290 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable audio output."));
\r
292 if(FAILED(output_->SetAudioCallback(this)))
\r
293 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not set audio callback."));
\r
295 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
\r
298 void enable_video(BMDDisplayMode display_mode)
\r
300 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
\r
301 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable video output."));
\r
303 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
\r
304 BOOST_THROW_EXCEPTION(caspar_exception()
\r
305 << msg_info(u8(print()) + " Failed to set playback completion callback.")
\r
306 << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
\r
309 void start_playback()
\r
311 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
\r
312 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Failed to schedule playback."));
\r
315 STDMETHOD (QueryInterface(REFIID, LPVOID*)) {return E_NOINTERFACE;}
\r
316 STDMETHOD_(ULONG, AddRef()) {return 1;}
\r
317 STDMETHOD_(ULONG, Release()) {return 1;}
\r
319 STDMETHOD(ScheduledPlaybackHasStopped())
\r
321 is_running_ = false;
\r
322 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
\r
326 STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))
\r
333 if(result == bmdOutputFrameDisplayedLate)
\r
335 graph_->set_tag("late-frame");
\r
336 video_scheduled_ += format_desc_.duration;
\r
337 audio_scheduled_ += reinterpret_cast<decklink_frame*>(completed_frame)->audio_data().size()/format_desc_.audio_channels;
\r
338 //++video_scheduled_;
\r
339 //audio_scheduled_ += format_desc_.audio_cadence[0];
\r
340 //++audio_scheduled_;
\r
342 else if(result == bmdOutputFrameDropped)
\r
343 graph_->set_tag("dropped-frame");
\r
344 else if(result == bmdOutputFrameFlushed)
\r
345 graph_->set_tag("flushed-frame");
\r
347 std::shared_ptr<core::read_frame> frame;
\r
348 video_frame_buffer_.pop(frame);
\r
349 schedule_next_video(make_safe_ptr(frame));
\r
351 unsigned long buffered;
\r
352 output_->GetBufferedVideoFrameCount(&buffered);
\r
353 graph_->set_value("buffered-video", static_cast<double>(buffered)/format_desc_.fps);
\r
357 lock(exception_mutex_, [&]
\r
359 exception_ = std::current_exception();
\r
367 STDMETHOD(RenderAudioSamples(BOOL preroll))
\r
376 if(++preroll_count_ >= buffer_size_)
\r
378 output_->EndAudioPreroll();
\r
382 schedule_next_audio(core::audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()], 0));
\r
386 std::shared_ptr<core::read_frame> frame;
\r
387 audio_frame_buffer_.pop(frame);
\r
388 schedule_next_audio(frame->audio_data());
\r
391 unsigned long buffered;
\r
392 output_->GetBufferedAudioSampleFrameCount(&buffered);
\r
393 graph_->set_value("buffered-audio", static_cast<double>(buffered)/(format_desc_.audio_cadence[0]*2));
\r
397 tbb::spin_mutex::scoped_lock lock(exception_mutex_);
\r
398 exception_ = std::current_exception();
\r
405 template<typename T>
\r
406 void schedule_next_audio(const T& audio_data)
\r
408 auto sample_frame_count = static_cast<int>(audio_data.size()/format_desc_.audio_channels);
\r
410 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
\r
412 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
\r
413 CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
\r
415 audio_scheduled_ += sample_frame_count;
\r
418 void schedule_next_video(const safe_ptr<core::read_frame>& frame)
\r
420 CComPtr<IDeckLinkVideoFrame> frame2(new decklink_frame(frame, format_desc_, config_.key_only));
\r
421 if(FAILED(output_->ScheduleVideoFrame(frame2, video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
\r
422 CASPAR_LOG(error) << print() << L" Failed to schedule video.";
\r
424 video_scheduled_ += format_desc_.duration;
\r
426 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
\r
427 tick_timer_.restart();
\r
430 void send(const safe_ptr<core::read_frame>& frame)
\r
432 auto exception = lock(exception_mutex_, [&]
\r
437 if(exception != nullptr)
\r
438 std::rethrow_exception(exception);
\r
441 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Is not running."));
\r
443 if(config_.embedded_audio)
\r
444 audio_frame_buffer_.push(frame);
\r
445 video_frame_buffer_.push(frame);
\r
448 std::wstring print() const
\r
450 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_) + L"-" +
\r
451 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
\r
455 struct decklink_consumer_proxy : public core::frame_consumer
\r
457 const configuration config_;
\r
458 std::unique_ptr<decklink_consumer> consumer_;
\r
459 std::vector<int> audio_cadence_;
\r
460 executor executor_;
\r
463 decklink_consumer_proxy(const configuration& config)
\r
465 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
\r
467 executor_.begin_invoke([=]
\r
469 ::CoInitialize(nullptr);
\r
473 ~decklink_consumer_proxy()
\r
475 executor_.invoke([=]
\r
479 auto str = print();
\r
481 CASPAR_LOG(info) << str << L" Successfully Uninitialized.";
\r
483 ::CoUninitialize();
\r
489 virtual void initialize(const core::video_format_desc& format_desc, int channel_index) override
\r
491 executor_.invoke([=]
\r
493 consumer_.reset(new decklink_consumer(config_, format_desc, channel_index));
\r
494 audio_cadence_ = format_desc.audio_cadence;
\r
496 CASPAR_LOG(info) << print() << L" Successfully Initialized.";
\r
500 virtual bool send(const safe_ptr<core::read_frame>& frame) override
\r
502 CASPAR_VERIFY(audio_cadence_.front() == static_cast<int>(frame->audio_data().size()));
\r
503 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1);
\r
505 consumer_->send(frame);
\r
509 virtual std::wstring print() const override
\r
511 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
\r
514 virtual boost::property_tree::wptree info() const override
\r
516 boost::property_tree::wptree info;
\r
517 info.add(L"type", L"decklink-consumer");
\r
518 info.add(L"key-only", config_.key_only);
\r
519 info.add(L"device", config_.device_index);
\r
520 info.add(L"low-latency", config_.low_latency);
\r
521 info.add(L"embedded-audio", config_.embedded_audio);
\r
522 info.add(L"low-latency", config_.low_latency);
\r
523 info.add(L"internal-key", config_.internal_key);
\r
527 virtual int buffer_depth() const override
\r
529 return config_.buffer_depth;
\r
532 virtual int index() const override
\r
534 return 300 + config_.device_index;
\r
538 safe_ptr<core::frame_consumer> create_consumer(const std::vector<std::wstring>& params)
\r
540 if(params.size() < 1 || params[0] != L"DECKLINK")
\r
541 return core::frame_consumer::empty();
\r
543 configuration config;
\r
545 if(params.size() > 1)
\r
546 config.device_index = boost::lexical_cast<int>(params[1]);
\r
548 config.internal_key = std::find(params.begin(), params.end(), L"INTERNAL_KEY") != params.end();
\r
549 config.low_latency = std::find(params.begin(), params.end(), L"LOW_LATENCY") != params.end();
\r
550 config.embedded_audio = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();
\r
551 config.key_only = std::find(params.begin(), params.end(), L"KEY_ONLY") != params.end();
\r
553 return make_safe<decklink_consumer_proxy>(config);
\r
556 safe_ptr<core::frame_consumer> create_consumer(const boost::property_tree::wptree& ptree)
\r
558 configuration config;
\r
560 config.internal_key = ptree.get(L"internal-key", config.internal_key);
\r
561 config.low_latency = ptree.get(L"low-latency", config.low_latency);
\r
562 config.key_only = ptree.get(L"key-only", config.key_only);
\r
563 config.device_index = ptree.get(L"device", config.device_index);
\r
564 config.embedded_audio = ptree.get(L"embedded-audio", config.embedded_audio);
\r
565 config.base_buffer_depth = ptree.get(L"buffer-depth", config.base_buffer_depth);
\r
567 return make_safe<decklink_consumer_proxy>(config);
\r
573 ##############################################################################
\r
579 BMD Developer Support
\r
580 developer@blackmagic-design.com
\r
582 -----------------------------------------------------------------------------
\r
584 Thanks for your inquiry. The minimum number of frames that you can preroll
\r
585 for scheduled playback is three frames for video and four frames for audio.
\r
586 As you mentioned if you preroll less frames then playback will not start or
\r
587 playback will be very sporadic. From our experience with Media Express, we
\r
588 recommended that at least seven frames are prerolled for smooth playback.
\r
590 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
\r
591 There can be around 3 frames worth of latency on scheduled output.
\r
592 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
\r
593 reduced or removed for scheduled playback. If the DisplayVideoFrameSync()
\r
594 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
\r
595 guarantee that the provided frame will be output as soon the previous
\r
596 frame output has been completed.
\r
597 ################################################################################
\r
601 ##############################################################################
\r
602 Async DMA Transfer without redundant copying
\r
607 BMD Developer Support
\r
608 developer@blackmagic-design.com
\r
610 -----------------------------------------------------------------------------
\r
612 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
\r
613 and providing a pointer to your video buffer when GetBytes() is called.
\r
614 This may help to keep copying to a minimum. Please ensure that the pixel
\r
615 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
\r
616 have to colourspace convert which may result in additional copying.
\r
617 ################################################################################
\r