2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
\r
4 * This file is part of CasparCG (www.casparcg.com).
\r
6 * CasparCG is free software: you can redistribute it and/or modify
\r
7 * it under the terms of the GNU General Public License as published by
\r
8 * the Free Software Foundation, either version 3 of the License, or
\r
9 * (at your option) any later version.
\r
11 * CasparCG is distributed in the hope that it will be useful,
\r
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
\r
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
\r
14 * GNU General Public License for more details.
\r
16 * You should have received a copy of the GNU General Public License
\r
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
\r
19 * Author: Robert Nagy, ronag89@gmail.com
\r
22 #include "../StdAfx.h"
\r
24 #include "decklink_consumer.h"
\r
26 #include "../util/util.h"
\r
28 #include "../interop/DeckLinkAPI_h.h"
\r
30 #include <core/mixer/read_frame.h>
\r
32 #include <common/concurrency/executor.h>
\r
33 #include <common/concurrency/lock.h>
\r
34 #include <common/diagnostics/graph.h>
\r
35 #include <common/exception/exceptions.h>
\r
36 #include <common/memory/memshfl.h>
\r
38 #include <core/consumer/frame_consumer.h>
\r
40 #include <tbb/concurrent_queue.h>
\r
41 #include <tbb/cache_aligned_allocator.h>
\r
43 #include <boost/assert.hpp>
\r
44 #include <boost/lexical_cast.hpp>
\r
45 #include <boost/circular_buffer.hpp>
\r
46 #include <boost/timer.hpp>
\r
47 #include <boost/property_tree/ptree.hpp>
\r
49 namespace caspar { namespace decklink {
\r
51 struct configuration
\r
54 bool embedded_audio;
\r
58 int base_buffer_depth;
\r
63 , embedded_audio(false)
\r
64 , internal_key(false)
\r
67 , base_buffer_depth(3)
\r
68 , buffer_depth(base_buffer_depth + (low_latency ? 0 : 1) + (embedded_audio ? 1 : 0)){}
\r
71 class decklink_frame : public IDeckLinkVideoFrame
\r
73 tbb::atomic<int> ref_count_;
\r
74 std::shared_ptr<core::read_frame> frame_;
\r
75 const core::video_format_desc format_desc_;
\r
77 const bool key_only_;
\r
78 std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>> data_;
\r
80 decklink_frame(const safe_ptr<core::read_frame>& frame, const core::video_format_desc& format_desc, bool key_only)
\r
82 , format_desc_(format_desc)
\r
83 , key_only_(key_only)
\r
90 STDMETHOD (QueryInterface(REFIID, LPVOID*))
\r
92 return E_NOINTERFACE;
\r
95 STDMETHOD_(ULONG, AddRef())
\r
97 return ++ref_count_;
\r
100 STDMETHOD_(ULONG, Release())
\r
102 if(--ref_count_ == 0)
\r
107 // IDecklinkVideoFrame
\r
109 STDMETHOD_(long, GetWidth()) {return format_desc_.width;}
\r
110 STDMETHOD_(long, GetHeight()) {return format_desc_.height;}
\r
111 STDMETHOD_(long, GetRowBytes()) {return format_desc_.width*4;}
\r
112 STDMETHOD_(BMDPixelFormat, GetPixelFormat()) {return bmdFormat8BitBGRA;}
\r
113 STDMETHOD_(BMDFrameFlags, GetFlags()) {return bmdFrameFlagDefault;}
\r
115 STDMETHOD(GetBytes(void** buffer))
\r
119 if(static_cast<int>(frame_->image_data().size()) != format_desc_.size)
\r
121 data_.resize(format_desc_.size, 0);
\r
122 *buffer = data_.data();
\r
128 data_.resize(frame_->image_data().size());
\r
129 aligned_memshfl(data_.data(), frame_->image_data().begin(), frame_->image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
\r
132 *buffer = data_.data();
\r
135 *buffer = const_cast<uint8_t*>(frame_->image_data().begin());
\r
139 CASPAR_LOG_CURRENT_EXCEPTION();
\r
146 STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}
\r
147 STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)) {return S_FALSE;}
\r
151 const boost::iterator_range<const int32_t*> audio_data()
\r
153 return frame_->audio_data();
\r
157 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
\r
159 const int channel_index_;
\r
160 const configuration config_;
\r
162 CComPtr<IDeckLink> decklink_;
\r
163 CComQIPtr<IDeckLinkOutput> output_;
\r
164 CComQIPtr<IDeckLinkConfiguration> configuration_;
\r
165 CComQIPtr<IDeckLinkKeyer> keyer_;
\r
167 tbb::spin_mutex exception_mutex_;
\r
168 std::exception_ptr exception_;
\r
170 tbb::atomic<bool> is_running_;
\r
172 const std::wstring model_name_;
\r
173 const core::video_format_desc format_desc_;
\r
174 const int buffer_size_;
\r
176 long long video_scheduled_;
\r
177 long long audio_scheduled_;
\r
179 int preroll_count_;
\r
181 boost::circular_buffer<std::vector<int32_t>> audio_container_;
\r
183 tbb::concurrent_bounded_queue<std::shared_ptr<core::read_frame>> video_frame_buffer_;
\r
184 tbb::concurrent_bounded_queue<std::shared_ptr<core::read_frame>> audio_frame_buffer_;
\r
186 safe_ptr<diagnostics::graph> graph_;
\r
187 boost::timer tick_timer_;
\r
190 decklink_consumer(const configuration& config, const core::video_format_desc& format_desc, int channel_index)
\r
191 : channel_index_(channel_index)
\r
193 , decklink_(get_device(config.device_index))
\r
194 , output_(decklink_)
\r
195 , configuration_(decklink_)
\r
196 , keyer_(decklink_)
\r
197 , model_name_(get_model_name(decklink_))
\r
198 , format_desc_(format_desc)
\r
199 , buffer_size_(config.buffer_depth) // Minimum buffer-size 3.
\r
200 , video_scheduled_(0)
\r
201 , audio_scheduled_(0)
\r
202 , preroll_count_(0)
\r
203 , audio_container_(buffer_size_+1)
\r
205 is_running_ = true;
\r
207 video_frame_buffer_.set_capacity(1);
\r
208 audio_frame_buffer_.set_capacity(1);
\r
210 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
\r
211 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
\r
212 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
\r
213 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
\r
214 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
\r
215 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
\r
216 graph_->set_text(print());
\r
217 diagnostics::register_graph(graph_);
\r
219 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));
\r
221 if(config.embedded_audio)
\r
224 set_latency(config.low_latency);
\r
225 set_keyer(config.internal_key);
\r
227 if(config.embedded_audio)
\r
228 output_->BeginAudioPreroll();
\r
230 for(int n = 0; n < buffer_size_; ++n)
\r
231 schedule_next_video(make_safe<core::read_frame>());
\r
233 if(!config.embedded_audio)
\r
237 ~decklink_consumer()
\r
239 is_running_ = false;
\r
240 video_frame_buffer_.try_push(std::make_shared<core::read_frame>());
\r
241 audio_frame_buffer_.try_push(std::make_shared<core::read_frame>());
\r
243 if(output_ != nullptr)
\r
245 output_->StopScheduledPlayback(0, nullptr, 0);
\r
246 if(config_.embedded_audio)
\r
247 output_->DisableAudioOutput();
\r
248 output_->DisableVideoOutput();
\r
252 void set_latency(bool low_latency)
\r
256 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
\r
257 CASPAR_LOG(info) << print() << L" Enabled normal-latency mode";
\r
261 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
\r
262 CASPAR_LOG(info) << print() << L" Enabled low-latency mode";
\r
266 void set_keyer(bool internal_key)
\r
270 if(FAILED(keyer_->Enable(FALSE)))
\r
271 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
\r
272 else if(FAILED(keyer_->SetLevel(255)))
\r
273 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
275 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";
\r
279 if(FAILED(keyer_->Enable(TRUE)))
\r
280 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
\r
281 else if(FAILED(keyer_->SetLevel(255)))
\r
282 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
284 CASPAR_LOG(info) << print() << L" Enabled external keyer.";
\r
288 void enable_audio()
\r
290 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped)))
\r
291 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable audio output."));
\r
293 if(FAILED(output_->SetAudioCallback(this)))
\r
294 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not set audio callback."));
\r
296 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
\r
299 void enable_video(BMDDisplayMode display_mode)
\r
301 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
\r
302 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable video output."));
\r
304 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
\r
305 BOOST_THROW_EXCEPTION(caspar_exception()
\r
306 << msg_info(u8(print()) + " Failed to set playback completion callback.")
\r
307 << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
\r
310 void start_playback()
\r
312 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
\r
313 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Failed to schedule playback."));
\r
316 STDMETHOD (QueryInterface(REFIID, LPVOID*)) {return E_NOINTERFACE;}
\r
317 STDMETHOD_(ULONG, AddRef()) {return 1;}
\r
318 STDMETHOD_(ULONG, Release()) {return 1;}
\r
320 STDMETHOD(ScheduledPlaybackHasStopped())
\r
322 is_running_ = false;
\r
323 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
\r
327 STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))
\r
334 if(result == bmdOutputFrameDisplayedLate)
\r
336 graph_->set_tag("late-frame");
\r
337 video_scheduled_ += format_desc_.duration;
\r
338 audio_scheduled_ += reinterpret_cast<decklink_frame*>(completed_frame)->audio_data().size()/format_desc_.audio_channels;
\r
339 //++video_scheduled_;
\r
340 //audio_scheduled_ += format_desc_.audio_cadence[0];
\r
341 //++audio_scheduled_;
\r
343 else if(result == bmdOutputFrameDropped)
\r
344 graph_->set_tag("dropped-frame");
\r
345 else if(result == bmdOutputFrameFlushed)
\r
346 graph_->set_tag("flushed-frame");
\r
348 std::shared_ptr<core::read_frame> frame;
\r
349 video_frame_buffer_.pop(frame);
\r
350 schedule_next_video(make_safe_ptr(frame));
\r
352 unsigned long buffered;
\r
353 output_->GetBufferedVideoFrameCount(&buffered);
\r
354 graph_->set_value("buffered-video", static_cast<double>(buffered)/format_desc_.fps);
\r
358 lock(exception_mutex_, [&]
\r
360 exception_ = std::current_exception();
\r
368 STDMETHOD(RenderAudioSamples(BOOL preroll))
\r
377 if(++preroll_count_ >= buffer_size_)
\r
379 output_->EndAudioPreroll();
\r
383 schedule_next_audio(core::audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()], 0));
\r
387 std::shared_ptr<core::read_frame> frame;
\r
388 audio_frame_buffer_.pop(frame);
\r
389 schedule_next_audio(frame->audio_data());
\r
392 unsigned long buffered;
\r
393 output_->GetBufferedAudioSampleFrameCount(&buffered);
\r
394 graph_->set_value("buffered-audio", static_cast<double>(buffered)/(format_desc_.audio_cadence[0]*2));
\r
398 tbb::spin_mutex::scoped_lock lock(exception_mutex_);
\r
399 exception_ = std::current_exception();
\r
406 template<typename T>
\r
407 void schedule_next_audio(const T& audio_data)
\r
409 auto sample_frame_count = static_cast<int>(audio_data.size()/format_desc_.audio_channels);
\r
411 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
\r
413 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
\r
414 CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
\r
416 audio_scheduled_ += sample_frame_count;
\r
419 void schedule_next_video(const safe_ptr<core::read_frame>& frame)
\r
421 CComPtr<IDeckLinkVideoFrame> frame2(new decklink_frame(frame, format_desc_, config_.key_only));
\r
422 if(FAILED(output_->ScheduleVideoFrame(frame2, video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
\r
423 CASPAR_LOG(error) << print() << L" Failed to schedule video.";
\r
425 video_scheduled_ += format_desc_.duration;
\r
427 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
\r
428 tick_timer_.restart();
\r
431 void send(const safe_ptr<core::read_frame>& frame)
\r
433 auto exception = lock(exception_mutex_, [&]
\r
438 if(exception != nullptr)
\r
439 std::rethrow_exception(exception);
\r
442 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Is not running."));
\r
444 if(config_.embedded_audio)
\r
445 audio_frame_buffer_.push(frame);
\r
446 video_frame_buffer_.push(frame);
\r
449 std::wstring print() const
\r
451 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_) + L"-" +
\r
452 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
\r
456 struct decklink_consumer_proxy : public core::frame_consumer
\r
458 const configuration config_;
\r
459 std::unique_ptr<decklink_consumer> consumer_;
\r
460 std::vector<int> audio_cadence_;
\r
461 executor executor_;
\r
464 decklink_consumer_proxy(const configuration& config)
\r
466 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
\r
468 executor_.begin_invoke([=]
\r
470 ::CoInitialize(nullptr);
\r
474 ~decklink_consumer_proxy()
\r
476 executor_.invoke([=]
\r
480 auto str = print();
\r
482 CASPAR_LOG(info) << str << L" Successfully Uninitialized.";
\r
484 ::CoUninitialize();
\r
490 virtual void initialize(const core::video_format_desc& format_desc, int channel_index) override
\r
492 executor_.invoke([=]
\r
494 consumer_.reset(new decklink_consumer(config_, format_desc, channel_index));
\r
495 audio_cadence_ = format_desc.audio_cadence;
\r
497 CASPAR_LOG(info) << print() << L" Successfully Initialized.";
\r
501 virtual bool send(const safe_ptr<core::read_frame>& frame) override
\r
503 BOOST_VERIFY(audio_cadence_.front() == static_cast<int>(frame->audio_data().size()));
\r
504 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1);
\r
506 consumer_->send(frame);
\r
510 virtual std::wstring print() const override
\r
512 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
\r
515 virtual boost::property_tree::wptree info() const override
\r
517 boost::property_tree::wptree info;
\r
518 info.add(L"type", L"decklink-consumer");
\r
519 info.add(L"key-only", config_.key_only);
\r
520 info.add(L"device", config_.device_index);
\r
521 info.add(L"low-latency", config_.low_latency);
\r
522 info.add(L"embedded-audio", config_.embedded_audio);
\r
523 info.add(L"low-latency", config_.low_latency);
\r
524 info.add(L"internal-key", config_.internal_key);
\r
528 virtual int buffer_depth() const override
\r
530 return config_.buffer_depth;
\r
533 virtual int index() const override
\r
535 return 300 + config_.device_index;
\r
539 safe_ptr<core::frame_consumer> create_consumer(const std::vector<std::wstring>& params)
\r
541 if(params.size() < 1 || params[0] != L"DECKLINK")
\r
542 return core::frame_consumer::empty();
\r
544 configuration config;
\r
546 if(params.size() > 1)
\r
547 config.device_index = boost::lexical_cast<int>(params[1]);
\r
549 config.internal_key = std::find(params.begin(), params.end(), L"INTERNAL_KEY") != params.end();
\r
550 config.low_latency = std::find(params.begin(), params.end(), L"LOW_LATENCY") != params.end();
\r
551 config.embedded_audio = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();
\r
552 config.key_only = std::find(params.begin(), params.end(), L"KEY_ONLY") != params.end();
\r
554 return make_safe<decklink_consumer_proxy>(config);
\r
557 safe_ptr<core::frame_consumer> create_consumer(const boost::property_tree::wptree& ptree)
\r
559 configuration config;
\r
561 config.internal_key = ptree.get(L"internal-key", config.internal_key);
\r
562 config.low_latency = ptree.get(L"low-latency", config.low_latency);
\r
563 config.key_only = ptree.get(L"key-only", config.key_only);
\r
564 config.device_index = ptree.get(L"device", config.device_index);
\r
565 config.embedded_audio = ptree.get(L"embedded-audio", config.embedded_audio);
\r
566 config.base_buffer_depth = ptree.get(L"buffer-depth", config.base_buffer_depth);
\r
568 return make_safe<decklink_consumer_proxy>(config);
\r
574 ##############################################################################
\r
580 BMD Developer Support
\r
581 developer@blackmagic-design.com
\r
583 -----------------------------------------------------------------------------
\r
585 Thanks for your inquiry. The minimum number of frames that you can preroll
\r
586 for scheduled playback is three frames for video and four frames for audio.
\r
587 As you mentioned if you preroll less frames then playback will not start or
\r
588 playback will be very sporadic. From our experience with Media Express, we
\r
589 recommended that at least seven frames are prerolled for smooth playback.
\r
591 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
\r
592 There can be around 3 frames worth of latency on scheduled output.
\r
593 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
\r
594 reduced or removed for scheduled playback. If the DisplayVideoFrameSync()
\r
595 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
\r
596 guarantee that the provided frame will be output as soon the previous
\r
597 frame output has been completed.
\r
598 ################################################################################
\r
602 ##############################################################################
\r
603 Async DMA Transfer without redundant copying
\r
608 BMD Developer Support
\r
609 developer@blackmagic-design.com
\r
611 -----------------------------------------------------------------------------
\r
613 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
\r
614 and providing a pointer to your video buffer when GetBytes() is called.
\r
615 This may help to keep copying to a minimum. Please ensure that the pixel
\r
616 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
\r
617 have to colourspace convert which may result in additional copying.
\r
618 ################################################################################
\r