2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
\r
4 * This file is part of CasparCG (www.casparcg.com).
\r
6 * CasparCG is free software: you can redistribute it and/or modify
\r
7 * it under the terms of the GNU General Public License as published by
\r
8 * the Free Software Foundation, either version 3 of the License, or
\r
9 * (at your option) any later version.
\r
11 * CasparCG is distributed in the hope that it will be useful,
\r
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
\r
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
\r
14 * GNU General Public License for more details.
\r
16 * You should have received a copy of the GNU General Public License
\r
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
\r
19 * Author: Robert Nagy, ronag89@gmail.com
\r
22 #include "../StdAfx.h"
\r
24 #include "decklink_consumer.h"
\r
26 #include "../util/util.h"
\r
28 #include "../interop/DeckLinkAPI_h.h"
\r
30 #include <core/mixer/read_frame.h>
\r
32 #include <common/concurrency/com_context.h>
\r
33 #include <common/diagnostics/graph.h>
\r
34 #include <common/exception/exceptions.h>
\r
35 #include <common/memory/memshfl.h>
\r
36 #include <common/utility/assert.h>
\r
38 #include <core/consumer/frame_consumer.h>
\r
40 #include <tbb/concurrent_queue.h>
\r
41 #include <tbb/cache_aligned_allocator.h>
\r
43 #include <boost/circular_buffer.hpp>
\r
44 #include <boost/timer.hpp>
\r
45 #include <boost/property_tree/ptree.hpp>
\r
47 namespace caspar { namespace decklink {
\r
49 struct configuration
\r
52 bool embedded_audio;
\r
56 int base_buffer_depth;
\r
61 , embedded_audio(false)
\r
62 , internal_key(false)
\r
65 , base_buffer_depth(3)
\r
66 , buffer_depth(base_buffer_depth + (low_latency ? 0 : 1) + (embedded_audio ? 1 : 0)){}
\r
69 class decklink_frame : public IDeckLinkVideoFrame
\r
71 tbb::atomic<int> ref_count_;
\r
72 std::shared_ptr<core::read_frame> frame_;
\r
73 const core::video_format_desc format_desc_;
\r
75 const bool key_only_;
\r
76 std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>> key_data_;
\r
78 decklink_frame(const safe_ptr<core::read_frame>& frame, const core::video_format_desc& format_desc, bool key_only)
\r
80 , format_desc_(format_desc)
\r
81 , key_only_(key_only)
\r
86 const boost::iterator_range<const int32_t*> audio_data()
\r
88 return frame_->audio_data();
\r
91 STDMETHOD (QueryInterface(REFIID, LPVOID*)) {return E_NOINTERFACE;}
\r
92 STDMETHOD_(ULONG, AddRef())
\r
94 return ++ref_count_;
\r
96 STDMETHOD_(ULONG, Release())
\r
104 STDMETHOD_(long, GetWidth()) {return format_desc_.width;}
\r
105 STDMETHOD_(long, GetHeight()) {return format_desc_.height;}
\r
106 STDMETHOD_(long, GetRowBytes()) {return format_desc_.width*4;}
\r
107 STDMETHOD_(BMDPixelFormat, GetPixelFormat()) {return bmdFormat8BitBGRA;}
\r
108 STDMETHOD_(BMDFrameFlags, GetFlags()) {return bmdFrameFlagDefault;}
\r
110 STDMETHOD(GetBytes(void** buffer))
\r
112 static std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>> zeros(1920*1080*4, 0);
\r
113 if(static_cast<size_t>(frame_->image_data().size()) != format_desc_.size)
\r
115 *buffer = zeros.data();
\r
120 *buffer = const_cast<uint8_t*>(frame_->image_data().begin());
\r
123 if(key_data_.empty())
\r
125 key_data_.resize(frame_->image_data().size());
\r
126 aligned_memshfl(key_data_.data(), frame_->image_data().begin(), frame_->image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
\r
129 *buffer = key_data_.data();
\r
135 STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}
\r
136 STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)) {return S_FALSE;}
\r
139 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
\r
141 const int channel_index_;
\r
142 const configuration config_;
\r
144 CComPtr<IDeckLink> decklink_;
\r
145 CComQIPtr<IDeckLinkOutput> output_;
\r
146 CComQIPtr<IDeckLinkConfiguration> configuration_;
\r
147 CComQIPtr<IDeckLinkKeyer> keyer_;
\r
149 tbb::spin_mutex exception_mutex_;
\r
150 std::exception_ptr exception_;
\r
152 tbb::atomic<bool> is_running_;
\r
154 const std::wstring model_name_;
\r
155 const core::video_format_desc format_desc_;
\r
156 const size_t buffer_size_;
\r
158 long long video_scheduled_;
\r
159 long long audio_scheduled_;
\r
161 size_t preroll_count_;
\r
163 boost::circular_buffer<std::vector<int32_t>> audio_container_;
\r
165 tbb::concurrent_bounded_queue<std::shared_ptr<core::read_frame>> video_frame_buffer_;
\r
166 tbb::concurrent_bounded_queue<std::shared_ptr<core::read_frame>> audio_frame_buffer_;
\r
168 safe_ptr<diagnostics::graph> graph_;
\r
169 boost::timer tick_timer_;
\r
172 decklink_consumer(const configuration& config, const core::video_format_desc& format_desc, int channel_index)
\r
173 : channel_index_(channel_index)
\r
175 , decklink_(get_device(config.device_index))
\r
176 , output_(decklink_)
\r
177 , configuration_(decklink_)
\r
178 , keyer_(decklink_)
\r
179 , model_name_(get_model_name(decklink_))
\r
180 , format_desc_(format_desc)
\r
181 , buffer_size_(config.buffer_depth) // Minimum buffer-size 3.
\r
182 , video_scheduled_(0)
\r
183 , audio_scheduled_(0)
\r
184 , preroll_count_(0)
\r
185 , audio_container_(buffer_size_+1)
\r
187 is_running_ = true;
\r
189 video_frame_buffer_.set_capacity(1);
\r
190 audio_frame_buffer_.set_capacity(1);
\r
192 graph_->add_guide("tick-time", 0.5);
\r
193 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
\r
194 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
\r
195 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
\r
196 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
\r
197 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
\r
198 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
\r
199 graph_->set_text(print());
\r
200 diagnostics::register_graph(graph_);
\r
202 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));
\r
204 if(config.embedded_audio)
\r
207 set_latency(config.low_latency);
\r
208 set_keyer(config.internal_key);
\r
210 if(config.embedded_audio)
\r
211 output_->BeginAudioPreroll();
\r
213 for(size_t n = 0; n < buffer_size_; ++n)
\r
214 schedule_next_video(make_safe<core::read_frame>());
\r
216 if(!config.embedded_audio)
\r
220 ~decklink_consumer()
\r
222 is_running_ = false;
\r
223 video_frame_buffer_.try_push(std::make_shared<core::read_frame>());
\r
224 audio_frame_buffer_.try_push(std::make_shared<core::read_frame>());
\r
226 if(output_ != nullptr)
\r
228 output_->StopScheduledPlayback(0, nullptr, 0);
\r
229 if(config_.embedded_audio)
\r
230 output_->DisableAudioOutput();
\r
231 output_->DisableVideoOutput();
\r
235 void set_latency(bool low_latency)
\r
239 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
\r
240 CASPAR_LOG(info) << print() << L" Enabled normal-latency mode";
\r
244 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
\r
245 CASPAR_LOG(info) << print() << L" Enabled low-latency mode";
\r
249 void set_keyer(bool internal_key)
\r
253 if(FAILED(keyer_->Enable(FALSE)))
\r
254 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
\r
255 else if(FAILED(keyer_->SetLevel(255)))
\r
256 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
258 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";
\r
262 if(FAILED(keyer_->Enable(TRUE)))
\r
263 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
\r
264 else if(FAILED(keyer_->SetLevel(255)))
\r
265 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
267 CASPAR_LOG(info) << print() << L" Enabled external keyer.";
\r
271 void enable_audio()
\r
273 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped)))
\r
274 BOOST_THROW_EXCEPTION(caspar_exception() << wmsg_info(print() + L" Could not enable audio output."));
\r
276 if(FAILED(output_->SetAudioCallback(this)))
\r
277 BOOST_THROW_EXCEPTION(caspar_exception() << wmsg_info(print() + L" Could not set audio callback."));
\r
279 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
\r
282 void enable_video(BMDDisplayMode display_mode)
\r
284 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
\r
285 BOOST_THROW_EXCEPTION(caspar_exception() << wmsg_info(print() + L" Could not enable video output."));
\r
287 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
\r
288 BOOST_THROW_EXCEPTION(caspar_exception()
\r
289 << wmsg_info(print() + L" Failed to set playback completion callback.")
\r
290 << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
\r
293 void start_playback()
\r
295 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
\r
296 BOOST_THROW_EXCEPTION(caspar_exception() << wmsg_info(print() + L" Failed to schedule playback."));
\r
299 STDMETHOD (QueryInterface(REFIID, LPVOID*)) {return E_NOINTERFACE;}
\r
300 STDMETHOD_(ULONG, AddRef()) {return 1;}
\r
301 STDMETHOD_(ULONG, Release()) {return 1;}
\r
303 STDMETHOD(ScheduledPlaybackHasStopped())
\r
305 is_running_ = false;
\r
306 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
\r
310 STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))
\r
317 if(result == bmdOutputFrameDisplayedLate)
\r
319 graph_->add_tag("late-frame");
\r
320 video_scheduled_ += format_desc_.duration;
\r
321 audio_scheduled_ += reinterpret_cast<decklink_frame*>(completed_frame)->audio_data().size()/format_desc_.audio_channels;
\r
322 //++video_scheduled_;
\r
323 //audio_scheduled_ += format_desc_.audio_cadence[0];
\r
324 //++audio_scheduled_;
\r
326 else if(result == bmdOutputFrameDropped)
\r
327 graph_->add_tag("dropped-frame");
\r
328 else if(result == bmdOutputFrameFlushed)
\r
329 graph_->add_tag("flushed-frame");
\r
331 std::shared_ptr<core::read_frame> frame;
\r
332 video_frame_buffer_.pop(frame);
\r
333 schedule_next_video(make_safe_ptr(frame));
\r
335 unsigned long buffered;
\r
336 output_->GetBufferedVideoFrameCount(&buffered);
\r
337 graph_->update_value("buffered-video", static_cast<double>(buffered)/format_desc_.fps);
\r
341 tbb::spin_mutex::scoped_lock lock(exception_mutex_);
\r
342 exception_ = std::current_exception();
\r
349 STDMETHOD(RenderAudioSamples(BOOL preroll))
\r
358 if(++preroll_count_ >= buffer_size_)
\r
360 output_->EndAudioPreroll();
\r
364 schedule_next_audio(core::audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()], 0));
\r
368 std::shared_ptr<core::read_frame> frame;
\r
369 audio_frame_buffer_.pop(frame);
\r
370 schedule_next_audio(frame->audio_data());
\r
373 unsigned long buffered;
\r
374 output_->GetBufferedAudioSampleFrameCount(&buffered);
\r
375 graph_->update_value("buffered-audio", static_cast<double>(buffered)/(format_desc_.audio_cadence[0]*2));
\r
379 tbb::spin_mutex::scoped_lock lock(exception_mutex_);
\r
380 exception_ = std::current_exception();
\r
387 template<typename T>
\r
388 void schedule_next_audio(const T& audio_data)
\r
390 const int sample_frame_count = static_cast<int>(audio_data.size())/format_desc_.audio_channels;
\r
392 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
\r
394 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
\r
395 CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
\r
397 audio_scheduled_ += sample_frame_count;
\r
400 void schedule_next_video(const safe_ptr<core::read_frame>& frame)
\r
402 CComPtr<IDeckLinkVideoFrame> frame2(new decklink_frame(frame, format_desc_, config_.key_only));
\r
403 if(FAILED(output_->ScheduleVideoFrame(frame2, video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
\r
404 CASPAR_LOG(error) << print() << L" Failed to schedule video.";
\r
406 video_scheduled_ += format_desc_.duration;
\r
408 graph_->update_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
\r
409 tick_timer_.restart();
\r
412 void send(const safe_ptr<core::read_frame>& frame)
\r
415 tbb::spin_mutex::scoped_lock lock(exception_mutex_);
\r
416 if(exception_ != nullptr)
\r
417 std::rethrow_exception(exception_);
\r
421 BOOST_THROW_EXCEPTION(caspar_exception() << wmsg_info(print() + L" Is not running."));
\r
423 if(config_.embedded_audio)
\r
424 audio_frame_buffer_.push(frame);
\r
425 video_frame_buffer_.push(frame);
\r
428 std::wstring print() const
\r
430 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_) + L"-" +
\r
431 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
\r
435 struct decklink_consumer_proxy : public core::frame_consumer
\r
437 const configuration config_;
\r
438 com_context<decklink_consumer> context_;
\r
439 std::vector<int> audio_cadence_;
\r
442 decklink_consumer_proxy(const configuration& config)
\r
444 , context_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
\r
448 ~decklink_consumer_proxy()
\r
452 auto str = print();
\r
454 CASPAR_LOG(info) << str << L" Successfully Uninitialized.";
\r
460 virtual void initialize(const core::video_format_desc& format_desc, int channel_index) override
\r
462 context_.reset([&]{return new decklink_consumer(config_, format_desc, channel_index);});
\r
463 audio_cadence_ = format_desc.audio_cadence;
\r
465 CASPAR_LOG(info) << print() << L" Successfully Initialized.";
\r
468 virtual bool send(const safe_ptr<core::read_frame>& frame) override
\r
470 CASPAR_VERIFY(audio_cadence_.front() == static_cast<size_t>(frame->audio_data().size()));
\r
471 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1);
\r
473 context_->send(frame);
\r
477 virtual std::wstring print() const override
\r
479 return context_ ? context_->print() : L"[decklink_consumer]";
\r
482 virtual boost::property_tree::wptree info() const override
\r
484 boost::property_tree::wptree info;
\r
485 info.add(L"type", L"decklink-consumer");
\r
486 info.add(L"key-only", config_.key_only);
\r
487 info.add(L"device", config_.device_index);
\r
488 info.add(L"low-latency", config_.low_latency);
\r
489 info.add(L"embedded-audio", config_.embedded_audio);
\r
490 info.add(L"low-latency", config_.low_latency);
\r
491 info.add(L"internal-key", config_.internal_key);
\r
495 virtual int buffer_depth() const override
\r
497 return config_.buffer_depth;
\r
500 virtual int index() const override
\r
502 return 300 + config_.device_index;
\r
506 safe_ptr<core::frame_consumer> create_consumer(const std::vector<std::wstring>& params)
\r
508 if(params.size() < 1 || params[0] != L"DECKLINK")
\r
509 return core::frame_consumer::empty();
\r
511 configuration config;
\r
513 if(params.size() > 1)
\r
514 config.device_index = lexical_cast_or_default<int>(params[1], config.device_index);
\r
516 config.internal_key = std::find(params.begin(), params.end(), L"INTERNAL_KEY") != params.end();
\r
517 config.low_latency = std::find(params.begin(), params.end(), L"LOW_LATENCY") != params.end();
\r
518 config.embedded_audio = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();
\r
519 config.key_only = std::find(params.begin(), params.end(), L"KEY_ONLY") != params.end();
\r
521 return make_safe<decklink_consumer_proxy>(config);
\r
524 safe_ptr<core::frame_consumer> create_consumer(const boost::property_tree::wptree& ptree)
\r
526 configuration config;
\r
528 config.internal_key = ptree.get(L"internal-key", config.internal_key);
\r
529 config.low_latency = ptree.get(L"low-latency", config.low_latency);
\r
530 config.key_only = ptree.get(L"key-only", config.key_only);
\r
531 config.device_index = ptree.get(L"device", config.device_index);
\r
532 config.embedded_audio = ptree.get(L"embedded-audio", config.embedded_audio);
\r
533 config.base_buffer_depth = ptree.get(L"buffer-depth", config.base_buffer_depth);
\r
535 return make_safe<decklink_consumer_proxy>(config);
\r
541 ##############################################################################
\r
547 BMD Developer Support
\r
548 developer@blackmagic-design.com
\r
550 -----------------------------------------------------------------------------
\r
552 Thanks for your inquiry. The minimum number of frames that you can preroll
\r
553 for scheduled playback is three frames for video and four frames for audio.
\r
554 As you mentioned if you preroll less frames then playback will not start or
\r
555 playback will be very sporadic. From our experience with Media Express, we
\r
556 recommended that at least seven frames are prerolled for smooth playback.
\r
558 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
\r
559 There can be around 3 frames worth of latency on scheduled output.
\r
560 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
\r
561 reduced or removed for scheduled playback. If the DisplayVideoFrameSync()
\r
562 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
\r
563 guarantee that the provided frame will be output as soon the previous
\r
564 frame output has been completed.
\r
565 ################################################################################
\r
569 ##############################################################################
\r
570 Async DMA Transfer without redundant copying
\r
575 BMD Developer Support
\r
576 developer@blackmagic-design.com
\r
578 -----------------------------------------------------------------------------
\r
580 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
\r
581 and providing a pointer to your video buffer when GetBytes() is called.
\r
582 This may help to keep copying to a minimum. Please ensure that the pixel
\r
583 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
\r
584 have to colourspace convert which may result in additional copying.
\r
585 ################################################################################
\r