2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
\r
4 * This file is part of CasparCG (www.casparcg.com).
\r
6 * CasparCG is free software: you can redistribute it and/or modify
\r
7 * it under the terms of the GNU General Public License as published by
\r
8 * the Free Software Foundation, either version 3 of the License, or
\r
9 * (at your option) any later version.
\r
11 * CasparCG is distributed in the hope that it will be useful,
\r
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
\r
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
\r
14 * GNU General Public License for more details.
\r
16 * You should have received a copy of the GNU General Public License
\r
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
\r
19 * Author: Robert Nagy, ronag89@gmail.com
\r
22 #include "../StdAfx.h"
\r
24 #include "decklink_consumer.h"
\r
26 #include "../util/util.h"
\r
28 #include "../interop/DeckLinkAPI_h.h"
\r
30 #include <core/frame/data_frame.h>
\r
31 #include <core/mixer/audio/audio_mixer.h>
\r
33 #include <common/concurrency/executor.h>
\r
34 #include <common/concurrency/lock.h>
\r
35 #include <common/diagnostics/graph.h>
\r
36 #include <common/except.h>
\r
37 #include <common/memory/memshfl.h>
\r
39 #include <core/consumer/frame_consumer.h>
\r
41 #include <tbb/concurrent_queue.h>
\r
42 #include <tbb/cache_aligned_allocator.h>
\r
44 #include <common/assert.h>
\r
45 #include <boost/lexical_cast.hpp>
\r
46 #include <boost/circular_buffer.hpp>
\r
47 #include <boost/timer.hpp>
\r
48 #include <boost/property_tree/ptree.hpp>
\r
50 namespace caspar { namespace decklink {
\r
52 struct configuration
\r
69 bool embedded_audio;
\r
73 int base_buffer_depth;
\r
77 , embedded_audio(false)
\r
78 , keyer(default_keyer)
\r
79 , latency(default_latency)
\r
81 , base_buffer_depth(3)
\r
85 int buffer_depth() const
\r
87 return base_buffer_depth + (latency == low_latency ? 0 : 1) + (embedded_audio ? 1 : 0);
\r
91 class decklink_frame : public IDeckLinkVideoFrame
\r
93 tbb::atomic<int> ref_count_;
\r
94 std::shared_ptr<const core::data_frame> frame_;
\r
95 const core::video_format_desc format_desc_;
\r
97 const bool key_only_;
\r
98 std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>> data_;
\r
100 decklink_frame(const spl::shared_ptr<const core::data_frame>& frame, const core::video_format_desc& format_desc, bool key_only)
\r
102 , format_desc_(format_desc)
\r
103 , key_only_(key_only)
\r
110 STDMETHOD (QueryInterface(REFIID, LPVOID*))
\r
112 return E_NOINTERFACE;
\r
115 STDMETHOD_(ULONG, AddRef())
\r
117 return ++ref_count_;
\r
120 STDMETHOD_(ULONG, Release())
\r
122 if(--ref_count_ == 0)
\r
127 // IDecklinkVideoFrame
\r
129 STDMETHOD_(long, GetWidth()) {return format_desc_.width;}
\r
130 STDMETHOD_(long, GetHeight()) {return format_desc_.height;}
\r
131 STDMETHOD_(long, GetRowBytes()) {return format_desc_.width*4;}
\r
132 STDMETHOD_(BMDPixelFormat, GetPixelFormat()) {return bmdFormat8BitBGRA;}
\r
133 STDMETHOD_(BMDFrameFlags, GetFlags()) {return bmdFrameFlagDefault;}
\r
135 STDMETHOD(GetBytes(void** buffer))
\r
139 if(static_cast<int>(frame_->image_data().size()) != format_desc_.size)
\r
141 data_.resize(format_desc_.size, 0);
\r
142 *buffer = data_.data();
\r
148 data_.resize(frame_->image_data().size());
\r
149 aligned_memshfl(data_.data(), frame_->image_data().begin(), frame_->image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
\r
151 *buffer = data_.data();
\r
154 *buffer = const_cast<uint8_t*>(frame_->image_data().begin());
\r
158 CASPAR_LOG_CURRENT_EXCEPTION();
\r
165 STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}
\r
166 STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)) {return S_FALSE;}
\r
170 const core::audio_buffer& audio_data()
\r
172 return frame_->audio_data();
\r
176 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
\r
178 const int channel_index_;
\r
179 const configuration config_;
\r
181 CComPtr<IDeckLink> decklink_;
\r
182 CComQIPtr<IDeckLinkOutput> output_;
\r
183 CComQIPtr<IDeckLinkConfiguration> configuration_;
\r
184 CComQIPtr<IDeckLinkKeyer> keyer_;
\r
185 CComQIPtr<IDeckLinkAttributes> attributes_;
\r
187 tbb::spin_mutex exception_mutex_;
\r
188 std::exception_ptr exception_;
\r
190 tbb::atomic<bool> is_running_;
\r
192 const std::wstring model_name_;
\r
193 const core::video_format_desc format_desc_;
\r
194 const int buffer_size_;
\r
196 long long video_scheduled_;
\r
197 long long audio_scheduled_;
\r
199 int preroll_count_;
\r
201 boost::circular_buffer<std::vector<int32_t>> audio_container_;
\r
203 tbb::concurrent_bounded_queue<std::shared_ptr<const core::data_frame>> video_frame_buffer_;
\r
204 tbb::concurrent_bounded_queue<std::shared_ptr<const core::data_frame>> audio_frame_buffer_;
\r
206 spl::shared_ptr<diagnostics::graph> graph_;
\r
207 boost::timer tick_timer_;
\r
210 decklink_consumer(const configuration& config, const core::video_format_desc& format_desc, int channel_index)
\r
211 : channel_index_(channel_index)
\r
213 , decklink_(get_device(config.device_index))
\r
214 , output_(decklink_)
\r
215 , configuration_(decklink_)
\r
216 , keyer_(decklink_)
\r
217 , attributes_(decklink_)
\r
218 , model_name_(get_model_name(decklink_))
\r
219 , format_desc_(format_desc)
\r
220 , buffer_size_(config.buffer_depth()) // Minimum buffer-size 3.
\r
221 , video_scheduled_(0)
\r
222 , audio_scheduled_(0)
\r
223 , preroll_count_(0)
\r
224 , audio_container_(buffer_size_+1)
\r
226 is_running_ = true;
\r
228 video_frame_buffer_.set_capacity(1);
\r
229 audio_frame_buffer_.set_capacity(1);
\r
231 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
\r
232 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
\r
233 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
\r
234 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
\r
235 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
\r
236 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
\r
237 graph_->set_text(print());
\r
238 diagnostics::register_graph(graph_);
\r
240 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));
\r
242 if(config.embedded_audio)
\r
245 set_latency(config.latency);
\r
246 set_keyer(config.keyer);
\r
248 if(config.embedded_audio)
\r
249 output_->BeginAudioPreroll();
\r
251 for(int n = 0; n < buffer_size_; ++n)
\r
252 schedule_next_video(core::data_frame::empty());
\r
254 if(!config.embedded_audio)
\r
258 ~decklink_consumer()
\r
260 is_running_ = false;
\r
261 video_frame_buffer_.try_push(core::data_frame::empty());
\r
262 audio_frame_buffer_.try_push(core::data_frame::empty());
\r
264 if(output_ != nullptr)
\r
266 output_->StopScheduledPlayback(0, nullptr, 0);
\r
267 if(config_.embedded_audio)
\r
268 output_->DisableAudioOutput();
\r
269 output_->DisableVideoOutput();
\r
273 void set_latency(configuration::latency_t latency)
\r
275 if(latency == configuration::low_latency)
\r
277 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
\r
278 CASPAR_LOG(info) << print() << L" Enabled low-latency mode.";
\r
280 else if(latency == configuration::normal_latency)
\r
282 configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
\r
283 CASPAR_LOG(info) << print() << L" Disabled low-latency mode.";
\r
287 void set_keyer(configuration::keyer_t keyer)
\r
289 if(keyer == configuration::internal_keyer)
\r
292 if(SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
\r
293 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
\r
294 else if(FAILED(keyer_->Enable(FALSE)))
\r
295 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";
\r
296 else if(FAILED(keyer_->SetLevel(255)))
\r
297 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
299 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";
\r
301 else if(keyer == configuration::external_keyer)
\r
304 if(SUCCEEDED(attributes_->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
\r
305 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
\r
306 else if(FAILED(keyer_->Enable(TRUE)))
\r
307 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";
\r
308 else if(FAILED(keyer_->SetLevel(255)))
\r
309 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";
\r
311 CASPAR_LOG(info) << print() << L" Enabled external keyer.";
\r
315 void enable_audio()
\r
317 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped)))
\r
318 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable audio output."));
\r
320 if(FAILED(output_->SetAudioCallback(this)))
\r
321 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not set audio callback."));
\r
323 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
\r
326 void enable_video(BMDDisplayMode display_mode)
\r
328 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
\r
329 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable video output."));
\r
331 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
\r
332 BOOST_THROW_EXCEPTION(caspar_exception()
\r
333 << msg_info(u8(print()) + " Failed to set playback completion callback.")
\r
334 << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
\r
337 void start_playback()
\r
339 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
\r
340 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Failed to schedule playback."));
\r
343 STDMETHOD (QueryInterface(REFIID, LPVOID*)) {return E_NOINTERFACE;}
\r
344 STDMETHOD_(ULONG, AddRef()) {return 1;}
\r
345 STDMETHOD_(ULONG, Release()) {return 1;}
\r
347 STDMETHOD(ScheduledPlaybackHasStopped())
\r
349 is_running_ = false;
\r
350 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
\r
354 STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))
\r
361 if(result == bmdOutputFrameDisplayedLate)
\r
363 graph_->set_tag("late-frame");
\r
364 video_scheduled_ += format_desc_.duration;
\r
365 audio_scheduled_ += reinterpret_cast<decklink_frame*>(completed_frame)->audio_data().size()/format_desc_.audio_channels;
\r
366 //++video_scheduled_;
\r
367 //audio_scheduled_ += format_desc_.audio_cadence[0];
\r
368 //++audio_scheduled_;
\r
370 else if(result == bmdOutputFrameDropped)
\r
371 graph_->set_tag("dropped-frame");
\r
372 else if(result == bmdOutputFrameFlushed)
\r
373 graph_->set_tag("flushed-frame");
\r
375 std::shared_ptr<const core::data_frame> frame;
\r
376 video_frame_buffer_.pop(frame);
\r
377 schedule_next_video(spl::make_shared_ptr(frame));
\r
379 unsigned long buffered;
\r
380 output_->GetBufferedVideoFrameCount(&buffered);
\r
381 graph_->set_value("buffered-video", static_cast<double>(buffered)/format_desc_.fps);
\r
385 lock(exception_mutex_, [&]
\r
387 exception_ = std::current_exception();
\r
395 STDMETHOD(RenderAudioSamples(BOOL preroll))
\r
404 if(++preroll_count_ >= buffer_size_)
\r
406 output_->EndAudioPreroll();
\r
410 schedule_next_audio(core::audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()], 0));
\r
414 std::shared_ptr<const core::data_frame> frame;
\r
415 audio_frame_buffer_.pop(frame);
\r
416 schedule_next_audio(frame->audio_data());
\r
419 unsigned long buffered;
\r
420 output_->GetBufferedAudioSampleFrameCount(&buffered);
\r
421 graph_->set_value("buffered-audio", static_cast<double>(buffered)/(format_desc_.audio_cadence[0]*2));
\r
425 tbb::spin_mutex::scoped_lock lock(exception_mutex_);
\r
426 exception_ = std::current_exception();
\r
433 template<typename T>
\r
434 void schedule_next_audio(const T& audio_data)
\r
436 auto sample_frame_count = static_cast<int>(audio_data.size()/format_desc_.audio_channels);
\r
438 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
\r
440 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
\r
441 CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
\r
443 audio_scheduled_ += sample_frame_count;
\r
446 void schedule_next_video(const spl::shared_ptr<const core::data_frame>& frame)
\r
448 CComPtr<IDeckLinkVideoFrame> frame2(new decklink_frame(frame, format_desc_, config_.key_only));
\r
449 if(FAILED(output_->ScheduleVideoFrame(frame2, video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
\r
450 CASPAR_LOG(error) << print() << L" Failed to schedule video.";
\r
452 video_scheduled_ += format_desc_.duration;
\r
454 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
\r
455 tick_timer_.restart();
\r
458 void send(const spl::shared_ptr<const core::data_frame>& frame)
\r
460 auto exception = lock(exception_mutex_, [&]
\r
465 if(exception != nullptr)
\r
466 std::rethrow_exception(exception);
\r
469 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Is not running."));
\r
471 if(config_.embedded_audio)
\r
472 audio_frame_buffer_.push(frame);
\r
473 video_frame_buffer_.push(frame);
\r
476 std::wstring print() const
\r
478 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_) + L"-" +
\r
479 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
\r
483 struct decklink_consumer_proxy : public core::frame_consumer
\r
485 const configuration config_;
\r
486 std::unique_ptr<decklink_consumer> consumer_;
\r
487 std::vector<int> audio_cadence_;
\r
488 executor executor_;
\r
491 decklink_consumer_proxy(const configuration& config)
\r
493 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
\r
495 executor_.begin_invoke([=]
\r
497 ::CoInitialize(nullptr);
\r
501 ~decklink_consumer_proxy()
\r
503 executor_.invoke([=]
\r
505 ::CoUninitialize();
\r
511 virtual void initialize(const core::video_format_desc& format_desc, int channel_index) override
\r
513 executor_.invoke([=]
\r
515 consumer_.reset(new decklink_consumer(config_, format_desc, channel_index));
\r
516 audio_cadence_ = format_desc.audio_cadence;
\r
520 virtual bool send(const spl::shared_ptr<const core::data_frame>& frame) override
\r
522 CASPAR_VERIFY(audio_cadence_.front() == static_cast<int>(frame->audio_data().size()));
\r
523 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1);
\r
525 consumer_->send(frame);
\r
529 virtual std::wstring print() const override
\r
531 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
\r
534 virtual boost::property_tree::wptree info() const override
\r
536 boost::property_tree::wptree info;
\r
537 info.add(L"type", L"decklink-consumer");
\r
538 info.add(L"key-only", config_.key_only);
\r
539 info.add(L"device", config_.device_index);
\r
540 info.add(L"low-latency", config_.low_latency);
\r
541 info.add(L"embedded-audio", config_.embedded_audio);
\r
542 info.add(L"low-latency", config_.low_latency);
\r
543 //info.add(L"internal-key", config_.internal_key);
\r
547 virtual int buffer_depth() const override
\r
549 return config_.buffer_depth();
\r
552 virtual int index() const override
\r
554 return 300 + config_.device_index;
\r
558 spl::shared_ptr<core::frame_consumer> create_consumer(const std::vector<std::wstring>& params)
\r
560 if(params.size() < 1 || params[0] != L"DECKLINK")
\r
561 return core::frame_consumer::empty();
\r
563 configuration config;
\r
565 if(params.size() > 1)
\r
566 config.device_index = boost::lexical_cast<int>(params[1]);
\r
568 if(std::find(params.begin(), params.end(), L"INTERNAL_KEY") != params.end())
\r
569 config.keyer = configuration::internal_keyer;
\r
570 else if(std::find(params.begin(), params.end(), L"EXTERNAL_KEY") != params.end())
\r
571 config.keyer = configuration::external_keyer;
\r
573 config.keyer = configuration::default_keyer;
\r
575 if(std::find(params.begin(), params.end(), L"LOW_LATENCY") != params.end())
\r
576 config.latency = configuration::low_latency;
\r
578 config.embedded_audio = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();
\r
579 config.key_only = std::find(params.begin(), params.end(), L"KEY_ONLY") != params.end();
\r
581 return spl::make_shared<decklink_consumer_proxy>(config);
\r
584 spl::shared_ptr<core::frame_consumer> create_consumer(const boost::property_tree::wptree& ptree)
\r
586 configuration config;
\r
588 auto keyer = ptree.get(L"keyer", L"external");
\r
589 if(keyer == L"external")
\r
590 config.keyer = configuration::external_keyer;
\r
591 else if(keyer == L"internal")
\r
592 config.keyer = configuration::internal_keyer;
\r
594 auto latency = ptree.get(L"latency", L"normal");
\r
595 if(latency == L"low")
\r
596 config.latency = configuration::low_latency;
\r
597 else if(latency == L"normal")
\r
598 config.latency = configuration::normal_latency;
\r
600 config.key_only = ptree.get(L"key-only", config.key_only);
\r
601 config.device_index = ptree.get(L"device", config.device_index);
\r
602 config.embedded_audio = ptree.get(L"embedded-audio", config.embedded_audio);
\r
603 config.base_buffer_depth = ptree.get(L"buffer-depth", config.base_buffer_depth);
\r
605 return spl::make_shared<decklink_consumer_proxy>(config);
\r
611 ##############################################################################
\r
617 BMD Developer Support
\r
618 developer@blackmagic-design.com
\r
620 -----------------------------------------------------------------------------
\r
622 Thanks for your inquiry. The minimum number of frames that you can preroll
\r
623 for scheduled playback is three frames for video and four frames for audio.
\r
624 As you mentioned if you preroll less frames then playback will not start or
\r
625 playback will be very sporadic. From our experience with Media Express, we
\r
626 recommended that at least seven frames are prerolled for smooth playback.
\r
628 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
\r
629 There can be around 3 frames worth of latency on scheduled output.
\r
630 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
\r
631 reduced or removed for scheduled playback. If the DisplayVideoFrameSync()
\r
632 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
\r
633 guarantee that the provided frame will be output as soon the previous
\r
634 frame output has been completed.
\r
635 ################################################################################
\r
639 ##############################################################################
\r
640 Async DMA Transfer without redundant copying
\r
645 BMD Developer Support
\r
646 developer@blackmagic-design.com
\r
648 -----------------------------------------------------------------------------
\r
650 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
\r
651 and providing a pointer to your video buffer when GetBytes() is called.
\r
652 This may help to keep copying to a minimum. Please ensure that the pixel
\r
653 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
\r
654 have to colourspace convert which may result in additional copying.
\r
655 ################################################################################
\r