git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Robert Nagy, ronag89@gmail.com
  20 */
  21
  22 #include "../StdAfx.h"
  23
  24 #include "decklink_consumer.h"
  25
  26 #include "../util/util.h"
  27 #include "../decklink.h"
  28
  29 #include "../decklink_api.h"
  30
  31 #include <core/frame/frame.h>
  32 #include <core/frame/audio_channel_layout.h>
  33 #include <core/mixer/audio/audio_mixer.h>
  34 #include <core/consumer/frame_consumer.h>
  35 #include <core/diagnostics/call_context.h>
  36 #include <core/help/help_sink.h>
  37 #include <core/help/help_repository.h>
  38
  39 #include <common/executor.h>
  40 #include <common/lock.h>
  41 #include <common/diagnostics/graph.h>
  42 #include <common/except.h>
  43 #include <common/memshfl.h>
  44 #include <common/memcpy.h>
  45 #include <common/no_init_proxy.h>
  46 #include <common/array.h>
  47 #include <common/future.h>
  48 #include <common/cache_aligned_vector.h>
  49 #include <common/timer.h>
  50 #include <common/param.h>
  51 #include <common/software_version.h>
  52
  53 #include <tbb/concurrent_queue.h>
  54
  55 #include <common/assert.h>
  56 #include <boost/lexical_cast.hpp>
  57 #include <boost/circular_buffer.hpp>
  58 #include <boost/property_tree/ptree.hpp>
  59 #include <boost/thread/mutex.hpp>
  60
  61 #include <future>
  62
  63 namespace caspar { namespace decklink {
  64
  65 struct configuration
  66 {
  67         enum class keyer_t
  68         {
  69                 internal_keyer,
  70                 external_keyer,
  71                 external_separate_device_keyer,
  72                 default_keyer                                   = external_keyer
  73         };
  74
  75         enum class latency_t
  76         {
  77                 low_latency,
  78                 normal_latency,
  79                 default_latency = normal_latency
  80         };
  81
  82         int                                                     device_index            = 1;
  83         int                                                     key_device_idx          = 0;
  84         bool                                            embedded_audio          = false;
  85         keyer_t                                         keyer                           = keyer_t::default_keyer;
  86         latency_t                                       latency                         = latency_t::default_latency;
  87         bool                                            key_only                        = false;
  88         int                                                     base_buffer_depth       = 3;
  89         core::audio_channel_layout      out_channel_layout      = core::audio_channel_layout::invalid();
  90
  91         int buffer_depth() const
  92         {
  93                 return base_buffer_depth + (latency == latency_t::low_latency ? 0 : 1);
  94         }
  95
  96         int key_device_index() const
  97         {
  98                 return key_device_idx == 0 ? device_index + 1 : key_device_idx;
  99         }
 100
 101         core::audio_channel_layout get_adjusted_layout(const core::audio_channel_layout& in_layout) const
 102         {
 103                 auto adjusted = out_channel_layout == core::audio_channel_layout::invalid() ? in_layout : out_channel_layout;
 104
 105                 if (adjusted.num_channels == 1) // Duplicate mono-signal into both left and right.
 106                 {
 107                         adjusted.num_channels = 2;
 108                         adjusted.channel_order.push_back(adjusted.channel_order.at(0)); // Usually FC -> FC FC
 109                 }
 110                 else if (adjusted.num_channels == 2)
 111                 {
 112                         adjusted.num_channels = 2;
 113                 }
 114                 else if (adjusted.num_channels <= 8)
 115                 {
 116                         adjusted.num_channels = 8;
 117                 }
 118                 else // Over 8 always pad to 16 or drop >16
 119                 {
 120                         adjusted.num_channels = 16;
 121                 }
 122
 123                 return adjusted;
 124         }
 125 };
 126
 127 template <typename Configuration>
 128 void set_latency(
 129                 const com_iface_ptr<Configuration>& config,
 130                 configuration::latency_t latency,
 131                 const std::wstring& print)
 132 {
 133         if (latency == configuration::latency_t::low_latency)
 134         {
 135                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
 136                 CASPAR_LOG(info) << print << L" Enabled low-latency mode.";
 137         }
 138         else if (latency == configuration::latency_t::normal_latency)
 139         {
 140                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
 141                 CASPAR_LOG(info) << print << L" Disabled low-latency mode.";
 142         }
 143 }
 144
 145 void set_keyer(
 146                 const com_iface_ptr<IDeckLinkAttributes>& attributes,
 147                 const com_iface_ptr<IDeckLinkKeyer>& decklink_keyer,
 148                 configuration::keyer_t keyer,
 149                 const std::wstring& print)
 150 {
 151         if (keyer == configuration::keyer_t::internal_keyer)
 152         {
 153                 BOOL value = true;
 154                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
 155                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 156                 else if (FAILED(decklink_keyer->Enable(FALSE)))
 157                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 158                 else if (FAILED(decklink_keyer->SetLevel(255)))
 159                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 160                 else
 161                         CASPAR_LOG(info) << print << L" Enabled internal keyer.";
 162         }
 163         else if (keyer == configuration::keyer_t::external_keyer)
 164         {
 165                 BOOL value = true;
 166                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
 167                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 168                 else if (FAILED(decklink_keyer->Enable(TRUE)))
 169                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 170                 else if (FAILED(decklink_keyer->SetLevel(255)))
 171                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 172                 else
 173                         CASPAR_LOG(info) << print << L" Enabled external keyer.";
 174         }
 175 }
 176
 177 class decklink_frame : public IDeckLinkVideoFrame
 178 {
 179         tbb::atomic<int>                                                                ref_count_;
 180         core::const_frame                                                               frame_;
 181         const core::video_format_desc                                   format_desc_;
 182
 183         const bool                                                                              key_only_;
 184         bool                                                                                    needs_to_copy_;
 185         cache_aligned_vector<no_init_proxy<uint8_t>>    data_;
 186 public:
 187         decklink_frame(core::const_frame frame, const core::video_format_desc& format_desc, bool key_only, bool will_attempt_dma)
 188                 : frame_(frame)
 189                 , format_desc_(format_desc)
 190                 , key_only_(key_only)
 191         {
 192                 ref_count_ = 0;
 193
 194                 bool dma_transfer_from_gl_buffer_impossible;
 195
 196 #if !defined(_MSC_VER)
 197                 // On Linux Decklink cannot DMA transfer from memory returned by glMapBuffer (at least on nvidia)
 198                 dma_transfer_from_gl_buffer_impossible = true;
 199 #else
 200                 // On Windows it is possible.
 201                 dma_transfer_from_gl_buffer_impossible = false;
 202 #endif
 203
 204                 needs_to_copy_ = will_attempt_dma && dma_transfer_from_gl_buffer_impossible;
 205         }
 206
 207         // IUnknown
 208
 209         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)
 210         {
 211                 return E_NOINTERFACE;
 212         }
 213
 214         virtual ULONG STDMETHODCALLTYPE AddRef()
 215         {
 216                 return ++ref_count_;
 217         }
 218
 219         virtual ULONG STDMETHODCALLTYPE Release()
 220         {
 221                 if(--ref_count_ == 0)
 222                 {
 223                         delete this;
 224
 225                         return 0;
 226                 }
 227
 228                 return ref_count_;
 229         }
 230
 231         // IDecklinkVideoFrame
 232
 233         virtual long STDMETHODCALLTYPE GetWidth()                   {return static_cast<long>(format_desc_.width);}
 234         virtual long STDMETHODCALLTYPE GetHeight()                  {return static_cast<long>(format_desc_.height);}
 235         virtual long STDMETHODCALLTYPE GetRowBytes()                {return static_cast<long>(format_desc_.width*4);}
 236         virtual BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat()   {return bmdFormat8BitBGRA;}
 237         virtual BMDFrameFlags STDMETHODCALLTYPE GetFlags()                      {return bmdFrameFlagDefault;}
 238
 239         virtual HRESULT STDMETHODCALLTYPE GetBytes(void** buffer)
 240         {
 241                 try
 242                 {
 243                         if(static_cast<int>(frame_.image_data().size()) != format_desc_.size)
 244                         {
 245                                 data_.resize(format_desc_.size);
 246                                 *buffer = data_.data();
 247                         }
 248                         else if(key_only_)
 249                         {
 250                                 if(data_.empty())
 251                                 {
 252                                         data_.resize(frame_.image_data().size());
 253                                         aligned_memshfl(data_.data(), frame_.image_data().begin(), frame_.image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
 254                                 }
 255                                 *buffer = data_.data();
 256                         }
 257                         else
 258                         {
 259                                 *buffer = const_cast<uint8_t*>(frame_.image_data().begin());
 260
 261                                 if (needs_to_copy_)
 262                                 {
 263                                         data_.resize(frame_.image_data().size());
 264                                         fast_memcpy(data_.data(), *buffer, frame_.image_data().size());
 265                                         *buffer = data_.data();
 266                                 }
 267                         }
 268                 }
 269                 catch(...)
 270                 {
 271                         CASPAR_LOG_CURRENT_EXCEPTION();
 272                         return E_FAIL;
 273                 }
 274
 275                 return S_OK;
 276         }
 277
 278         virtual HRESULT STDMETHODCALLTYPE GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode) {return S_FALSE;}
 279         virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)              {return S_FALSE;}
 280
 281         // decklink_frame
 282
 283         const core::audio_buffer& audio_data()
 284         {
 285                 return frame_.audio_data();
 286         }
 287
 288         int64_t get_age_millis() const
 289         {
 290                 return frame_.get_age_millis();
 291         }
 292 };
 293
 294 template <typename Configuration>
 295 struct key_video_context : public IDeckLinkVideoOutputCallback, boost::noncopyable
 296 {
 297         const configuration                                     config_;
 298         com_ptr<IDeckLink>                                      decklink_                                       = get_device(config_.key_device_index());
 299         com_iface_ptr<IDeckLinkOutput>          output_                                         = iface_cast<IDeckLinkOutput>(decklink_);
 300         com_iface_ptr<IDeckLinkKeyer>           keyer_                                          = iface_cast<IDeckLinkKeyer>(decklink_, true);
 301         com_iface_ptr<IDeckLinkAttributes>      attributes_                                     = iface_cast<IDeckLinkAttributes>(decklink_);
 302         com_iface_ptr<Configuration>            configuration_                          = iface_cast<Configuration>(decklink_);
 303         tbb::atomic<int64_t>                            current_presentation_delay_;
 304         tbb::atomic<int64_t>                            scheduled_frames_completed_;
 305
 306         key_video_context(const configuration& config, const std::wstring& print)
 307                 : config_(config)
 308         {
 309                 current_presentation_delay_ = 0;
 310                 scheduled_frames_completed_ = 0;
 311
 312                 set_latency(configuration_, config.latency, print);
 313                 set_keyer(attributes_, keyer_, config.keyer, print);
 314
 315                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 316                         CASPAR_THROW_EXCEPTION(caspar_exception()
 317                                         << msg_info(print + L" Failed to set key playback completion callback.")
 318                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 319         }
 320
 321         template<typename Print>
 322         void enable_video(BMDDisplayMode display_mode, const Print& print)
 323         {
 324                 if (FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 325                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable key video output."));
 326
 327                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 328                         CASPAR_THROW_EXCEPTION(caspar_exception()
 329                                         << msg_info(print() + L" Failed to set key playback completion callback.")
 330                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 331         }
 332
 333         virtual ~key_video_context()
 334         {
 335                 if (output_)
 336                 {
 337                         output_->StopScheduledPlayback(0, nullptr, 0);
 338                         output_->DisableVideoOutput();
 339                 }
 340         }
 341
 342         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 343         virtual ULONG STDMETHODCALLTYPE AddRef()                                                        {return 1;}
 344         virtual ULONG STDMETHODCALLTYPE Release()                                                       {return 1;}
 345
 346         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 347         {
 348                 return S_OK;
 349         }
 350
 351         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(
 352                         IDeckLinkVideoFrame* completed_frame,
 353                         BMDOutputFrameCompletionResult result)
 354         {
 355                 auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 356                 current_presentation_delay_ = dframe->get_age_millis();
 357                 ++scheduled_frames_completed_;
 358
 359                 // Let the fill callback keep the pace, so no scheduling here.
 360
 361                 return S_OK;
 362         }
 363 };
 364
 365 template <typename Configuration>
 366 struct decklink_consumer : public IDeckLinkVideoOutputCallback, boost::noncopyable
 367 {
 368         const int                                                                                       channel_index_;
 369         const configuration                                                                     config_;
 370
 371         com_ptr<IDeckLink>                                                                      decklink_                               = get_device(config_.device_index);
 372         com_iface_ptr<IDeckLinkOutput>                                          output_                                 = iface_cast<IDeckLinkOutput>(decklink_);
 373         com_iface_ptr<Configuration>                                            configuration_                  = iface_cast<Configuration>(decklink_);
 374         com_iface_ptr<IDeckLinkKeyer>                                           keyer_                                  = iface_cast<IDeckLinkKeyer>(decklink_, true);
 375         com_iface_ptr<IDeckLinkAttributes>                                      attributes_                             = iface_cast<IDeckLinkAttributes>(decklink_);
 376
 377         tbb::spin_mutex                                                                         exception_mutex_;
 378         std::exception_ptr                                                                      exception_;
 379
 380         tbb::atomic<bool>                                                                       is_running_;
 381
 382         const std::wstring                                                                      model_name_                             = get_model_name(decklink_);
 383         bool                                                                                            will_attempt_dma_;
 384         const core::video_format_desc                                           format_desc_;
 385         const core::audio_channel_layout                                        in_channel_layout_;
 386         const core::audio_channel_layout                                        out_channel_layout_             = config_.get_adjusted_layout(in_channel_layout_);
 387         core::audio_channel_remapper                                            channel_remapper_               { in_channel_layout_, out_channel_layout_ };
 388         const int                                                                                       buffer_size_                    = config_.buffer_depth(); // Minimum buffer-size 3.
 389
 390         long long                                                                                       video_scheduled_                = 0;
 391         long long                                                                                       audio_scheduled_                = 0;
 392
 393         int                                                                                                     preroll_count_                  = 0;
 394
 395         boost::circular_buffer<std::vector<int32_t>>            audio_container_                { buffer_size_ + 1 };
 396
 397         tbb::concurrent_bounded_queue<core::const_frame>        frame_buffer_;
 398
 399         spl::shared_ptr<diagnostics::graph>                                     graph_;
 400         caspar::timer                                                                           tick_timer_;
 401         boost::mutex                                                                            send_completion_mutex_;
 402         std::packaged_task<bool ()>                                                     send_completion_;
 403         reference_signal_detector                                                       reference_signal_detector_      { output_ };
 404         tbb::atomic<int64_t>                                                            current_presentation_delay_;
 405         tbb::atomic<int64_t>                                                            scheduled_frames_completed_;
 406         std::unique_ptr<key_video_context<Configuration>>       key_context_;
 407
 408 public:
 409         decklink_consumer(
 410                         const configuration& config,
 411                         const core::video_format_desc& format_desc,
 412                         const core::audio_channel_layout& in_channel_layout,
 413                         int channel_index)
 414                 : channel_index_(channel_index)
 415                 , config_(config)
 416                 , format_desc_(format_desc)
 417                 , in_channel_layout_(in_channel_layout)
 418         {
 419                 is_running_ = true;
 420                 current_presentation_delay_ = 0;
 421                 scheduled_frames_completed_ = 0;
 422
 423                 frame_buffer_.set_capacity(1);
 424
 425                 if (config.keyer == configuration::keyer_t::external_separate_device_keyer)
 426                         key_context_.reset(new key_video_context<Configuration>(config, print()));
 427
 428                 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
 429                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
 430                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
 431                 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
 432                 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
 433                 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
 434
 435                 if (key_context_)
 436                 {
 437                         graph_->set_color("key-offset", diagnostics::color(1.0f, 0.0f, 0.0f));
 438                 }
 439
 440                 graph_->set_text(print());
 441                 diagnostics::register_graph(graph_);
 442
 443                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault, will_attempt_dma_));
 444
 445                 if(config.embedded_audio)
 446                         enable_audio();
 447
 448                 set_latency(configuration_, config.latency, print());
 449                 set_keyer(attributes_, keyer_, config.keyer, print());
 450
 451                 if(config.embedded_audio)
 452                         output_->BeginAudioPreroll();
 453
 454                 for (int n = 0; n < buffer_size_; ++n)
 455                 {
 456                         if (config.embedded_audio)
 457                                 schedule_next_audio(core::mutable_audio_buffer(format_desc_.audio_cadence[n % format_desc_.audio_cadence.size()] * out_channel_layout_.num_channels, 0));
 458
 459                         schedule_next_video(core::const_frame::empty());
 460                 }
 461
 462                 if (config.embedded_audio)
 463                 {
 464                         // Preroll one extra frame worth of audio
 465                         schedule_next_audio(core::mutable_audio_buffer(format_desc_.audio_cadence[buffer_size_ % format_desc_.audio_cadence.size()] * out_channel_layout_.num_channels, 0));
 466                         output_->EndAudioPreroll();
 467                 }
 468
 469                 start_playback();
 470         }
 471
 472         ~decklink_consumer()
 473         {
 474                 is_running_ = false;
 475                 frame_buffer_.try_push(core::const_frame::empty());
 476
 477                 if(output_ != nullptr)
 478                 {
 479                         output_->StopScheduledPlayback(0, nullptr, 0);
 480                         if(config_.embedded_audio)
 481                                 output_->DisableAudioOutput();
 482                         output_->DisableVideoOutput();
 483                 }
 484         }
 485
 486         void enable_audio()
 487         {
 488                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, out_channel_layout_.num_channels, bmdAudioOutputStreamTimestamped)))
 489                                 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable audio output."));
 490
 491                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
 492         }
 493
 494         void enable_video(BMDDisplayMode display_mode)
 495         {
 496                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 497                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable fill video output."));
 498
 499                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 500                         CASPAR_THROW_EXCEPTION(caspar_exception()
 501                                                                         << msg_info(print() + L" Failed to set fill playback completion callback.")
 502                                                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 503
 504                 if (key_context_)
 505                         key_context_->enable_video(display_mode, [this]() { return print(); });
 506         }
 507
 508         void start_playback()
 509         {
 510                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 511                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule fill playback."));
 512
 513                 if (key_context_ && FAILED(key_context_->output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 514                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule key playback."));
 515         }
 516
 517         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 518         virtual ULONG STDMETHODCALLTYPE AddRef()                                        {return 1;}
 519         virtual ULONG STDMETHODCALLTYPE Release()                               {return 1;}
 520
 521         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 522         {
 523                 is_running_ = false;
 524                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
 525                 return S_OK;
 526         }
 527
 528         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result)
 529         {
 530                 if(!is_running_)
 531                         return E_FAIL;
 532
 533                 try
 534                 {
 535                         auto tick_time = tick_timer_.elapsed()*format_desc_.fps * 0.5;
 536                         graph_->set_value("tick-time", tick_time);
 537                         tick_timer_.restart();
 538
 539                         reference_signal_detector_.detect_change([this]() { return print(); });
 540
 541                         auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 542                         current_presentation_delay_ = dframe->get_age_millis();
 543                         ++scheduled_frames_completed_;
 544
 545                         if (key_context_)
 546                                 graph_->set_value(
 547                                                 "key-offset",
 548                                                 static_cast<double>(
 549                                                                 scheduled_frames_completed_
 550                                                                 - key_context_->scheduled_frames_completed_)
 551                                                 * 0.1 + 0.5);
 552
 553                         if(result == bmdOutputFrameDisplayedLate)
 554                         {
 555                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "late-frame");
 556                                 video_scheduled_ += format_desc_.duration;
 557                                 audio_scheduled_ += dframe->audio_data().size() / in_channel_layout_.num_channels;
 558                         }
 559                         else if(result == bmdOutputFrameDropped)
 560                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "dropped-frame");
 561                         else if(result == bmdOutputFrameFlushed)
 562                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "flushed-frame");
 563
 564                         UINT32 buffered;
 565                         output_->GetBufferedVideoFrameCount(&buffered);
 566                         graph_->set_value("buffered-video", static_cast<double>(buffered) / (config_.buffer_depth()));
 567
 568                         if (config_.embedded_audio)
 569                         {
 570                                 output_->GetBufferedAudioSampleFrameCount(&buffered);
 571                                 graph_->set_value("buffered-audio", static_cast<double>(buffered) / (format_desc_.audio_cadence[0] * config_.buffer_depth()));
 572                         }
 573
 574                         auto frame = core::const_frame::empty();
 575
 576                         frame_buffer_.pop(frame);
 577
 578                         {
 579                                 boost::lock_guard<boost::mutex> lock(send_completion_mutex_);
 580
 581                                 if (send_completion_.valid())
 582                                 {
 583                                         send_completion_();
 584                                         send_completion_ = std::packaged_task<bool()>();
 585                                 }
 586                         }
 587
 588                         if (!is_running_)
 589                                 return E_FAIL;
 590
 591                         if (config_.embedded_audio)
 592                                 schedule_next_audio(channel_remapper_.mix_and_rearrange(frame.audio_data()));
 593
 594                         schedule_next_video(frame);
 595                 }
 596                 catch(...)
 597                 {
 598                         lock(exception_mutex_, [&]
 599                         {
 600                                 exception_ = std::current_exception();
 601                         });
 602                         return E_FAIL;
 603                 }
 604
 605                 return S_OK;
 606         }
 607
 608         template<typename T>
 609         void schedule_next_audio(const T& audio_data)
 610         {
 611                 auto sample_frame_count = static_cast<int>(audio_data.size()/out_channel_layout_.num_channels);
 612
 613                 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
 614
 615                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
 616                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
 617
 618                 audio_scheduled_ += sample_frame_count;
 619         }
 620
 621         void schedule_next_video(core::const_frame frame)
 622         {
 623                 if (key_context_)
 624                 {
 625                         auto key_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, true, will_attempt_dma_));
 626                         if (FAILED(key_context_->output_->ScheduleVideoFrame(get_raw(key_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 627                                 CASPAR_LOG(error) << print() << L" Failed to schedule key video.";
 628                 }
 629
 630                 auto fill_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, config_.key_only, will_attempt_dma_));
 631                 if (FAILED(output_->ScheduleVideoFrame(get_raw(fill_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 632                         CASPAR_LOG(error) << print() << L" Failed to schedule fill video.";
 633
 634                 video_scheduled_ += format_desc_.duration;
 635         }
 636
 637         std::future<bool> send(core::const_frame frame)
 638         {
 639                 auto exception = lock(exception_mutex_, [&]
 640                 {
 641                         return exception_;
 642                 });
 643
 644                 if(exception != nullptr)
 645                         std::rethrow_exception(exception);
 646
 647                 if(!is_running_)
 648                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Is not running."));
 649
 650                 if (frame_buffer_.try_push(frame))
 651                         return make_ready_future(true);
 652
 653                 boost::lock_guard<boost::mutex> lock(send_completion_mutex_);
 654
 655                 send_completion_ = std::packaged_task<bool ()>([frame, this] () mutable -> bool
 656                 {
 657                         frame_buffer_.push(frame);
 658
 659                         return true;
 660                 });
 661
 662                 return send_completion_.get_future();
 663         }
 664
 665         std::wstring print() const
 666         {
 667                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 668                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 669                                 boost::lexical_cast<std::wstring>(config_.device_index) +
 670                                 L"&&" +
 671                                 boost::lexical_cast<std::wstring>(config_.key_device_index()) +
 672                                 L"|" +
 673                                 format_desc_.name + L"]";
 674                 else
 675                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 676                                 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
 677         }
 678 };
 679
 680 template <typename Configuration>
 681 struct decklink_consumer_proxy : public core::frame_consumer
 682 {
 683         core::monitor::subject                                                          monitor_subject_;
 684         const configuration                                                                     config_;
 685         std::unique_ptr<decklink_consumer<Configuration>>       consumer_;
 686         core::video_format_desc                                                         format_desc_;
 687         executor                                                                                        executor_;
 688 public:
 689
 690         decklink_consumer_proxy(const configuration& config)
 691                 : config_(config)
 692                 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
 693         {
 694                 auto ctx = core::diagnostics::call_context::for_thread();
 695                 executor_.begin_invoke([=]
 696                 {
 697                         core::diagnostics::call_context::for_thread() = ctx;
 698                         com_initialize();
 699                 });
 700         }
 701
 702         ~decklink_consumer_proxy()
 703         {
 704                 executor_.invoke([=]
 705                 {
 706                         consumer_.reset();
 707                         com_uninitialize();
 708                 });
 709         }
 710
 711         // frame_consumer
 712
 713         void initialize(const core::video_format_desc& format_desc, const core::audio_channel_layout& channel_layout, int channel_index) override
 714         {
 715                 format_desc_ = format_desc;
 716                 executor_.invoke([=]
 717                 {
 718                         consumer_.reset();
 719                         consumer_.reset(new decklink_consumer<Configuration>(config_, format_desc, channel_layout, channel_index));
 720                 });
 721         }
 722
 723         std::future<bool> send(core::const_frame frame) override
 724         {
 725                 return consumer_->send(frame);
 726         }
 727
 728         std::wstring print() const override
 729         {
 730                 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
 731         }
 732
 733         std::wstring name() const override
 734         {
 735                 return L"decklink";
 736         }
 737
 738         boost::property_tree::wptree info() const override
 739         {
 740                 boost::property_tree::wptree info;
 741                 info.add(L"type", L"decklink");
 742                 info.add(L"key-only", config_.key_only);
 743                 info.add(L"device", config_.device_index);
 744
 745                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 746                 {
 747                         info.add(L"key-device", config_.key_device_index());
 748                 }
 749
 750                 info.add(L"low-latency", config_.latency == configuration::latency_t::low_latency);
 751                 info.add(L"embedded-audio", config_.embedded_audio);
 752                 info.add(L"presentation-frame-age", presentation_frame_age_millis());
 753                 //info.add(L"internal-key", config_.internal_key);
 754                 return info;
 755         }
 756
 757         int buffer_depth() const override
 758         {
 759                 return config_.buffer_depth() + 2;
 760         }
 761
 762         int index() const override
 763         {
 764                 return 300 + config_.device_index;
 765         }
 766
 767         int64_t presentation_frame_age_millis() const override
 768         {
 769                 return consumer_ ? static_cast<int64_t>(consumer_->current_presentation_delay_) : 0;
 770         }
 771
 772         core::monitor::subject& monitor_output()
 773         {
 774                 return monitor_subject_;
 775         }
 776 };
 777
 778 const software_version<3>& get_driver_version()
 779 {
 780         static software_version<3> version(u8(get_version()));
 781
 782         return version;
 783 }
 784
 785 const software_version<3> get_new_configuration_api_version()
 786 {
 787         static software_version<3> NEW_CONFIGURATION_API("10.2");
 788
 789         return NEW_CONFIGURATION_API;
 790 }
 791
 792 void describe_consumer(core::help_sink& sink, const core::help_repository& repo)
 793 {
 794         sink.short_description(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 795         sink.syntax(L"DECKLINK "
 796                                 L"{[device_index:int]|1} "
 797                                 L"{[keyer:INTERNAL_KEY,EXTERNAL_KEY,EXTERNAL_SEPARATE_DEVICE_KEY]} "
 798                                 L"{[low_latency:LOW_LATENCY]} "
 799                                 L"{[embedded_audio:EMBEDDED_AUDIO]} "
 800                                 L"{[key_only:KEY_ONLY]} "
 801                                 L"{CHANNEL_LAYOUT [channel_layout:string]}");
 802         sink.para()->text(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 803         sink.definitions()
 804                 ->item(L"device_index", L"The Blackmagic video card to use (See Blackmagic control panel for card order). Default is 1.")
 805                 ->item(L"keyer",
 806                                 L"If given tries to enable either internal or external keying. Not all Blackmagic cards supports this. "
 807                                 L"There is also a third experimental option (EXTERNAL_SEPARATE_DEVICE_KEY) which allocates device_index + 1 for synhronized key output.")
 808                 ->item(L"low_latency", L"Tries to enable low latency if given.")
 809                 ->item(L"embedded_audio", L"Embeds the audio into the SDI signal if given.")
 810                 ->item(L"key_only",
 811                                 L" will extract only the alpha channel from the "
 812                                 L"channel. This is useful when you have two SDI video cards, and neither has native support "
 813                                 L"for separate fill/key output")
 814                 ->item(L"channel_layout", L"If specified, overrides the audio channel layout used by the channel.");
 815         sink.para()->text(L"Examples:");
 816         sink.example(L">> ADD 1 DECKLINK", L"for using the default device_index of 1.");
 817         sink.example(L">> ADD 1 DECKLINK 2", L"uses device_index 2.");
 818         sink.example(L">> ADD 1 DECKLINK 1 EXTERNAL_KEY EMBEDDED_AUDIO");
 819         sink.example(
 820                         L">> ADD 1 DECKLINK 1 EMBEDDED_AUDIO\n"
 821                         L">> ADD 1 DECKLINK 2 KEY_ONLY", L"uses device with index 1 as fill output with audio and device with index 2 as key output.");
 822         sink.example(
 823                         L">> ADD 1 DECKLINK 1 EXTERNAL_SEPARATE_DEVICE_KEY EMBEDDED_AUDIO",
 824                         L"Uses device 2 for key output. May give better sync between key and fill than the previous method.");
 825 }
 826
 827 spl::shared_ptr<core::frame_consumer> create_consumer(
 828                 const std::vector<std::wstring>& params, core::interaction_sink*, std::vector<spl::shared_ptr<core::video_channel>> channels)
 829 {
 830         if (params.size() < 1 || !boost::iequals(params.at(0), L"DECKLINK"))
 831                 return core::frame_consumer::empty();
 832
 833         configuration config;
 834
 835         if (params.size() > 1)
 836                 config.device_index = boost::lexical_cast<int>(params.at(1));
 837
 838         if (contains_param(L"INTERNAL_KEY", params))
 839                 config.keyer = configuration::keyer_t::internal_keyer;
 840         else if (contains_param(L"EXTERNAL_KEY", params))
 841                 config.keyer = configuration::keyer_t::external_keyer;
 842         else if (contains_param(L"EXTERNAL_SEPARATE_DEVICE_KEY", params))
 843                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 844         else
 845                 config.keyer = configuration::keyer_t::default_keyer;
 846
 847         if (contains_param(L"LOW_LATENCY", params))
 848                 config.latency = configuration::latency_t::low_latency;
 849
 850         config.embedded_audio   = contains_param(L"EMBEDDED_AUDIO", params);
 851         config.key_only                 = contains_param(L"KEY_ONLY", params);
 852
 853         auto channel_layout = get_param(L"CHANNEL_LAYOUT", params);
 854
 855         if (!channel_layout.empty())
 856         {
 857                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(channel_layout);
 858
 859                 if (!found_layout)
 860                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + channel_layout + L" not found."));
 861
 862                 config.out_channel_layout = *found_layout;
 863         }
 864
 865         bool old_configuration_api = get_driver_version() < get_new_configuration_api_version();
 866
 867         if (old_configuration_api)
 868                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration_v10_2>>(config);
 869         else
 870                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration>>(config);
 871 }
 872
 873 spl::shared_ptr<core::frame_consumer> create_preconfigured_consumer(
 874                 const boost::property_tree::wptree& ptree, core::interaction_sink*, std::vector<spl::shared_ptr<core::video_channel>> channels)
 875 {
 876         configuration config;
 877
 878         auto keyer = ptree.get(L"keyer", L"default");
 879         if(keyer == L"external")
 880                 config.keyer = configuration::keyer_t::external_keyer;
 881         else if(keyer == L"internal")
 882                 config.keyer = configuration::keyer_t::internal_keyer;
 883         else if (keyer == L"external_separate_device")
 884                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 885
 886         auto latency = ptree.get(L"latency", L"default");
 887         if(latency == L"low")
 888                 config.latency = configuration::latency_t::low_latency;
 889         else if(latency == L"normal")
 890                 config.latency = configuration::latency_t::normal_latency;
 891
 892         auto channel_layout = ptree.get_optional<std::wstring>(L"channel-layout");
 893
 894         if (channel_layout)
 895         {
 896                 CASPAR_SCOPED_CONTEXT_MSG("/channel-layout")
 897
 898                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(*channel_layout);
 899
 900                 if (!found_layout)
 901                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + *channel_layout + L" not found."));
 902
 903                 config.out_channel_layout = *found_layout;
 904         }
 905
 906         config.key_only                         = ptree.get(L"key-only",                config.key_only);
 907         config.device_index                     = ptree.get(L"device",                  config.device_index);
 908         config.key_device_idx           = ptree.get(L"key-device",              config.key_device_idx);
 909         config.embedded_audio           = ptree.get(L"embedded-audio",  config.embedded_audio);
 910         config.base_buffer_depth        = ptree.get(L"buffer-depth",    config.base_buffer_depth);
 911
 912         bool old_configuration_api = get_driver_version() < get_new_configuration_api_version();
 913
 914         if (old_configuration_api)
 915                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration_v10_2>>(config);
 916         else
 917                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration>>(config);
 918 }
 919
 920 }}
 921
 922 /*
 923 ##############################################################################
 924 Pre-rolling
 925
 926 Mail: 2011-05-09
 927
 928 Yoshan
 929 BMD Developer Support
 930 developer@blackmagic-design.com
 931
 932 -----------------------------------------------------------------------------
 933
 934 Thanks for your inquiry. The minimum number of frames that you can preroll
 935 for scheduled playback is three frames for video and four frames for audio.
 936 As you mentioned if you preroll less frames then playback will not start or
 937 playback will be very sporadic. From our experience with Media Express, we
 938 recommended that at least seven frames are prerolled for smooth playback.
 939
 940 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
 941 There can be around 3 frames worth of latency on scheduled output.
 942 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
 943 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync()
 944 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
 945 guarantee that the provided frame will be output as soon the previous
 946 frame output has been completed.
 947 ################################################################################
 948 */
 949
 950 /*
 951 ##############################################################################
 952 Async DMA Transfer without redundant copying
 953
 954 Mail: 2011-05-10
 955
 956 Yoshan
 957 BMD Developer Support
 958 developer@blackmagic-design.com
 959
 960 -----------------------------------------------------------------------------
 961
 962 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
 963 and providing a pointer to your video buffer when GetBytes() is called.
 964 This may help to keep copying to a minimum. Please ensure that the pixel
 965 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
 966 have to colourspace convert which may result in additional copying.
 967 ################################################################################
 968 */