git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Robert Nagy, ronag89@gmail.com
  20 */
  21
  22 #include "../StdAfx.h"
  23
  24 #include "decklink_consumer.h"
  25
  26 #include "../util/util.h"
  27 #include "../decklink.h"
  28
  29 #include "../decklink_api.h"
  30
  31 #include <core/frame/frame.h>
  32 #include <core/frame/audio_channel_layout.h>
  33 #include <core/mixer/audio/audio_mixer.h>
  34 #include <core/consumer/frame_consumer.h>
  35 #include <core/diagnostics/call_context.h>
  36 #include <core/help/help_sink.h>
  37 #include <core/help/help_repository.h>
  38
  39 #include <common/executor.h>
  40 #include <common/lock.h>
  41 #include <common/diagnostics/graph.h>
  42 #include <common/except.h>
  43 #include <common/memshfl.h>
  44 #include <common/memcpy.h>
  45 #include <common/no_init_proxy.h>
  46 #include <common/array.h>
  47 #include <common/future.h>
  48 #include <common/cache_aligned_vector.h>
  49 #include <common/timer.h>
  50 #include <common/param.h>
  51 #include <common/software_version.h>
  52
  53 #include <tbb/concurrent_queue.h>
  54
  55 #include <common/assert.h>
  56 #include <boost/lexical_cast.hpp>
  57 #include <boost/circular_buffer.hpp>
  58 #include <boost/property_tree/ptree.hpp>
  59 #include <boost/thread/mutex.hpp>
  60
  61 #include <future>
  62
  63 namespace caspar { namespace decklink {
  64
  65 struct configuration
  66 {
  67         enum class keyer_t
  68         {
  69                 internal_keyer,
  70                 external_keyer,
  71                 external_separate_device_keyer,
  72                 default_keyer                                   = external_keyer
  73         };
  74
  75         enum class latency_t
  76         {
  77                 low_latency,
  78                 normal_latency,
  79                 default_latency = normal_latency
  80         };
  81
  82         int                                                     device_index            = 1;
  83         int                                                     key_device_idx          = 0;
  84         bool                                            embedded_audio          = false;
  85         keyer_t                                         keyer                           = keyer_t::default_keyer;
  86         latency_t                                       latency                         = latency_t::default_latency;
  87         bool                                            key_only                        = false;
  88         int                                                     base_buffer_depth       = 3;
  89         core::audio_channel_layout      out_channel_layout      = core::audio_channel_layout::invalid();
  90
  91         int buffer_depth() const
  92         {
  93                 return base_buffer_depth + (latency == latency_t::low_latency ? 0 : 1);
  94         }
  95
  96         int key_device_index() const
  97         {
  98                 return key_device_idx == 0 ? device_index + 1 : key_device_idx;
  99         }
 100
 101         core::audio_channel_layout get_adjusted_layout(const core::audio_channel_layout& in_layout) const
 102         {
 103                 auto adjusted = out_channel_layout == core::audio_channel_layout::invalid() ? in_layout : out_channel_layout;
 104
 105                 if (adjusted.num_channels == 1) // Duplicate mono-signal into both left and right.
 106                 {
 107                         adjusted.num_channels = 2;
 108                         adjusted.channel_order.push_back(adjusted.channel_order.at(0)); // Usually FC -> FC FC
 109                 }
 110                 else if (adjusted.num_channels == 2)
 111                 {
 112                         adjusted.num_channels = 2;
 113                 }
 114                 else if (adjusted.num_channels <= 8)
 115                 {
 116                         adjusted.num_channels = 8;
 117                 }
 118                 else // Over 8 always pad to 16 or drop >16
 119                 {
 120                         adjusted.num_channels = 16;
 121                 }
 122
 123                 return adjusted;
 124         }
 125 };
 126
 127 template <typename Configuration>
 128 void set_latency(
 129                 const com_iface_ptr<Configuration>& config,
 130                 configuration::latency_t latency,
 131                 const std::wstring& print)
 132 {
 133         if (latency == configuration::latency_t::low_latency)
 134         {
 135                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
 136                 CASPAR_LOG(info) << print << L" Enabled low-latency mode.";
 137         }
 138         else if (latency == configuration::latency_t::normal_latency)
 139         {
 140                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
 141                 CASPAR_LOG(info) << print << L" Disabled low-latency mode.";
 142         }
 143 }
 144
 145 void set_keyer(
 146                 const com_iface_ptr<IDeckLinkAttributes>& attributes,
 147                 const com_iface_ptr<IDeckLinkKeyer>& decklink_keyer,
 148                 configuration::keyer_t keyer,
 149                 const std::wstring& print)
 150 {
 151         if (keyer == configuration::keyer_t::internal_keyer)
 152         {
 153                 BOOL value = true;
 154                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
 155                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 156                 else if (FAILED(decklink_keyer->Enable(FALSE)))
 157                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 158                 else if (FAILED(decklink_keyer->SetLevel(255)))
 159                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 160                 else
 161                         CASPAR_LOG(info) << print << L" Enabled internal keyer.";
 162         }
 163         else if (keyer == configuration::keyer_t::external_keyer)
 164         {
 165                 BOOL value = true;
 166                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
 167                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 168                 else if (FAILED(decklink_keyer->Enable(TRUE)))
 169                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 170                 else if (FAILED(decklink_keyer->SetLevel(255)))
 171                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 172                 else
 173                         CASPAR_LOG(info) << print << L" Enabled external keyer.";
 174         }
 175 }
 176
 177 class decklink_frame : public IDeckLinkVideoFrame
 178 {
 179         tbb::atomic<int>                                                                ref_count_;
 180         core::const_frame                                                               frame_;
 181         const core::video_format_desc                                   format_desc_;
 182
 183         const bool                                                                              key_only_;
 184         bool                                                                                    needs_to_copy_;
 185         cache_aligned_vector<no_init_proxy<uint8_t>>    data_;
 186 public:
 187         decklink_frame(core::const_frame frame, const core::video_format_desc& format_desc, bool key_only, bool will_attempt_dma)
 188                 : frame_(frame)
 189                 , format_desc_(format_desc)
 190                 , key_only_(key_only)
 191         {
 192                 ref_count_ = 0;
 193
 194                 bool dma_transfer_from_gl_buffer_impossible;
 195
 196 #if !defined(_MSC_VER)
 197                 // On Linux Decklink cannot DMA transfer from memory returned by glMapBuffer (at least on nvidia)
 198                 dma_transfer_from_gl_buffer_impossible = true;
 199 #else
 200                 // On Windows it is possible.
 201                 dma_transfer_from_gl_buffer_impossible = false;
 202 #endif
 203
 204                 needs_to_copy_ = will_attempt_dma && dma_transfer_from_gl_buffer_impossible;
 205         }
 206
 207         // IUnknown
 208
 209         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)
 210         {
 211                 return E_NOINTERFACE;
 212         }
 213
 214         virtual ULONG STDMETHODCALLTYPE AddRef()
 215         {
 216                 return ++ref_count_;
 217         }
 218
 219         virtual ULONG STDMETHODCALLTYPE Release()
 220         {
 221                 if(--ref_count_ == 0)
 222                 {
 223                         delete this;
 224
 225                         return 0;
 226                 }
 227
 228                 return ref_count_;
 229         }
 230
 231         // IDecklinkVideoFrame
 232
 233         virtual long STDMETHODCALLTYPE GetWidth()                   {return static_cast<long>(format_desc_.width);}
 234         virtual long STDMETHODCALLTYPE GetHeight()                  {return static_cast<long>(format_desc_.height);}
 235         virtual long STDMETHODCALLTYPE GetRowBytes()                {return static_cast<long>(format_desc_.width*4);}
 236         virtual BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat()   {return bmdFormat8BitBGRA;}
 237         virtual BMDFrameFlags STDMETHODCALLTYPE GetFlags()                      {return bmdFrameFlagDefault;}
 238
 239         virtual HRESULT STDMETHODCALLTYPE GetBytes(void** buffer)
 240         {
 241                 try
 242                 {
 243                         if(static_cast<int>(frame_.image_data().size()) != format_desc_.size)
 244                         {
 245                                 data_.resize(format_desc_.size);
 246                                 *buffer = data_.data();
 247                         }
 248                         else if(key_only_)
 249                         {
 250                                 if(data_.empty())
 251                                 {
 252                                         data_.resize(frame_.image_data().size());
 253                                         aligned_memshfl(data_.data(), frame_.image_data().begin(), frame_.image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
 254                                 }
 255                                 *buffer = data_.data();
 256                         }
 257                         else
 258                         {
 259                                 *buffer = const_cast<uint8_t*>(frame_.image_data().begin());
 260
 261                                 if (needs_to_copy_)
 262                                 {
 263                                         data_.resize(frame_.image_data().size());
 264                                         fast_memcpy(data_.data(), *buffer, frame_.image_data().size());
 265                                         *buffer = data_.data();
 266                                 }
 267                         }
 268                 }
 269                 catch(...)
 270                 {
 271                         CASPAR_LOG_CURRENT_EXCEPTION();
 272                         return E_FAIL;
 273                 }
 274
 275                 return S_OK;
 276         }
 277
 278         virtual HRESULT STDMETHODCALLTYPE GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode) {return S_FALSE;}
 279         virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)              {return S_FALSE;}
 280
 281         // decklink_frame
 282
 283         const core::audio_buffer& audio_data()
 284         {
 285                 return frame_.audio_data();
 286         }
 287
 288         int64_t get_age_millis() const
 289         {
 290                 return frame_.get_age_millis();
 291         }
 292 };
 293
 294 template <typename Configuration>
 295 struct key_video_context : public IDeckLinkVideoOutputCallback, boost::noncopyable
 296 {
 297         const configuration                                     config_;
 298         com_ptr<IDeckLink>                                      decklink_                                       = get_device(config_.key_device_index());
 299         com_iface_ptr<IDeckLinkOutput>          output_                                         = iface_cast<IDeckLinkOutput>(decklink_);
 300         com_iface_ptr<IDeckLinkKeyer>           keyer_                                          = iface_cast<IDeckLinkKeyer>(decklink_, true);
 301         com_iface_ptr<IDeckLinkAttributes>      attributes_                                     = iface_cast<IDeckLinkAttributes>(decklink_);
 302         com_iface_ptr<Configuration>            configuration_                          = iface_cast<Configuration>(decklink_);
 303         tbb::atomic<int64_t>                            current_presentation_delay_;
 304         tbb::atomic<int64_t>                            scheduled_frames_completed_;
 305
 306         key_video_context(const configuration& config, const std::wstring& print)
 307                 : config_(config)
 308         {
 309                 current_presentation_delay_ = 0;
 310                 scheduled_frames_completed_ = 0;
 311
 312                 set_latency(configuration_, config.latency, print);
 313                 set_keyer(attributes_, keyer_, config.keyer, print);
 314
 315                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 316                         CASPAR_THROW_EXCEPTION(caspar_exception()
 317                                         << msg_info(print + L" Failed to set key playback completion callback.")
 318                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 319         }
 320
 321         template<typename Print>
 322         void enable_video(BMDDisplayMode display_mode, const Print& print)
 323         {
 324                 if (FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 325                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable key video output."));
 326
 327                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 328                         CASPAR_THROW_EXCEPTION(caspar_exception()
 329                                         << msg_info(print() + L" Failed to set key playback completion callback.")
 330                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 331         }
 332
 333         virtual ~key_video_context()
 334         {
 335                 if (output_)
 336                 {
 337                         output_->StopScheduledPlayback(0, nullptr, 0);
 338                         output_->DisableVideoOutput();
 339                 }
 340         }
 341
 342         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 343         virtual ULONG STDMETHODCALLTYPE AddRef()                                                        {return 1;}
 344         virtual ULONG STDMETHODCALLTYPE Release()                                                       {return 1;}
 345
 346         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 347         {
 348                 return S_OK;
 349         }
 350
 351         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(
 352                         IDeckLinkVideoFrame* completed_frame,
 353                         BMDOutputFrameCompletionResult result)
 354         {
 355                 auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 356                 current_presentation_delay_ = dframe->get_age_millis();
 357                 ++scheduled_frames_completed_;
 358
 359                 // Let the fill callback keep the pace, so no scheduling here.
 360
 361                 return S_OK;
 362         }
 363 };
 364
 365 template <typename Configuration>
 366 struct decklink_consumer : public IDeckLinkVideoOutputCallback, boost::noncopyable
 367 {
 368         const int                                                                                       channel_index_;
 369         const configuration                                                                     config_;
 370
 371         com_ptr<IDeckLink>                                                                      decklink_                               = get_device(config_.device_index);
 372         com_iface_ptr<IDeckLinkOutput>                                          output_                                 = iface_cast<IDeckLinkOutput>(decklink_);
 373         com_iface_ptr<Configuration>                                            configuration_                  = iface_cast<Configuration>(decklink_);
 374         com_iface_ptr<IDeckLinkKeyer>                                           keyer_                                  = iface_cast<IDeckLinkKeyer>(decklink_, true);
 375         com_iface_ptr<IDeckLinkAttributes>                                      attributes_                             = iface_cast<IDeckLinkAttributes>(decklink_);
 376
 377         tbb::spin_mutex                                                                         exception_mutex_;
 378         std::exception_ptr                                                                      exception_;
 379
 380         tbb::atomic<bool>                                                                       is_running_;
 381
 382         const std::wstring                                                                      model_name_                             = get_model_name(decklink_);
 383         bool                                                                                            will_attempt_dma_;
 384         const core::video_format_desc                                           format_desc_;
 385         const core::audio_channel_layout                                        in_channel_layout_;
 386         const core::audio_channel_layout                                        out_channel_layout_             = config_.get_adjusted_layout(in_channel_layout_);
 387         core::audio_channel_remapper                                            channel_remapper_               { in_channel_layout_, out_channel_layout_ };
 388         const int                                                                                       buffer_size_                    = config_.buffer_depth(); // Minimum buffer-size 3.
 389
 390         long long                                                                                       video_scheduled_                = 0;
 391         long long                                                                                       audio_scheduled_                = 0;
 392
 393         int                                                                                                     preroll_count_                  = 0;
 394
 395         boost::circular_buffer<std::vector<int32_t>>            audio_container_                { buffer_size_ + 1 };
 396
 397         tbb::concurrent_bounded_queue<core::const_frame>        frame_buffer_;
 398         caspar::semaphore                                                                       ready_for_new_frames_   { 0 };
 399
 400         spl::shared_ptr<diagnostics::graph>                                     graph_;
 401         caspar::timer                                                                           tick_timer_;
 402         reference_signal_detector                                                       reference_signal_detector_      { output_ };
 403         tbb::atomic<int64_t>                                                            current_presentation_delay_;
 404         tbb::atomic<int64_t>                                                            scheduled_frames_completed_;
 405         std::unique_ptr<key_video_context<Configuration>>       key_context_;
 406
 407 public:
 408         decklink_consumer(
 409                         const configuration& config,
 410                         const core::video_format_desc& format_desc,
 411                         const core::audio_channel_layout& in_channel_layout,
 412                         int channel_index)
 413                 : channel_index_(channel_index)
 414                 , config_(config)
 415                 , format_desc_(format_desc)
 416                 , in_channel_layout_(in_channel_layout)
 417         {
 418                 is_running_ = true;
 419                 current_presentation_delay_ = 0;
 420                 scheduled_frames_completed_ = 0;
 421
 422                 frame_buffer_.set_capacity(1);
 423
 424                 if (config.keyer == configuration::keyer_t::external_separate_device_keyer)
 425                         key_context_.reset(new key_video_context<Configuration>(config, print()));
 426
 427                 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
 428                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
 429                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
 430                 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
 431                 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
 432                 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
 433
 434                 if (key_context_)
 435                 {
 436                         graph_->set_color("key-offset", diagnostics::color(1.0f, 0.0f, 0.0f));
 437                 }
 438
 439                 graph_->set_text(print());
 440                 diagnostics::register_graph(graph_);
 441
 442                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault, will_attempt_dma_));
 443
 444                 if(config.embedded_audio)
 445                         enable_audio();
 446
 447                 set_latency(configuration_, config.latency, print());
 448                 set_keyer(attributes_, keyer_, config.keyer, print());
 449
 450                 if(config.embedded_audio)
 451                         output_->BeginAudioPreroll();
 452
 453                 for (int n = 0; n < buffer_size_; ++n)
 454                 {
 455                         if (config.embedded_audio)
 456                                 schedule_next_audio(core::mutable_audio_buffer(format_desc_.audio_cadence[n % format_desc_.audio_cadence.size()] * out_channel_layout_.num_channels, 0));
 457
 458                         schedule_next_video(core::const_frame::empty());
 459                 }
 460
 461                 if (config.embedded_audio)
 462                 {
 463                         // Preroll one extra frame worth of audio
 464                         schedule_next_audio(core::mutable_audio_buffer(format_desc_.audio_cadence[buffer_size_ % format_desc_.audio_cadence.size()] * out_channel_layout_.num_channels, 0));
 465                         output_->EndAudioPreroll();
 466                 }
 467
 468                 start_playback();
 469         }
 470
 471         ~decklink_consumer()
 472         {
 473                 is_running_ = false;
 474                 frame_buffer_.try_push(core::const_frame::empty());
 475
 476                 if(output_ != nullptr)
 477                 {
 478                         output_->StopScheduledPlayback(0, nullptr, 0);
 479                         if(config_.embedded_audio)
 480                                 output_->DisableAudioOutput();
 481                         output_->DisableVideoOutput();
 482                 }
 483         }
 484
 485         void enable_audio()
 486         {
 487                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, out_channel_layout_.num_channels, bmdAudioOutputStreamTimestamped)))
 488                                 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable audio output."));
 489
 490                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
 491         }
 492
 493         void enable_video(BMDDisplayMode display_mode)
 494         {
 495                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 496                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable fill video output."));
 497
 498                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 499                         CASPAR_THROW_EXCEPTION(caspar_exception()
 500                                                                         << msg_info(print() + L" Failed to set fill playback completion callback.")
 501                                                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 502
 503                 if (key_context_)
 504                         key_context_->enable_video(display_mode, [this]() { return print(); });
 505         }
 506
 507         void start_playback()
 508         {
 509                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 510                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule fill playback."));
 511
 512                 if (key_context_ && FAILED(key_context_->output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 513                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule key playback."));
 514         }
 515
 516         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 517         virtual ULONG STDMETHODCALLTYPE AddRef()                                        {return 1;}
 518         virtual ULONG STDMETHODCALLTYPE Release()                               {return 1;}
 519
 520         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 521         {
 522                 is_running_ = false;
 523                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
 524                 return S_OK;
 525         }
 526
 527         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result)
 528         {
 529                 if(!is_running_)
 530                         return E_FAIL;
 531
 532                 try
 533                 {
 534                         auto tick_time = tick_timer_.elapsed()*format_desc_.fps * 0.5;
 535                         graph_->set_value("tick-time", tick_time);
 536                         tick_timer_.restart();
 537
 538                         reference_signal_detector_.detect_change([this]() { return print(); });
 539
 540                         auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 541                         current_presentation_delay_ = dframe->get_age_millis();
 542                         ++scheduled_frames_completed_;
 543
 544                         if (key_context_)
 545                                 graph_->set_value(
 546                                                 "key-offset",
 547                                                 static_cast<double>(
 548                                                                 scheduled_frames_completed_
 549                                                                 - key_context_->scheduled_frames_completed_)
 550                                                 * 0.1 + 0.5);
 551
 552                         if(result == bmdOutputFrameDisplayedLate)
 553                         {
 554                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "late-frame");
 555                                 video_scheduled_ += format_desc_.duration;
 556                                 audio_scheduled_ += dframe->audio_data().size() / in_channel_layout_.num_channels;
 557                         }
 558                         else if(result == bmdOutputFrameDropped)
 559                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "dropped-frame");
 560                         else if(result == bmdOutputFrameFlushed)
 561                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "flushed-frame");
 562
 563                         UINT32 buffered;
 564                         output_->GetBufferedVideoFrameCount(&buffered);
 565                         graph_->set_value("buffered-video", static_cast<double>(buffered) / (config_.buffer_depth()));
 566
 567                         if (config_.embedded_audio)
 568                         {
 569                                 output_->GetBufferedAudioSampleFrameCount(&buffered);
 570                                 graph_->set_value("buffered-audio", static_cast<double>(buffered) / (format_desc_.audio_cadence[0] * config_.buffer_depth()));
 571                         }
 572
 573                         auto frame = core::const_frame::empty();
 574
 575                         frame_buffer_.pop(frame);
 576                         ready_for_new_frames_.release();
 577
 578                         if (!is_running_)
 579                                 return E_FAIL;
 580
 581                         if (config_.embedded_audio)
 582                                 schedule_next_audio(channel_remapper_.mix_and_rearrange(frame.audio_data()));
 583
 584                         schedule_next_video(frame);
 585                 }
 586                 catch(...)
 587                 {
 588                         lock(exception_mutex_, [&]
 589                         {
 590                                 exception_ = std::current_exception();
 591                         });
 592                         return E_FAIL;
 593                 }
 594
 595                 return S_OK;
 596         }
 597
 598         template<typename T>
 599         void schedule_next_audio(const T& audio_data)
 600         {
 601                 auto sample_frame_count = static_cast<int>(audio_data.size()/out_channel_layout_.num_channels);
 602
 603                 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
 604
 605                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
 606                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
 607
 608                 audio_scheduled_ += sample_frame_count;
 609         }
 610
 611         void schedule_next_video(core::const_frame frame)
 612         {
 613                 if (key_context_)
 614                 {
 615                         auto key_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, true, will_attempt_dma_));
 616                         if (FAILED(key_context_->output_->ScheduleVideoFrame(get_raw(key_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 617                                 CASPAR_LOG(error) << print() << L" Failed to schedule key video.";
 618                 }
 619
 620                 auto fill_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, config_.key_only, will_attempt_dma_));
 621                 if (FAILED(output_->ScheduleVideoFrame(get_raw(fill_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 622                         CASPAR_LOG(error) << print() << L" Failed to schedule fill video.";
 623
 624                 video_scheduled_ += format_desc_.duration;
 625         }
 626
 627         std::future<bool> send(core::const_frame frame)
 628         {
 629                 auto exception = lock(exception_mutex_, [&]
 630                 {
 631                         return exception_;
 632                 });
 633
 634                 if(exception != nullptr)
 635                         std::rethrow_exception(exception);
 636
 637                 if(!is_running_)
 638                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Is not running."));
 639
 640                 frame_buffer_.push(frame);
 641
 642                 auto send_completion = spl::make_shared<std::promise<bool>>();
 643
 644                 ready_for_new_frames_.acquire(1, [send_completion]
 645                 {
 646                         send_completion->set_value(true);
 647                 });
 648
 649                 return send_completion->get_future();
 650         }
 651
 652         std::wstring print() const
 653         {
 654                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 655                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 656                                 boost::lexical_cast<std::wstring>(config_.device_index) +
 657                                 L"&&" +
 658                                 boost::lexical_cast<std::wstring>(config_.key_device_index()) +
 659                                 L"|" +
 660                                 format_desc_.name + L"]";
 661                 else
 662                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 663                                 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
 664         }
 665 };
 666
 667 template <typename Configuration>
 668 struct decklink_consumer_proxy : public core::frame_consumer
 669 {
 670         core::monitor::subject                                                          monitor_subject_;
 671         const configuration                                                                     config_;
 672         std::unique_ptr<decklink_consumer<Configuration>>       consumer_;
 673         core::video_format_desc                                                         format_desc_;
 674         executor                                                                                        executor_;
 675 public:
 676
 677         decklink_consumer_proxy(const configuration& config)
 678                 : config_(config)
 679                 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
 680         {
 681                 auto ctx = core::diagnostics::call_context::for_thread();
 682                 executor_.begin_invoke([=]
 683                 {
 684                         core::diagnostics::call_context::for_thread() = ctx;
 685                         com_initialize();
 686                 });
 687         }
 688
 689         ~decklink_consumer_proxy()
 690         {
 691                 executor_.invoke([=]
 692                 {
 693                         consumer_.reset();
 694                         com_uninitialize();
 695                 });
 696         }
 697
 698         // frame_consumer
 699
 700         void initialize(const core::video_format_desc& format_desc, const core::audio_channel_layout& channel_layout, int channel_index) override
 701         {
 702                 format_desc_ = format_desc;
 703                 executor_.invoke([=]
 704                 {
 705                         consumer_.reset();
 706                         consumer_.reset(new decklink_consumer<Configuration>(config_, format_desc, channel_layout, channel_index));
 707                 });
 708         }
 709
 710         std::future<bool> send(core::const_frame frame) override
 711         {
 712                 return consumer_->send(frame);
 713         }
 714
 715         std::wstring print() const override
 716         {
 717                 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
 718         }
 719
 720         std::wstring name() const override
 721         {
 722                 return L"decklink";
 723         }
 724
 725         boost::property_tree::wptree info() const override
 726         {
 727                 boost::property_tree::wptree info;
 728                 info.add(L"type", L"decklink");
 729                 info.add(L"key-only", config_.key_only);
 730                 info.add(L"device", config_.device_index);
 731
 732                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 733                 {
 734                         info.add(L"key-device", config_.key_device_index());
 735                 }
 736
 737                 info.add(L"low-latency", config_.latency == configuration::latency_t::low_latency);
 738                 info.add(L"embedded-audio", config_.embedded_audio);
 739                 info.add(L"presentation-frame-age", presentation_frame_age_millis());
 740                 //info.add(L"internal-key", config_.internal_key);
 741                 return info;
 742         }
 743
 744         int buffer_depth() const override
 745         {
 746                 return config_.buffer_depth() + 2;
 747         }
 748
 749         int index() const override
 750         {
 751                 return 300 + config_.device_index;
 752         }
 753
 754         int64_t presentation_frame_age_millis() const override
 755         {
 756                 return consumer_ ? static_cast<int64_t>(consumer_->current_presentation_delay_) : 0;
 757         }
 758
 759         core::monitor::subject& monitor_output()
 760         {
 761                 return monitor_subject_;
 762         }
 763 };
 764
 765 const software_version<3>& get_driver_version()
 766 {
 767         static software_version<3> version(u8(get_version()));
 768
 769         return version;
 770 }
 771
 772 const software_version<3> get_new_configuration_api_version()
 773 {
 774         static software_version<3> NEW_CONFIGURATION_API("10.2");
 775
 776         return NEW_CONFIGURATION_API;
 777 }
 778
 779 void describe_consumer(core::help_sink& sink, const core::help_repository& repo)
 780 {
 781         sink.short_description(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 782         sink.syntax(L"DECKLINK "
 783                                 L"{[device_index:int]|1} "
 784                                 L"{[keyer:INTERNAL_KEY,EXTERNAL_KEY,EXTERNAL_SEPARATE_DEVICE_KEY]} "
 785                                 L"{[low_latency:LOW_LATENCY]} "
 786                                 L"{[embedded_audio:EMBEDDED_AUDIO]} "
 787                                 L"{[key_only:KEY_ONLY]} "
 788                                 L"{CHANNEL_LAYOUT [channel_layout:string]}");
 789         sink.para()->text(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 790         sink.definitions()
 791                 ->item(L"device_index", L"The Blackmagic video card to use (See Blackmagic control panel for card order). Default is 1.")
 792                 ->item(L"keyer",
 793                                 L"If given tries to enable either internal or external keying. Not all Blackmagic cards supports this. "
 794                                 L"There is also a third experimental option (EXTERNAL_SEPARATE_DEVICE_KEY) which allocates device_index + 1 for synhronized key output.")
 795                 ->item(L"low_latency", L"Tries to enable low latency if given.")
 796                 ->item(L"embedded_audio", L"Embeds the audio into the SDI signal if given.")
 797                 ->item(L"key_only",
 798                                 L" will extract only the alpha channel from the "
 799                                 L"channel. This is useful when you have two SDI video cards, and neither has native support "
 800                                 L"for separate fill/key output")
 801                 ->item(L"channel_layout", L"If specified, overrides the audio channel layout used by the channel.");
 802         sink.para()->text(L"Examples:");
 803         sink.example(L">> ADD 1 DECKLINK", L"for using the default device_index of 1.");
 804         sink.example(L">> ADD 1 DECKLINK 2", L"uses device_index 2.");
 805         sink.example(L">> ADD 1 DECKLINK 1 EXTERNAL_KEY EMBEDDED_AUDIO");
 806         sink.example(
 807                         L">> ADD 1 DECKLINK 1 EMBEDDED_AUDIO\n"
 808                         L">> ADD 1 DECKLINK 2 KEY_ONLY", L"uses device with index 1 as fill output with audio and device with index 2 as key output.");
 809         sink.example(
 810                         L">> ADD 1 DECKLINK 1 EXTERNAL_SEPARATE_DEVICE_KEY EMBEDDED_AUDIO",
 811                         L"Uses device 2 for key output. May give better sync between key and fill than the previous method.");
 812 }
 813
 814 spl::shared_ptr<core::frame_consumer> create_consumer(
 815                 const std::vector<std::wstring>& params, core::interaction_sink*, std::vector<spl::shared_ptr<core::video_channel>> channels)
 816 {
 817         if (params.size() < 1 || !boost::iequals(params.at(0), L"DECKLINK"))
 818                 return core::frame_consumer::empty();
 819
 820         configuration config;
 821
 822         if (params.size() > 1)
 823                 config.device_index = boost::lexical_cast<int>(params.at(1));
 824
 825         if (contains_param(L"INTERNAL_KEY", params))
 826                 config.keyer = configuration::keyer_t::internal_keyer;
 827         else if (contains_param(L"EXTERNAL_KEY", params))
 828                 config.keyer = configuration::keyer_t::external_keyer;
 829         else if (contains_param(L"EXTERNAL_SEPARATE_DEVICE_KEY", params))
 830                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 831         else
 832                 config.keyer = configuration::keyer_t::default_keyer;
 833
 834         if (contains_param(L"LOW_LATENCY", params))
 835                 config.latency = configuration::latency_t::low_latency;
 836
 837         config.embedded_audio   = contains_param(L"EMBEDDED_AUDIO", params);
 838         config.key_only                 = contains_param(L"KEY_ONLY", params);
 839
 840         auto channel_layout = get_param(L"CHANNEL_LAYOUT", params);
 841
 842         if (!channel_layout.empty())
 843         {
 844                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(channel_layout);
 845
 846                 if (!found_layout)
 847                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + channel_layout + L" not found."));
 848
 849                 config.out_channel_layout = *found_layout;
 850         }
 851
 852         bool old_configuration_api = get_driver_version() < get_new_configuration_api_version();
 853
 854         if (old_configuration_api)
 855                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration_v10_2>>(config);
 856         else
 857                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration>>(config);
 858 }
 859
 860 spl::shared_ptr<core::frame_consumer> create_preconfigured_consumer(
 861                 const boost::property_tree::wptree& ptree, core::interaction_sink*, std::vector<spl::shared_ptr<core::video_channel>> channels)
 862 {
 863         configuration config;
 864
 865         auto keyer = ptree.get(L"keyer", L"default");
 866         if(keyer == L"external")
 867                 config.keyer = configuration::keyer_t::external_keyer;
 868         else if(keyer == L"internal")
 869                 config.keyer = configuration::keyer_t::internal_keyer;
 870         else if (keyer == L"external_separate_device")
 871                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 872
 873         auto latency = ptree.get(L"latency", L"default");
 874         if(latency == L"low")
 875                 config.latency = configuration::latency_t::low_latency;
 876         else if(latency == L"normal")
 877                 config.latency = configuration::latency_t::normal_latency;
 878
 879         auto channel_layout = ptree.get_optional<std::wstring>(L"channel-layout");
 880
 881         if (channel_layout)
 882         {
 883                 CASPAR_SCOPED_CONTEXT_MSG("/channel-layout")
 884
 885                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(*channel_layout);
 886
 887                 if (!found_layout)
 888                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + *channel_layout + L" not found."));
 889
 890                 config.out_channel_layout = *found_layout;
 891         }
 892
 893         config.key_only                         = ptree.get(L"key-only",                config.key_only);
 894         config.device_index                     = ptree.get(L"device",                  config.device_index);
 895         config.key_device_idx           = ptree.get(L"key-device",              config.key_device_idx);
 896         config.embedded_audio           = ptree.get(L"embedded-audio",  config.embedded_audio);
 897         config.base_buffer_depth        = ptree.get(L"buffer-depth",    config.base_buffer_depth);
 898
 899         bool old_configuration_api = get_driver_version() < get_new_configuration_api_version();
 900
 901         if (old_configuration_api)
 902                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration_v10_2>>(config);
 903         else
 904                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration>>(config);
 905 }
 906
 907 }}
 908
 909 /*
 910 ##############################################################################
 911 Pre-rolling
 912
 913 Mail: 2011-05-09
 914
 915 Yoshan
 916 BMD Developer Support
 917 developer@blackmagic-design.com
 918
 919 -----------------------------------------------------------------------------
 920
 921 Thanks for your inquiry. The minimum number of frames that you can preroll
 922 for scheduled playback is three frames for video and four frames for audio.
 923 As you mentioned if you preroll less frames then playback will not start or
 924 playback will be very sporadic. From our experience with Media Express, we
 925 recommended that at least seven frames are prerolled for smooth playback.
 926
 927 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
 928 There can be around 3 frames worth of latency on scheduled output.
 929 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
 930 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync()
 931 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
 932 guarantee that the provided frame will be output as soon the previous
 933 frame output has been completed.
 934 ################################################################################
 935 */
 936
 937 /*
 938 ##############################################################################
 939 Async DMA Transfer without redundant copying
 940
 941 Mail: 2011-05-10
 942
 943 Yoshan
 944 BMD Developer Support
 945 developer@blackmagic-design.com
 946
 947 -----------------------------------------------------------------------------
 948
 949 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
 950 and providing a pointer to your video buffer when GetBytes() is called.
 951 This may help to keep copying to a minimum. Please ensure that the pixel
 952 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
 953 have to colourspace convert which may result in additional copying.
 954 ################################################################################
 955 */