git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Robert Nagy, ronag89@gmail.com
  20 */
  21
  22 #include "../StdAfx.h"
  23
  24 #include "decklink_consumer.h"
  25
  26 #include "../util/util.h"
  27 #include "../decklink.h"
  28
  29 #include "../decklink_api.h"
  30
  31 #include <core/frame/frame.h>
  32 #include <core/frame/audio_channel_layout.h>
  33 #include <core/mixer/audio/audio_mixer.h>
  34 #include <core/consumer/frame_consumer.h>
  35 #include <core/diagnostics/call_context.h>
  36 #include <core/help/help_sink.h>
  37 #include <core/help/help_repository.h>
  38
  39 #include <common/executor.h>
  40 #include <common/lock.h>
  41 #include <common/diagnostics/graph.h>
  42 #include <common/except.h>
  43 #include <common/memshfl.h>
  44 #include <common/memcpy.h>
  45 #include <common/no_init_proxy.h>
  46 #include <common/array.h>
  47 #include <common/future.h>
  48 #include <common/cache_aligned_vector.h>
  49 #include <common/timer.h>
  50 #include <common/param.h>
  51 #include <common/software_version.h>
  52
  53 #include <tbb/concurrent_queue.h>
  54
  55 #include <common/assert.h>
  56 #include <boost/lexical_cast.hpp>
  57 #include <boost/circular_buffer.hpp>
  58 #include <boost/property_tree/ptree.hpp>
  59 #include <boost/thread/mutex.hpp>
  60
  61 #include <future>
  62
  63 namespace caspar { namespace decklink {
  64
  65 struct configuration
  66 {
  67         enum class keyer_t
  68         {
  69                 internal_keyer,
  70                 external_keyer,
  71                 external_separate_device_keyer,
  72                 default_keyer                                   = external_keyer
  73         };
  74
  75         enum class latency_t
  76         {
  77                 low_latency,
  78                 normal_latency,
  79                 default_latency = normal_latency
  80         };
  81
  82         int                                                     device_index            = 1;
  83         int                                                     key_device_idx          = 0;
  84         bool                                            embedded_audio          = false;
  85         keyer_t                                         keyer                           = keyer_t::default_keyer;
  86         latency_t                                       latency                         = latency_t::default_latency;
  87         bool                                            key_only                        = false;
  88         int                                                     base_buffer_depth       = 3;
  89         core::audio_channel_layout      out_channel_layout      = core::audio_channel_layout::invalid();
  90
  91         int buffer_depth() const
  92         {
  93                 return base_buffer_depth + (latency == latency_t::low_latency ? 0 : 1);
  94         }
  95
  96         int key_device_index() const
  97         {
  98                 return key_device_idx == 0 ? device_index + 1 : key_device_idx;
  99         }
 100
 101         core::audio_channel_layout get_adjusted_layout(const core::audio_channel_layout& in_layout) const
 102         {
 103                 auto adjusted = out_channel_layout == core::audio_channel_layout::invalid() ? in_layout : out_channel_layout;
 104
 105                 if (adjusted.num_channels == 1) // Duplicate mono-signal into both left and right.
 106                 {
 107                         adjusted.num_channels = 2;
 108                         adjusted.channel_order.push_back(adjusted.channel_order.at(0)); // Usually FC -> FC FC
 109                 }
 110                 else if (adjusted.num_channels == 2)
 111                 {
 112                         adjusted.num_channels = 2;
 113                 }
 114                 else if (adjusted.num_channels <= 8)
 115                 {
 116                         adjusted.num_channels = 8;
 117                 }
 118                 else // Over 8 always pad to 16 or drop >16
 119                 {
 120                         adjusted.num_channels = 16;
 121                 }
 122
 123                 return adjusted;
 124         }
 125 };
 126
 127 template <typename Configuration>
 128 void set_latency(
 129                 const com_iface_ptr<Configuration>& config,
 130                 configuration::latency_t latency,
 131                 const std::wstring& print)
 132 {
 133         if (latency == configuration::latency_t::low_latency)
 134         {
 135                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
 136                 CASPAR_LOG(info) << print << L" Enabled low-latency mode.";
 137         }
 138         else if (latency == configuration::latency_t::normal_latency)
 139         {
 140                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
 141                 CASPAR_LOG(info) << print << L" Disabled low-latency mode.";
 142         }
 143 }
 144
 145 void set_keyer(
 146                 const com_iface_ptr<IDeckLinkAttributes>& attributes,
 147                 const com_iface_ptr<IDeckLinkKeyer>& decklink_keyer,
 148                 configuration::keyer_t keyer,
 149                 const std::wstring& print)
 150 {
 151         if (keyer == configuration::keyer_t::internal_keyer)
 152         {
 153                 BOOL value = true;
 154                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
 155                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 156                 else if (FAILED(decklink_keyer->Enable(FALSE)))
 157                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 158                 else if (FAILED(decklink_keyer->SetLevel(255)))
 159                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 160                 else
 161                         CASPAR_LOG(info) << print << L" Enabled internal keyer.";
 162         }
 163         else if (keyer == configuration::keyer_t::external_keyer)
 164         {
 165                 BOOL value = true;
 166                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
 167                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 168                 else if (FAILED(decklink_keyer->Enable(TRUE)))
 169                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 170                 else if (FAILED(decklink_keyer->SetLevel(255)))
 171                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 172                 else
 173                         CASPAR_LOG(info) << print << L" Enabled external keyer.";
 174         }
 175 }
 176
 177 class decklink_frame : public IDeckLinkVideoFrame
 178 {
 179         tbb::atomic<int>                                                                ref_count_;
 180         core::const_frame                                                               frame_;
 181         const core::video_format_desc                                   format_desc_;
 182
 183         const bool                                                                              key_only_;
 184         bool                                                                                    needs_to_copy_;
 185         cache_aligned_vector<no_init_proxy<uint8_t>>    data_;
 186 public:
 187         decklink_frame(core::const_frame frame, const core::video_format_desc& format_desc, bool key_only, bool will_attempt_dma)
 188                 : frame_(frame)
 189                 , format_desc_(format_desc)
 190                 , key_only_(key_only)
 191         {
 192                 ref_count_ = 0;
 193
 194                 bool dma_transfer_from_gl_buffer_impossible;
 195
 196 #if !defined(_MSC_VER)
 197                 // On Linux Decklink cannot DMA transfer from memory returned by glMapBuffer (at least on nvidia)
 198                 dma_transfer_from_gl_buffer_impossible = true;
 199 #else
 200                 // On Windows it is possible.
 201                 dma_transfer_from_gl_buffer_impossible = false;
 202 #endif
 203
 204                 needs_to_copy_ = will_attempt_dma && dma_transfer_from_gl_buffer_impossible;
 205         }
 206
 207         // IUnknown
 208
 209         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)
 210         {
 211                 return E_NOINTERFACE;
 212         }
 213
 214         virtual ULONG STDMETHODCALLTYPE AddRef()
 215         {
 216                 return ++ref_count_;
 217         }
 218
 219         virtual ULONG STDMETHODCALLTYPE Release()
 220         {
 221                 if(--ref_count_ == 0)
 222                 {
 223                         delete this;
 224
 225                         return 0;
 226                 }
 227
 228                 return ref_count_;
 229         }
 230
 231         // IDecklinkVideoFrame
 232
 233         virtual long STDMETHODCALLTYPE GetWidth()                   {return static_cast<long>(format_desc_.width);}
 234         virtual long STDMETHODCALLTYPE GetHeight()                  {return static_cast<long>(format_desc_.height);}
 235         virtual long STDMETHODCALLTYPE GetRowBytes()                {return static_cast<long>(format_desc_.width*4);}
 236         virtual BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat()   {return bmdFormat8BitBGRA;}
 237         virtual BMDFrameFlags STDMETHODCALLTYPE GetFlags()                      {return bmdFrameFlagDefault;}
 238
 239         virtual HRESULT STDMETHODCALLTYPE GetBytes(void** buffer)
 240         {
 241                 try
 242                 {
 243                         if(static_cast<int>(frame_.image_data().size()) != format_desc_.size)
 244                         {
 245                                 data_.resize(format_desc_.size);
 246                                 *buffer = data_.data();
 247                         }
 248                         else if(key_only_)
 249                         {
 250                                 if(data_.empty())
 251                                 {
 252                                         data_.resize(frame_.image_data().size());
 253                                         aligned_memshfl(data_.data(), frame_.image_data().begin(), frame_.image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
 254                                 }
 255                                 *buffer = data_.data();
 256                         }
 257                         else
 258                         {
 259                                 *buffer = const_cast<uint8_t*>(frame_.image_data().begin());
 260
 261                                 if (needs_to_copy_)
 262                                 {
 263                                         data_.resize(frame_.image_data().size());
 264                                         fast_memcpy(data_.data(), *buffer, frame_.image_data().size());
 265                                         *buffer = data_.data();
 266                                 }
 267                         }
 268                 }
 269                 catch(...)
 270                 {
 271                         CASPAR_LOG_CURRENT_EXCEPTION();
 272                         return E_FAIL;
 273                 }
 274
 275                 return S_OK;
 276         }
 277
 278         virtual HRESULT STDMETHODCALLTYPE GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode) {return S_FALSE;}
 279         virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)              {return S_FALSE;}
 280
 281         // decklink_frame
 282
 283         const core::audio_buffer& audio_data()
 284         {
 285                 return frame_.audio_data();
 286         }
 287
 288         int64_t get_age_millis() const
 289         {
 290                 return frame_.get_age_millis();
 291         }
 292 };
 293
 294 template <typename Configuration>
 295 struct key_video_context : public IDeckLinkVideoOutputCallback, boost::noncopyable
 296 {
 297         const configuration                                     config_;
 298         com_ptr<IDeckLink>                                      decklink_                                       = get_device(config_.key_device_index());
 299         com_iface_ptr<IDeckLinkOutput>          output_                                         = iface_cast<IDeckLinkOutput>(decklink_);
 300         com_iface_ptr<IDeckLinkKeyer>           keyer_                                          = iface_cast<IDeckLinkKeyer>(decklink_, true);
 301         com_iface_ptr<IDeckLinkAttributes>      attributes_                                     = iface_cast<IDeckLinkAttributes>(decklink_);
 302         com_iface_ptr<Configuration>            configuration_                          = iface_cast<Configuration>(decklink_);
 303         tbb::atomic<int64_t>                            current_presentation_delay_;
 304         tbb::atomic<int64_t>                            scheduled_frames_completed_;
 305
 306         key_video_context(const configuration& config, const std::wstring& print)
 307                 : config_(config)
 308         {
 309                 current_presentation_delay_ = 0;
 310                 scheduled_frames_completed_ = 0;
 311
 312                 set_latency(configuration_, config.latency, print);
 313                 set_keyer(attributes_, keyer_, config.keyer, print);
 314
 315                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 316                         CASPAR_THROW_EXCEPTION(caspar_exception()
 317                                         << msg_info(print + L" Failed to set key playback completion callback.")
 318                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 319         }
 320
 321         template<typename Print>
 322         void enable_video(BMDDisplayMode display_mode, const Print& print)
 323         {
 324                 if (FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 325                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable key video output."));
 326
 327                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 328                         CASPAR_THROW_EXCEPTION(caspar_exception()
 329                                         << msg_info(print() + L" Failed to set key playback completion callback.")
 330                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 331         }
 332
 333         virtual ~key_video_context()
 334         {
 335                 if (output_)
 336                 {
 337                         output_->StopScheduledPlayback(0, nullptr, 0);
 338                         output_->DisableVideoOutput();
 339                 }
 340         }
 341
 342         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 343         virtual ULONG STDMETHODCALLTYPE AddRef()                                                        {return 1;}
 344         virtual ULONG STDMETHODCALLTYPE Release()                                                       {return 1;}
 345
 346         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 347         {
 348                 return S_OK;
 349         }
 350
 351         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(
 352                         IDeckLinkVideoFrame* completed_frame,
 353                         BMDOutputFrameCompletionResult result)
 354         {
 355                 auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 356                 current_presentation_delay_ = dframe->get_age_millis();
 357                 ++scheduled_frames_completed_;
 358
 359                 // Let the fill callback keep the pace, so no scheduling here.
 360
 361                 return S_OK;
 362         }
 363 };
 364
 365 template <typename Configuration>
 366 struct decklink_consumer : public IDeckLinkVideoOutputCallback, boost::noncopyable
 367 {
 368         const int                                                                                       channel_index_;
 369         const configuration                                                                     config_;
 370
 371         com_ptr<IDeckLink>                                                                      decklink_                               = get_device(config_.device_index);
 372         com_iface_ptr<IDeckLinkOutput>                                          output_                                 = iface_cast<IDeckLinkOutput>(decklink_);
 373         com_iface_ptr<Configuration>                                            configuration_                  = iface_cast<Configuration>(decklink_);
 374         com_iface_ptr<IDeckLinkKeyer>                                           keyer_                                  = iface_cast<IDeckLinkKeyer>(decklink_, true);
 375         com_iface_ptr<IDeckLinkAttributes>                                      attributes_                             = iface_cast<IDeckLinkAttributes>(decklink_);
 376
 377         tbb::spin_mutex                                                                         exception_mutex_;
 378         std::exception_ptr                                                                      exception_;
 379
 380         tbb::atomic<bool>                                                                       is_running_;
 381
 382         const std::wstring                                                                      model_name_                             = get_model_name(decklink_);
 383         bool                                                                                            will_attempt_dma_;
 384         const core::video_format_desc                                           format_desc_;
 385         const core::audio_channel_layout                                        in_channel_layout_;
 386         const core::audio_channel_layout                                        out_channel_layout_             = config_.get_adjusted_layout(in_channel_layout_);
 387         core::audio_channel_remapper                                            channel_remapper_               { in_channel_layout_, out_channel_layout_ };
 388         const int                                                                                       buffer_size_                    = config_.buffer_depth(); // Minimum buffer-size 3.
 389
 390         long long                                                                                       video_scheduled_                = 0;
 391         long long                                                                                       audio_scheduled_                = 0;
 392
 393         int                                                                                                     preroll_count_                  = 0;
 394
 395         boost::circular_buffer<std::vector<int32_t>>            audio_container_                { buffer_size_ + 1 };
 396
 397         tbb::concurrent_bounded_queue<core::const_frame>        frame_buffer_;
 398
 399         spl::shared_ptr<diagnostics::graph>                                     graph_;
 400         caspar::timer                                                                           tick_timer_;
 401         boost::mutex                                                                            send_completion_mutex_;
 402         std::packaged_task<bool ()>                                                     send_completion_;
 403         reference_signal_detector                                                       reference_signal_detector_      { output_ };
 404         tbb::atomic<int64_t>                                                            current_presentation_delay_;
 405         tbb::atomic<int64_t>                                                            scheduled_frames_completed_;
 406         std::unique_ptr<key_video_context<Configuration>>       key_context_;
 407
 408 public:
 409         decklink_consumer(
 410                         const configuration& config,
 411                         const core::video_format_desc& format_desc,
 412                         const core::audio_channel_layout& in_channel_layout,
 413                         int channel_index)
 414                 : channel_index_(channel_index)
 415                 , config_(config)
 416                 , format_desc_(format_desc)
 417                 , in_channel_layout_(in_channel_layout)
 418         {
 419                 is_running_ = true;
 420                 current_presentation_delay_ = 0;
 421                 scheduled_frames_completed_ = 0;
 422
 423                 frame_buffer_.set_capacity(1);
 424
 425                 if (config.keyer == configuration::keyer_t::external_separate_device_keyer)
 426                         key_context_.reset(new key_video_context<Configuration>(config, print()));
 427
 428                 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
 429                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
 430                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
 431                 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
 432                 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
 433                 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
 434
 435                 if (key_context_)
 436                 {
 437                         graph_->set_color("key-offset", diagnostics::color(1.0f, 0.0f, 0.0f));
 438                 }
 439
 440                 graph_->set_text(print());
 441                 diagnostics::register_graph(graph_);
 442
 443                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault, will_attempt_dma_));
 444
 445                 if(config.embedded_audio)
 446                         enable_audio();
 447
 448                 set_latency(configuration_, config.latency, print());
 449                 set_keyer(attributes_, keyer_, config.keyer, print());
 450
 451                 if(config.embedded_audio)
 452                         output_->BeginAudioPreroll();
 453
 454                 for (int n = 0; n < buffer_size_; ++n)
 455                 {
 456                         if (config.embedded_audio)
 457                                 schedule_next_audio(core::mutable_audio_buffer(format_desc_.audio_cadence[n % format_desc_.audio_cadence.size()] * out_channel_layout_.num_channels, 0));
 458
 459                         schedule_next_video(core::const_frame::empty());
 460                 }
 461
 462                 if (config.embedded_audio)
 463                 {
 464                         // Preroll one extra frame worth of audio
 465                         schedule_next_audio(core::mutable_audio_buffer(format_desc_.audio_cadence[buffer_size_ % format_desc_.audio_cadence.size()] * out_channel_layout_.num_channels, 0));
 466                         output_->EndAudioPreroll();
 467                 }
 468
 469                 start_playback();
 470         }
 471
 472         ~decklink_consumer()
 473         {
 474                 is_running_ = false;
 475                 frame_buffer_.try_push(core::const_frame::empty());
 476
 477                 if(output_ != nullptr)
 478                 {
 479                         output_->StopScheduledPlayback(0, nullptr, 0);
 480                         if(config_.embedded_audio)
 481                                 output_->DisableAudioOutput();
 482                         output_->DisableVideoOutput();
 483                 }
 484         }
 485
 486         void enable_audio()
 487         {
 488                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, out_channel_layout_.num_channels, bmdAudioOutputStreamTimestamped)))
 489                                 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable audio output."));
 490
 491                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
 492         }
 493
 494         void enable_video(BMDDisplayMode display_mode)
 495         {
 496                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 497                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable fill video output."));
 498
 499                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 500                         CASPAR_THROW_EXCEPTION(caspar_exception()
 501                                                                         << msg_info(print() + L" Failed to set fill playback completion callback.")
 502                                                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 503
 504                 if (key_context_)
 505                         key_context_->enable_video(display_mode, [this]() { return print(); });
 506         }
 507
 508         void start_playback()
 509         {
 510                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 511                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule fill playback."));
 512
 513                 if (key_context_ && FAILED(key_context_->output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 514                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule key playback."));
 515         }
 516
 517         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 518         virtual ULONG STDMETHODCALLTYPE AddRef()                                        {return 1;}
 519         virtual ULONG STDMETHODCALLTYPE Release()                               {return 1;}
 520
 521         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 522         {
 523                 is_running_ = false;
 524                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
 525                 return S_OK;
 526         }
 527
 528         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result)
 529         {
 530                 if(!is_running_)
 531                         return E_FAIL;
 532
 533                 try
 534                 {
 535                         auto tick_time = tick_timer_.elapsed()*format_desc_.fps * 0.5;
 536                         graph_->set_value("tick-time", tick_time);
 537                         tick_timer_.restart();
 538
 539                         reference_signal_detector_.detect_change([this]() { return print(); });
 540
 541                         auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 542                         current_presentation_delay_ = dframe->get_age_millis();
 543                         ++scheduled_frames_completed_;
 544
 545                         if (key_context_)
 546                                 graph_->set_value(
 547                                                 "key-offset",
 548                                                 static_cast<double>(
 549                                                                 scheduled_frames_completed_
 550                                                                 - key_context_->scheduled_frames_completed_)
 551                                                 * 0.1 + 0.5);
 552
 553                         if(result == bmdOutputFrameDisplayedLate)
 554                         {
 555                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "late-frame");
 556                                 video_scheduled_ += format_desc_.duration;
 557                                 audio_scheduled_ += dframe->audio_data().size() / in_channel_layout_.num_channels;
 558                         }
 559                         else if(result == bmdOutputFrameDropped)
 560                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "dropped-frame");
 561                         else if(result == bmdOutputFrameFlushed)
 562                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "flushed-frame");
 563
 564                         UINT32 buffered;
 565                         output_->GetBufferedVideoFrameCount(&buffered);
 566                         graph_->set_value("buffered-video", static_cast<double>(buffered) / (config_.buffer_depth()));
 567
 568                         if (config_.embedded_audio)
 569                         {
 570                                 output_->GetBufferedAudioSampleFrameCount(&buffered);
 571                                 graph_->set_value("buffered-audio", static_cast<double>(buffered) / (format_desc_.audio_cadence[0] * config_.buffer_depth()));
 572                         }
 573
 574                         auto frame = core::const_frame::empty();
 575
 576                         frame_buffer_.pop(frame);
 577
 578                         {
 579                                 boost::lock_guard<boost::mutex> lock(send_completion_mutex_);
 580
 581                                 if (send_completion_.valid())
 582                                 {
 583                                         send_completion_();
 584                                         send_completion_ = std::packaged_task<bool()>();
 585                                 }
 586                         }
 587
 588                         if (config_.embedded_audio)
 589                                 schedule_next_audio(channel_remapper_.mix_and_rearrange(frame.audio_data()));
 590
 591                         schedule_next_video(frame);
 592                 }
 593                 catch(...)
 594                 {
 595                         lock(exception_mutex_, [&]
 596                         {
 597                                 exception_ = std::current_exception();
 598                         });
 599                         return E_FAIL;
 600                 }
 601
 602                 return S_OK;
 603         }
 604
 605         template<typename T>
 606         void schedule_next_audio(const T& audio_data)
 607         {
 608                 auto sample_frame_count = static_cast<int>(audio_data.size()/out_channel_layout_.num_channels);
 609
 610                 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
 611
 612                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
 613                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
 614
 615                 audio_scheduled_ += sample_frame_count;
 616         }
 617
 618         void schedule_next_video(core::const_frame frame)
 619         {
 620                 if (key_context_)
 621                 {
 622                         auto key_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, true, will_attempt_dma_));
 623                         if (FAILED(key_context_->output_->ScheduleVideoFrame(get_raw(key_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 624                                 CASPAR_LOG(error) << print() << L" Failed to schedule key video.";
 625                 }
 626
 627                 auto fill_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, config_.key_only, will_attempt_dma_));
 628                 if (FAILED(output_->ScheduleVideoFrame(get_raw(fill_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 629                         CASPAR_LOG(error) << print() << L" Failed to schedule fill video.";
 630
 631                 video_scheduled_ += format_desc_.duration;
 632         }
 633
 634         std::future<bool> send(core::const_frame frame)
 635         {
 636                 auto exception = lock(exception_mutex_, [&]
 637                 {
 638                         return exception_;
 639                 });
 640
 641                 if(exception != nullptr)
 642                         std::rethrow_exception(exception);
 643
 644                 if(!is_running_)
 645                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Is not running."));
 646
 647                 if (frame_buffer_.try_push(frame))
 648                         return make_ready_future(true);
 649
 650                 boost::lock_guard<boost::mutex> lock(send_completion_mutex_);
 651
 652                 send_completion_ = std::packaged_task<bool ()>([frame, this] () mutable -> bool
 653                 {
 654                         frame_buffer_.push(frame);
 655
 656                         return true;
 657                 });
 658
 659                 return send_completion_.get_future();
 660         }
 661
 662         std::wstring print() const
 663         {
 664                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 665                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 666                                 boost::lexical_cast<std::wstring>(config_.device_index) +
 667                                 L"&&" +
 668                                 boost::lexical_cast<std::wstring>(config_.key_device_index()) +
 669                                 L"|" +
 670                                 format_desc_.name + L"]";
 671                 else
 672                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 673                                 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
 674         }
 675 };
 676
 677 template <typename Configuration>
 678 struct decklink_consumer_proxy : public core::frame_consumer
 679 {
 680         core::monitor::subject                                                          monitor_subject_;
 681         const configuration                                                                     config_;
 682         std::unique_ptr<decklink_consumer<Configuration>>       consumer_;
 683         core::video_format_desc                                                         format_desc_;
 684         executor                                                                                        executor_;
 685 public:
 686
 687         decklink_consumer_proxy(const configuration& config)
 688                 : config_(config)
 689                 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
 690         {
 691                 auto ctx = core::diagnostics::call_context::for_thread();
 692                 executor_.begin_invoke([=]
 693                 {
 694                         core::diagnostics::call_context::for_thread() = ctx;
 695                         com_initialize();
 696                 });
 697         }
 698
 699         ~decklink_consumer_proxy()
 700         {
 701                 executor_.invoke([=]
 702                 {
 703                         consumer_.reset();
 704                         com_uninitialize();
 705                 });
 706         }
 707
 708         // frame_consumer
 709
 710         void initialize(const core::video_format_desc& format_desc, const core::audio_channel_layout& channel_layout, int channel_index) override
 711         {
 712                 format_desc_ = format_desc;
 713                 executor_.invoke([=]
 714                 {
 715                         consumer_.reset();
 716                         consumer_.reset(new decklink_consumer<Configuration>(config_, format_desc, channel_layout, channel_index));
 717                 });
 718         }
 719
 720         std::future<bool> send(core::const_frame frame) override
 721         {
 722                 return consumer_->send(frame);
 723         }
 724
 725         std::wstring print() const override
 726         {
 727                 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
 728         }
 729
 730         std::wstring name() const override
 731         {
 732                 return L"decklink";
 733         }
 734
 735         boost::property_tree::wptree info() const override
 736         {
 737                 boost::property_tree::wptree info;
 738                 info.add(L"type", L"decklink");
 739                 info.add(L"key-only", config_.key_only);
 740                 info.add(L"device", config_.device_index);
 741
 742                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 743                 {
 744                         info.add(L"key-device", config_.key_device_index());
 745                 }
 746
 747                 info.add(L"low-latency", config_.latency == configuration::latency_t::low_latency);
 748                 info.add(L"embedded-audio", config_.embedded_audio);
 749                 info.add(L"presentation-frame-age", presentation_frame_age_millis());
 750                 //info.add(L"internal-key", config_.internal_key);
 751                 return info;
 752         }
 753
 754         int buffer_depth() const override
 755         {
 756                 return config_.buffer_depth() + 2;
 757         }
 758
 759         int index() const override
 760         {
 761                 return 300 + config_.device_index;
 762         }
 763
 764         int64_t presentation_frame_age_millis() const override
 765         {
 766                 return consumer_ ? static_cast<int64_t>(consumer_->current_presentation_delay_) : 0;
 767         }
 768
 769         core::monitor::subject& monitor_output()
 770         {
 771                 return monitor_subject_;
 772         }
 773 };
 774
 775 const software_version<3>& get_driver_version()
 776 {
 777         static software_version<3> version(u8(get_version()));
 778
 779         return version;
 780 }
 781
 782 const software_version<3> get_new_configuration_api_version()
 783 {
 784         static software_version<3> NEW_CONFIGURATION_API("10.2");
 785
 786         return NEW_CONFIGURATION_API;
 787 }
 788
 789 void describe_consumer(core::help_sink& sink, const core::help_repository& repo)
 790 {
 791         sink.short_description(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 792         sink.syntax(L"DECKLINK "
 793                                 L"{[device_index:int]|1} "
 794                                 L"{[keyer:INTERNAL_KEY,EXTERNAL_KEY,EXTERNAL_SEPARATE_DEVICE_KEY]} "
 795                                 L"{[low_latency:LOW_LATENCY]} "
 796                                 L"{[embedded_audio:EMBEDDED_AUDIO]} "
 797                                 L"{[key_only:KEY_ONLY]} "
 798                                 L"{CHANNEL_LAYOUT [channel_layout:string]}");
 799         sink.para()->text(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 800         sink.definitions()
 801                 ->item(L"device_index", L"The Blackmagic video card to use (See Blackmagic control panel for card order). Default is 1.")
 802                 ->item(L"keyer",
 803                                 L"If given tries to enable either internal or external keying. Not all Blackmagic cards supports this. "
 804                                 L"There is also a third experimental option (EXTERNAL_SEPARATE_DEVICE_KEY) which allocates device_index + 1 for synhronized key output.")
 805                 ->item(L"low_latency", L"Tries to enable low latency if given.")
 806                 ->item(L"embedded_audio", L"Embeds the audio into the SDI signal if given.")
 807                 ->item(L"key_only",
 808                                 L" will extract only the alpha channel from the "
 809                                 L"channel. This is useful when you have two SDI video cards, and neither has native support "
 810                                 L"for separate fill/key output")
 811                 ->item(L"channel_layout", L"If specified, overrides the audio channel layout used by the channel.");
 812         sink.para()->text(L"Examples:");
 813         sink.example(L">> ADD 1 DECKLINK", L"for using the default device_index of 1.");
 814         sink.example(L">> ADD 1 DECKLINK 2", L"uses device_index 2.");
 815         sink.example(L">> ADD 1 DECKLINK 1 EXTERNAL_KEY EMBEDDED_AUDIO");
 816         sink.example(
 817                         L">> ADD 1 DECKLINK 1 EMBEDDED_AUDIO\n"
 818                         L">> ADD 1 DECKLINK 2 KEY_ONLY", L"uses device with index 1 as fill output with audio and device with index 2 as key output.");
 819         sink.example(
 820                         L">> ADD 1 DECKLINK 1 EXTERNAL_SEPARATE_DEVICE_KEY EMBEDDED_AUDIO",
 821                         L"Uses device 2 for key output. May give better sync between key and fill than the previous method.");
 822 }
 823
 824 spl::shared_ptr<core::frame_consumer> create_consumer(
 825                 const std::vector<std::wstring>& params, core::interaction_sink*, std::vector<spl::shared_ptr<core::video_channel>> channels)
 826 {
 827         if (params.size() < 1 || !boost::iequals(params.at(0), L"DECKLINK"))
 828                 return core::frame_consumer::empty();
 829
 830         configuration config;
 831
 832         if (params.size() > 1)
 833                 config.device_index = boost::lexical_cast<int>(params.at(1));
 834
 835         if (contains_param(L"INTERNAL_KEY", params))
 836                 config.keyer = configuration::keyer_t::internal_keyer;
 837         else if (contains_param(L"EXTERNAL_KEY", params))
 838                 config.keyer = configuration::keyer_t::external_keyer;
 839         else if (contains_param(L"EXTERNAL_SEPARATE_DEVICE_KEY", params))
 840                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 841         else
 842                 config.keyer = configuration::keyer_t::default_keyer;
 843
 844         if (contains_param(L"LOW_LATENCY", params))
 845                 config.latency = configuration::latency_t::low_latency;
 846
 847         config.embedded_audio   = contains_param(L"EMBEDDED_AUDIO", params);
 848         config.key_only                 = contains_param(L"KEY_ONLY", params);
 849
 850         auto channel_layout = get_param(L"CHANNEL_LAYOUT", params);
 851
 852         if (!channel_layout.empty())
 853         {
 854                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(channel_layout);
 855
 856                 if (!found_layout)
 857                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + channel_layout + L" not found."));
 858
 859                 config.out_channel_layout = *found_layout;
 860         }
 861
 862         bool old_configuration_api = get_driver_version() < get_new_configuration_api_version();
 863
 864         if (old_configuration_api)
 865                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration_v10_2>>(config);
 866         else
 867                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration>>(config);
 868 }
 869
 870 spl::shared_ptr<core::frame_consumer> create_preconfigured_consumer(
 871                 const boost::property_tree::wptree& ptree, core::interaction_sink*, std::vector<spl::shared_ptr<core::video_channel>> channels)
 872 {
 873         configuration config;
 874
 875         auto keyer = ptree.get(L"keyer", L"default");
 876         if(keyer == L"external")
 877                 config.keyer = configuration::keyer_t::external_keyer;
 878         else if(keyer == L"internal")
 879                 config.keyer = configuration::keyer_t::internal_keyer;
 880         else if (keyer == L"external_separate_device")
 881                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 882
 883         auto latency = ptree.get(L"latency", L"default");
 884         if(latency == L"low")
 885                 config.latency = configuration::latency_t::low_latency;
 886         else if(latency == L"normal")
 887                 config.latency = configuration::latency_t::normal_latency;
 888
 889         auto channel_layout = ptree.get_optional<std::wstring>(L"channel-layout");
 890
 891         if (channel_layout)
 892         {
 893                 CASPAR_SCOPED_CONTEXT_MSG("/channel-layout")
 894
 895                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(*channel_layout);
 896
 897                 if (!found_layout)
 898                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + *channel_layout + L" not found."));
 899
 900                 config.out_channel_layout = *found_layout;
 901         }
 902
 903         config.key_only                         = ptree.get(L"key-only",                config.key_only);
 904         config.device_index                     = ptree.get(L"device",                  config.device_index);
 905         config.key_device_idx           = ptree.get(L"key-device",              config.key_device_idx);
 906         config.embedded_audio           = ptree.get(L"embedded-audio",  config.embedded_audio);
 907         config.base_buffer_depth        = ptree.get(L"buffer-depth",    config.base_buffer_depth);
 908
 909         bool old_configuration_api = get_driver_version() < get_new_configuration_api_version();
 910
 911         if (old_configuration_api)
 912                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration_v10_2>>(config);
 913         else
 914                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration>>(config);
 915 }
 916
 917 }}
 918
 919 /*
 920 ##############################################################################
 921 Pre-rolling
 922
 923 Mail: 2011-05-09
 924
 925 Yoshan
 926 BMD Developer Support
 927 developer@blackmagic-design.com
 928
 929 -----------------------------------------------------------------------------
 930
 931 Thanks for your inquiry. The minimum number of frames that you can preroll
 932 for scheduled playback is three frames for video and four frames for audio.
 933 As you mentioned if you preroll less frames then playback will not start or
 934 playback will be very sporadic. From our experience with Media Express, we
 935 recommended that at least seven frames are prerolled for smooth playback.
 936
 937 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
 938 There can be around 3 frames worth of latency on scheduled output.
 939 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
 940 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync()
 941 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
 942 guarantee that the provided frame will be output as soon the previous
 943 frame output has been completed.
 944 ################################################################################
 945 */
 946
 947 /*
 948 ##############################################################################
 949 Async DMA Transfer without redundant copying
 950
 951 Mail: 2011-05-10
 952
 953 Yoshan
 954 BMD Developer Support
 955 developer@blackmagic-design.com
 956
 957 -----------------------------------------------------------------------------
 958
 959 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
 960 and providing a pointer to your video buffer when GetBytes() is called.
 961 This may help to keep copying to a minimum. Please ensure that the pixel
 962 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
 963 have to colourspace convert which may result in additional copying.
 964 ################################################################################
 965 */