git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Robert Nagy, ronag89@gmail.com
  20 */
  21
  22 #include "../StdAfx.h"
  23
  24 #include "decklink_consumer.h"
  25
  26 #include "../util/util.h"
  27 #include "../decklink.h"
  28
  29 #include "../decklink_api.h"
  30
  31 #include <core/frame/frame.h>
  32 #include <core/frame/audio_channel_layout.h>
  33 #include <core/mixer/audio/audio_mixer.h>
  34 #include <core/consumer/frame_consumer.h>
  35 #include <core/diagnostics/call_context.h>
  36 #include <core/help/help_sink.h>
  37 #include <core/help/help_repository.h>
  38
  39 #include <common/executor.h>
  40 #include <common/lock.h>
  41 #include <common/diagnostics/graph.h>
  42 #include <common/except.h>
  43 #include <common/memshfl.h>
  44 #include <common/memcpy.h>
  45 #include <common/no_init_proxy.h>
  46 #include <common/array.h>
  47 #include <common/future.h>
  48 #include <common/cache_aligned_vector.h>
  49 #include <common/timer.h>
  50 #include <common/param.h>
  51 #include <common/software_version.h>
  52
  53 #include <tbb/concurrent_queue.h>
  54
  55 #include <common/assert.h>
  56 #include <boost/lexical_cast.hpp>
  57 #include <boost/circular_buffer.hpp>
  58 #include <boost/property_tree/ptree.hpp>
  59 #include <boost/thread/mutex.hpp>
  60
  61 #include <future>
  62
  63 namespace caspar { namespace decklink {
  64
  65 struct configuration
  66 {
  67         enum class keyer_t
  68         {
  69                 internal_keyer,
  70                 external_keyer,
  71                 external_separate_device_keyer,
  72                 default_keyer                                   = external_keyer
  73         };
  74
  75         enum class latency_t
  76         {
  77                 low_latency,
  78                 normal_latency,
  79                 default_latency = normal_latency
  80         };
  81
  82         int                                                     device_index            = 1;
  83         int                                                     key_device_idx          = 0;
  84         bool                                            embedded_audio          = false;
  85         keyer_t                                         keyer                           = keyer_t::default_keyer;
  86         latency_t                                       latency                         = latency_t::default_latency;
  87         bool                                            key_only                        = false;
  88         int                                                     base_buffer_depth       = 3;
  89         core::audio_channel_layout      out_channel_layout      = core::audio_channel_layout::invalid();
  90
  91         int buffer_depth() const
  92         {
  93                 return base_buffer_depth + (latency == latency_t::low_latency ? 0 : 1);
  94         }
  95
  96         int key_device_index() const
  97         {
  98                 return key_device_idx == 0 ? device_index + 1 : key_device_idx;
  99         }
 100
 101         core::audio_channel_layout get_adjusted_layout(const core::audio_channel_layout& in_layout) const
 102         {
 103                 auto adjusted = out_channel_layout == core::audio_channel_layout::invalid() ? in_layout : out_channel_layout;
 104
 105                 if (adjusted.num_channels == 1) // Duplicate mono-signal into both left and right.
 106                 {
 107                         adjusted.num_channels = 2;
 108                         adjusted.channel_order.push_back(adjusted.channel_order.at(0)); // Usually FC -> FC FC
 109                 }
 110                 else if (adjusted.num_channels == 2)
 111                 {
 112                         adjusted.num_channels = 2;
 113                 }
 114                 else if (adjusted.num_channels <= 8)
 115                 {
 116                         adjusted.num_channels = 8;
 117                 }
 118                 else // Over 8 always pad to 16 or drop >16
 119                 {
 120                         adjusted.num_channels = 16;
 121                 }
 122
 123                 return adjusted;
 124         }
 125 };
 126
 127 template <typename Configuration>
 128 void set_latency(
 129                 const com_iface_ptr<Configuration>& config,
 130                 configuration::latency_t latency,
 131                 const std::wstring& print)
 132 {
 133         if (latency == configuration::latency_t::low_latency)
 134         {
 135                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
 136                 CASPAR_LOG(info) << print << L" Enabled low-latency mode.";
 137         }
 138         else if (latency == configuration::latency_t::normal_latency)
 139         {
 140                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
 141                 CASPAR_LOG(info) << print << L" Disabled low-latency mode.";
 142         }
 143 }
 144
 145 void set_keyer(
 146                 const com_iface_ptr<IDeckLinkAttributes>& attributes,
 147                 const com_iface_ptr<IDeckLinkKeyer>& decklink_keyer,
 148                 configuration::keyer_t keyer,
 149                 const std::wstring& print)
 150 {
 151         if (keyer == configuration::keyer_t::internal_keyer)
 152         {
 153                 BOOL value = true;
 154                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
 155                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 156                 else if (FAILED(decklink_keyer->Enable(FALSE)))
 157                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 158                 else if (FAILED(decklink_keyer->SetLevel(255)))
 159                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 160                 else
 161                         CASPAR_LOG(info) << print << L" Enabled internal keyer.";
 162         }
 163         else if (keyer == configuration::keyer_t::external_keyer)
 164         {
 165                 BOOL value = true;
 166                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
 167                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 168                 else if (FAILED(decklink_keyer->Enable(TRUE)))
 169                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 170                 else if (FAILED(decklink_keyer->SetLevel(255)))
 171                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 172                 else
 173                         CASPAR_LOG(info) << print << L" Enabled external keyer.";
 174         }
 175 }
 176
 177 class decklink_frame : public IDeckLinkVideoFrame
 178 {
 179         tbb::atomic<int>                                                                ref_count_;
 180         core::const_frame                                                               frame_;
 181         const core::video_format_desc                                   format_desc_;
 182
 183         const bool                                                                              key_only_;
 184         bool                                                                                    needs_to_copy_;
 185         cache_aligned_vector<no_init_proxy<uint8_t>>    data_;
 186 public:
 187         decklink_frame(core::const_frame frame, const core::video_format_desc& format_desc, bool key_only, bool will_attempt_dma)
 188                 : frame_(frame)
 189                 , format_desc_(format_desc)
 190                 , key_only_(key_only)
 191         {
 192                 ref_count_ = 0;
 193
 194                 bool dma_transfer_from_gl_buffer_impossible;
 195
 196 #if !defined(_MSC_VER)
 197                 // On Linux Decklink cannot DMA transfer from memory returned by glMapBuffer (at least on nvidia)
 198                 dma_transfer_from_gl_buffer_impossible = true;
 199 #else
 200                 // On Windows it is possible.
 201                 dma_transfer_from_gl_buffer_impossible = false;
 202 #endif
 203
 204                 needs_to_copy_ = will_attempt_dma && dma_transfer_from_gl_buffer_impossible;
 205         }
 206
 207         // IUnknown
 208
 209         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)
 210         {
 211                 return E_NOINTERFACE;
 212         }
 213
 214         virtual ULONG STDMETHODCALLTYPE AddRef()
 215         {
 216                 return ++ref_count_;
 217         }
 218
 219         virtual ULONG STDMETHODCALLTYPE Release()
 220         {
 221                 if(--ref_count_ == 0)
 222                 {
 223                         delete this;
 224
 225                         return 0;
 226                 }
 227
 228                 return ref_count_;
 229         }
 230
 231         // IDecklinkVideoFrame
 232
 233         virtual long STDMETHODCALLTYPE GetWidth()                   {return static_cast<long>(format_desc_.width);}
 234         virtual long STDMETHODCALLTYPE GetHeight()                  {return static_cast<long>(format_desc_.height);}
 235         virtual long STDMETHODCALLTYPE GetRowBytes()                {return static_cast<long>(format_desc_.width*4);}
 236         virtual BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat()   {return bmdFormat8BitBGRA;}
 237         virtual BMDFrameFlags STDMETHODCALLTYPE GetFlags()                      {return bmdFrameFlagDefault;}
 238
 239         virtual HRESULT STDMETHODCALLTYPE GetBytes(void** buffer)
 240         {
 241                 try
 242                 {
 243                         if(static_cast<int>(frame_.image_data().size()) != format_desc_.size)
 244                         {
 245                                 data_.resize(format_desc_.size);
 246                                 *buffer = data_.data();
 247                         }
 248                         else if(key_only_)
 249                         {
 250                                 if(data_.empty())
 251                                 {
 252                                         data_.resize(frame_.image_data().size());
 253                                         aligned_memshfl(data_.data(), frame_.image_data().begin(), frame_.image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
 254                                 }
 255                                 *buffer = data_.data();
 256                         }
 257                         else
 258                         {
 259                                 *buffer = const_cast<uint8_t*>(frame_.image_data().begin());
 260
 261                                 if (needs_to_copy_)
 262                                 {
 263                                         data_.resize(frame_.image_data().size());
 264                                         fast_memcpy(data_.data(), *buffer, frame_.image_data().size());
 265                                         *buffer = data_.data();
 266                                 }
 267                         }
 268                 }
 269                 catch(...)
 270                 {
 271                         CASPAR_LOG_CURRENT_EXCEPTION();
 272                         return E_FAIL;
 273                 }
 274
 275                 return S_OK;
 276         }
 277
 278         virtual HRESULT STDMETHODCALLTYPE GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode) {return S_FALSE;}
 279         virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)              {return S_FALSE;}
 280
 281         // decklink_frame
 282
 283         const core::audio_buffer& audio_data()
 284         {
 285                 return frame_.audio_data();
 286         }
 287
 288         int64_t get_age_millis() const
 289         {
 290                 return frame_.get_age_millis();
 291         }
 292 };
 293
 294 template <typename Configuration>
 295 struct key_video_context : public IDeckLinkVideoOutputCallback, boost::noncopyable
 296 {
 297         const configuration                                     config_;
 298         com_ptr<IDeckLink>                                      decklink_                                       = get_device(config_.key_device_index());
 299         com_iface_ptr<IDeckLinkOutput>          output_                                         = iface_cast<IDeckLinkOutput>(decklink_);
 300         com_iface_ptr<IDeckLinkKeyer>           keyer_                                          = iface_cast<IDeckLinkKeyer>(decklink_, true);
 301         com_iface_ptr<IDeckLinkAttributes>      attributes_                                     = iface_cast<IDeckLinkAttributes>(decklink_);
 302         com_iface_ptr<Configuration>            configuration_                          = iface_cast<Configuration>(decklink_);
 303         tbb::atomic<int64_t>                            current_presentation_delay_;
 304         tbb::atomic<int64_t>                            scheduled_frames_completed_;
 305
 306         key_video_context(const configuration& config, const std::wstring& print)
 307                 : config_(config)
 308         {
 309                 current_presentation_delay_ = 0;
 310                 scheduled_frames_completed_ = 0;
 311
 312                 set_latency(configuration_, config.latency, print);
 313                 set_keyer(attributes_, keyer_, config.keyer, print);
 314
 315                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 316                         CASPAR_THROW_EXCEPTION(caspar_exception()
 317                                         << msg_info(print + L" Failed to set key playback completion callback.")
 318                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 319         }
 320
 321         template<typename Print>
 322         void enable_video(BMDDisplayMode display_mode, const Print& print)
 323         {
 324                 if (FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 325                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable key video output."));
 326
 327                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 328                         CASPAR_THROW_EXCEPTION(caspar_exception()
 329                                         << msg_info(print() + L" Failed to set key playback completion callback.")
 330                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 331         }
 332
 333         virtual ~key_video_context()
 334         {
 335                 if (output_)
 336                 {
 337                         output_->StopScheduledPlayback(0, nullptr, 0);
 338                         output_->DisableVideoOutput();
 339                 }
 340         }
 341
 342         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 343         virtual ULONG STDMETHODCALLTYPE AddRef()                                                        {return 1;}
 344         virtual ULONG STDMETHODCALLTYPE Release()                                                       {return 1;}
 345
 346         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 347         {
 348                 return S_OK;
 349         }
 350
 351         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(
 352                         IDeckLinkVideoFrame* completed_frame,
 353                         BMDOutputFrameCompletionResult result)
 354         {
 355                 auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 356                 current_presentation_delay_ = dframe->get_age_millis();
 357                 ++scheduled_frames_completed_;
 358
 359                 // Let the fill callback keep the pace, so no scheduling here.
 360
 361                 return S_OK;
 362         }
 363 };
 364
 365 template <typename Configuration>
 366 struct decklink_consumer : public IDeckLinkVideoOutputCallback, boost::noncopyable
 367 {
 368         const int                                                                                       channel_index_;
 369         const configuration                                                                     config_;
 370
 371         com_ptr<IDeckLink>                                                                      decklink_                               = get_device(config_.device_index);
 372         com_iface_ptr<IDeckLinkOutput>                                          output_                                 = iface_cast<IDeckLinkOutput>(decklink_);
 373         com_iface_ptr<Configuration>                                            configuration_                  = iface_cast<Configuration>(decklink_);
 374         com_iface_ptr<IDeckLinkKeyer>                                           keyer_                                  = iface_cast<IDeckLinkKeyer>(decklink_, true);
 375         com_iface_ptr<IDeckLinkAttributes>                                      attributes_                             = iface_cast<IDeckLinkAttributes>(decklink_);
 376
 377         tbb::spin_mutex                                                                         exception_mutex_;
 378         std::exception_ptr                                                                      exception_;
 379
 380         tbb::atomic<bool>                                                                       is_running_;
 381
 382         const std::wstring                                                                      model_name_                             = get_model_name(decklink_);
 383         bool                                                                                            will_attempt_dma_;
 384         const core::video_format_desc                                           format_desc_;
 385         const core::audio_channel_layout                                        in_channel_layout_;
 386         const core::audio_channel_layout                                        out_channel_layout_             = config_.get_adjusted_layout(in_channel_layout_);
 387         core::audio_channel_remapper                                            channel_remapper_               { in_channel_layout_, out_channel_layout_ };
 388         const int                                                                                       buffer_size_                    = config_.buffer_depth(); // Minimum buffer-size 3.
 389
 390         long long                                                                                       video_scheduled_                = 0;
 391         long long                                                                                       audio_scheduled_                = 0;
 392
 393         int                                                                                                     preroll_count_                  = 0;
 394
 395         boost::circular_buffer<std::vector<int32_t>>            audio_container_                { buffer_size_ + 1 };
 396
 397         tbb::concurrent_bounded_queue<core::const_frame>        frame_buffer_;
 398
 399         spl::shared_ptr<diagnostics::graph>                                     graph_;
 400         caspar::timer                                                                           tick_timer_;
 401         boost::mutex                                                                            send_completion_mutex_;
 402         std::packaged_task<bool ()>                                                     send_completion_;
 403         reference_signal_detector                                                       reference_signal_detector_      { output_ };
 404         tbb::atomic<int64_t>                                                            current_presentation_delay_;
 405         tbb::atomic<int64_t>                                                            scheduled_frames_completed_;
 406         std::unique_ptr<key_video_context<Configuration>>       key_context_;
 407
 408 public:
 409         decklink_consumer(
 410                         const configuration& config,
 411                         const core::video_format_desc& format_desc,
 412                         const core::audio_channel_layout& in_channel_layout,
 413                         int channel_index)
 414                 : channel_index_(channel_index)
 415                 , config_(config)
 416                 , format_desc_(format_desc)
 417                 , in_channel_layout_(in_channel_layout)
 418         {
 419                 is_running_ = true;
 420                 current_presentation_delay_ = 0;
 421                 scheduled_frames_completed_ = 0;
 422
 423                 frame_buffer_.set_capacity(1);
 424
 425                 if (config.keyer == configuration::keyer_t::external_separate_device_keyer)
 426                         key_context_.reset(new key_video_context<Configuration>(config, print()));
 427
 428                 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
 429                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
 430                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
 431                 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
 432                 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
 433                 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
 434
 435                 if (key_context_)
 436                 {
 437                         graph_->set_color("key-offset", diagnostics::color(1.0f, 0.0f, 0.0f));
 438                 }
 439
 440                 graph_->set_text(print());
 441                 diagnostics::register_graph(graph_);
 442
 443                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault, will_attempt_dma_));
 444
 445                 if(config.embedded_audio)
 446                         enable_audio();
 447
 448                 set_latency(configuration_, config.latency, print());
 449                 set_keyer(attributes_, keyer_, config.keyer, print());
 450
 451                 if(config.embedded_audio)
 452                         output_->BeginAudioPreroll();
 453
 454                 for (int n = 0; n < buffer_size_; ++n)
 455                 {
 456                         if (config.embedded_audio)
 457                                 schedule_next_audio(core::mutable_audio_buffer(format_desc_.audio_cadence[n % format_desc_.audio_cadence.size()] * out_channel_layout_.num_channels, 0));
 458
 459                         schedule_next_video(core::const_frame::empty());
 460                 }
 461
 462                 if (config.embedded_audio)
 463                 {
 464                         // Preroll one extra frame worth of audio
 465                         schedule_next_audio(core::mutable_audio_buffer(format_desc_.audio_cadence[buffer_size_ % format_desc_.audio_cadence.size()] * out_channel_layout_.num_channels, 0));
 466                         output_->EndAudioPreroll();
 467                 }
 468
 469                 start_playback();
 470         }
 471
 472         ~decklink_consumer()
 473         {
 474                 is_running_ = false;
 475                 frame_buffer_.try_push(core::const_frame::empty());
 476
 477                 if(output_ != nullptr)
 478                 {
 479                         output_->StopScheduledPlayback(0, nullptr, 0);
 480                         if(config_.embedded_audio)
 481                                 output_->DisableAudioOutput();
 482                         output_->DisableVideoOutput();
 483                 }
 484         }
 485
 486         void enable_audio()
 487         {
 488                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, out_channel_layout_.num_channels, bmdAudioOutputStreamTimestamped)))
 489                                 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable audio output."));
 490
 491                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
 492         }
 493
 494         void enable_video(BMDDisplayMode display_mode)
 495         {
 496                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 497                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable fill video output."));
 498
 499                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 500                         CASPAR_THROW_EXCEPTION(caspar_exception()
 501                                                                         << msg_info(print() + L" Failed to set fill playback completion callback.")
 502                                                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 503
 504                 if (key_context_)
 505                         key_context_->enable_video(display_mode, [this]() { return print(); });
 506         }
 507
 508         void start_playback()
 509         {
 510                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 511                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule fill playback."));
 512
 513                 if (key_context_ && FAILED(key_context_->output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 514                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule key playback."));
 515         }
 516
 517         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 518         virtual ULONG STDMETHODCALLTYPE AddRef()                                        {return 1;}
 519         virtual ULONG STDMETHODCALLTYPE Release()                               {return 1;}
 520
 521         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 522         {
 523                 is_running_ = false;
 524                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
 525                 return S_OK;
 526         }
 527
 528         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result)
 529         {
 530                 if(!is_running_)
 531                         return E_FAIL;
 532
 533                 try
 534                 {
 535                         auto tick_time = tick_timer_.elapsed()*format_desc_.fps * 0.5;
 536                         graph_->set_value("tick-time", tick_time);
 537                         tick_timer_.restart();
 538
 539                         reference_signal_detector_.detect_change([this]() { return print(); });
 540
 541                         auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 542                         current_presentation_delay_ = dframe->get_age_millis();
 543                         ++scheduled_frames_completed_;
 544
 545                         if (key_context_)
 546                                 graph_->set_value(
 547                                                 "key-offset",
 548                                                 static_cast<double>(
 549                                                                 scheduled_frames_completed_
 550                                                                 - key_context_->scheduled_frames_completed_)
 551                                                 * 0.1 + 0.5);
 552
 553                         if(result == bmdOutputFrameDisplayedLate)
 554                         {
 555                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "late-frame");
 556                                 video_scheduled_ += format_desc_.duration;
 557                                 audio_scheduled_ += dframe->audio_data().size() / in_channel_layout_.num_channels;
 558                         }
 559                         else if(result == bmdOutputFrameDropped)
 560                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "dropped-frame");
 561                         else if(result == bmdOutputFrameFlushed)
 562                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "flushed-frame");
 563
 564                         UINT32 buffered;
 565                         output_->GetBufferedVideoFrameCount(&buffered);
 566                         graph_->set_value("buffered-video", static_cast<double>(buffered) / (config_.buffer_depth()));
 567
 568                         if (config_.embedded_audio)
 569                         {
 570                                 output_->GetBufferedAudioSampleFrameCount(&buffered);
 571                                 graph_->set_value("buffered-audio", static_cast<double>(buffered) / (format_desc_.audio_cadence[0] * config_.buffer_depth()));
 572                         }
 573
 574                         auto frame = core::const_frame::empty();
 575
 576                         frame_buffer_.pop(frame);
 577
 578                         {
 579                                 boost::lock_guard<boost::mutex> lock(send_completion_mutex_);
 580
 581                                 if (send_completion_.valid())
 582                                 {
 583                                         send_completion_();
 584                                         send_completion_ = std::packaged_task<bool()>();
 585                                 }
 586                         }
 587
 588                         if (!is_running_)
 589                                 return E_FAIL;
 590
 591                         if (config_.embedded_audio)
 592                                 schedule_next_audio(channel_remapper_.mix_and_rearrange(frame.audio_data()));
 593
 594                         schedule_next_video(frame);
 595                 }
 596                 catch(...)
 597                 {
 598                         lock(exception_mutex_, [&]
 599                         {
 600                                 exception_ = std::current_exception();
 601                         });
 602                         return E_FAIL;
 603                 }
 604
 605                 return S_OK;
 606         }
 607
 608         template<typename T>
 609         void schedule_next_audio(const T& audio_data)
 610         {
 611                 auto sample_frame_count = static_cast<int>(audio_data.size()/out_channel_layout_.num_channels);
 612
 613                 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
 614
 615                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
 616                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
 617
 618                 audio_scheduled_ += sample_frame_count;
 619         }
 620
 621         void schedule_next_video(core::const_frame frame)
 622         {
 623                 if (key_context_)
 624                 {
 625                         auto key_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, true, will_attempt_dma_));
 626                         if (FAILED(key_context_->output_->ScheduleVideoFrame(get_raw(key_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 627                                 CASPAR_LOG(error) << print() << L" Failed to schedule key video.";
 628                 }
 629
 630                 auto fill_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, config_.key_only, will_attempt_dma_));
 631                 if (FAILED(output_->ScheduleVideoFrame(get_raw(fill_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 632                         CASPAR_LOG(error) << print() << L" Failed to schedule fill video.";
 633
 634                 video_scheduled_ += format_desc_.duration;
 635         }
 636
 637         std::future<bool> send(core::const_frame frame)
 638         {
 639                 auto exception = lock(exception_mutex_, [&]
 640                 {
 641                         return exception_;
 642                 });
 643
 644                 if(exception != nullptr)
 645                         std::rethrow_exception(exception);
 646
 647                 if(!is_running_)
 648                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Is not running."));
 649
 650                 boost::lock_guard<boost::mutex> lock(send_completion_mutex_);
 651
 652                 if (frame_buffer_.try_push(frame))
 653                 {
 654                         send_completion_ = std::packaged_task<bool()>();
 655                         return make_ready_future(true);
 656                 }
 657
 658                 send_completion_ = std::packaged_task<bool ()>([frame, this] () mutable -> bool
 659                 {
 660                         frame_buffer_.push(frame);
 661
 662                         return true;
 663                 });
 664
 665                 return send_completion_.get_future();
 666         }
 667
 668         std::wstring print() const
 669         {
 670                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 671                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 672                                 boost::lexical_cast<std::wstring>(config_.device_index) +
 673                                 L"&&" +
 674                                 boost::lexical_cast<std::wstring>(config_.key_device_index()) +
 675                                 L"|" +
 676                                 format_desc_.name + L"]";
 677                 else
 678                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 679                                 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
 680         }
 681 };
 682
 683 template <typename Configuration>
 684 struct decklink_consumer_proxy : public core::frame_consumer
 685 {
 686         core::monitor::subject                                                          monitor_subject_;
 687         const configuration                                                                     config_;
 688         std::unique_ptr<decklink_consumer<Configuration>>       consumer_;
 689         core::video_format_desc                                                         format_desc_;
 690         executor                                                                                        executor_;
 691 public:
 692
 693         decklink_consumer_proxy(const configuration& config)
 694                 : config_(config)
 695                 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
 696         {
 697                 auto ctx = core::diagnostics::call_context::for_thread();
 698                 executor_.begin_invoke([=]
 699                 {
 700                         core::diagnostics::call_context::for_thread() = ctx;
 701                         com_initialize();
 702                 });
 703         }
 704
 705         ~decklink_consumer_proxy()
 706         {
 707                 executor_.invoke([=]
 708                 {
 709                         consumer_.reset();
 710                         com_uninitialize();
 711                 });
 712         }
 713
 714         // frame_consumer
 715
 716         void initialize(const core::video_format_desc& format_desc, const core::audio_channel_layout& channel_layout, int channel_index) override
 717         {
 718                 format_desc_ = format_desc;
 719                 executor_.invoke([=]
 720                 {
 721                         consumer_.reset();
 722                         consumer_.reset(new decklink_consumer<Configuration>(config_, format_desc, channel_layout, channel_index));
 723                 });
 724         }
 725
 726         std::future<bool> send(core::const_frame frame) override
 727         {
 728                 return consumer_->send(frame);
 729         }
 730
 731         std::wstring print() const override
 732         {
 733                 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
 734         }
 735
 736         std::wstring name() const override
 737         {
 738                 return L"decklink";
 739         }
 740
 741         boost::property_tree::wptree info() const override
 742         {
 743                 boost::property_tree::wptree info;
 744                 info.add(L"type", L"decklink");
 745                 info.add(L"key-only", config_.key_only);
 746                 info.add(L"device", config_.device_index);
 747
 748                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 749                 {
 750                         info.add(L"key-device", config_.key_device_index());
 751                 }
 752
 753                 info.add(L"low-latency", config_.latency == configuration::latency_t::low_latency);
 754                 info.add(L"embedded-audio", config_.embedded_audio);
 755                 info.add(L"presentation-frame-age", presentation_frame_age_millis());
 756                 //info.add(L"internal-key", config_.internal_key);
 757                 return info;
 758         }
 759
 760         int buffer_depth() const override
 761         {
 762                 return config_.buffer_depth() + 2;
 763         }
 764
 765         int index() const override
 766         {
 767                 return 300 + config_.device_index;
 768         }
 769
 770         int64_t presentation_frame_age_millis() const override
 771         {
 772                 return consumer_ ? static_cast<int64_t>(consumer_->current_presentation_delay_) : 0;
 773         }
 774
 775         core::monitor::subject& monitor_output()
 776         {
 777                 return monitor_subject_;
 778         }
 779 };
 780
 781 const software_version<3>& get_driver_version()
 782 {
 783         static software_version<3> version(u8(get_version()));
 784
 785         return version;
 786 }
 787
 788 const software_version<3> get_new_configuration_api_version()
 789 {
 790         static software_version<3> NEW_CONFIGURATION_API("10.2");
 791
 792         return NEW_CONFIGURATION_API;
 793 }
 794
 795 void describe_consumer(core::help_sink& sink, const core::help_repository& repo)
 796 {
 797         sink.short_description(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 798         sink.syntax(L"DECKLINK "
 799                                 L"{[device_index:int]|1} "
 800                                 L"{[keyer:INTERNAL_KEY,EXTERNAL_KEY,EXTERNAL_SEPARATE_DEVICE_KEY]} "
 801                                 L"{[low_latency:LOW_LATENCY]} "
 802                                 L"{[embedded_audio:EMBEDDED_AUDIO]} "
 803                                 L"{[key_only:KEY_ONLY]} "
 804                                 L"{CHANNEL_LAYOUT [channel_layout:string]}");
 805         sink.para()->text(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 806         sink.definitions()
 807                 ->item(L"device_index", L"The Blackmagic video card to use (See Blackmagic control panel for card order). Default is 1.")
 808                 ->item(L"keyer",
 809                                 L"If given tries to enable either internal or external keying. Not all Blackmagic cards supports this. "
 810                                 L"There is also a third experimental option (EXTERNAL_SEPARATE_DEVICE_KEY) which allocates device_index + 1 for synhronized key output.")
 811                 ->item(L"low_latency", L"Tries to enable low latency if given.")
 812                 ->item(L"embedded_audio", L"Embeds the audio into the SDI signal if given.")
 813                 ->item(L"key_only",
 814                                 L" will extract only the alpha channel from the "
 815                                 L"channel. This is useful when you have two SDI video cards, and neither has native support "
 816                                 L"for separate fill/key output")
 817                 ->item(L"channel_layout", L"If specified, overrides the audio channel layout used by the channel.");
 818         sink.para()->text(L"Examples:");
 819         sink.example(L">> ADD 1 DECKLINK", L"for using the default device_index of 1.");
 820         sink.example(L">> ADD 1 DECKLINK 2", L"uses device_index 2.");
 821         sink.example(L">> ADD 1 DECKLINK 1 EXTERNAL_KEY EMBEDDED_AUDIO");
 822         sink.example(
 823                         L">> ADD 1 DECKLINK 1 EMBEDDED_AUDIO\n"
 824                         L">> ADD 1 DECKLINK 2 KEY_ONLY", L"uses device with index 1 as fill output with audio and device with index 2 as key output.");
 825         sink.example(
 826                         L">> ADD 1 DECKLINK 1 EXTERNAL_SEPARATE_DEVICE_KEY EMBEDDED_AUDIO",
 827                         L"Uses device 2 for key output. May give better sync between key and fill than the previous method.");
 828 }
 829
 830 spl::shared_ptr<core::frame_consumer> create_consumer(
 831                 const std::vector<std::wstring>& params, core::interaction_sink*, std::vector<spl::shared_ptr<core::video_channel>> channels)
 832 {
 833         if (params.size() < 1 || !boost::iequals(params.at(0), L"DECKLINK"))
 834                 return core::frame_consumer::empty();
 835
 836         configuration config;
 837
 838         if (params.size() > 1)
 839                 config.device_index = boost::lexical_cast<int>(params.at(1));
 840
 841         if (contains_param(L"INTERNAL_KEY", params))
 842                 config.keyer = configuration::keyer_t::internal_keyer;
 843         else if (contains_param(L"EXTERNAL_KEY", params))
 844                 config.keyer = configuration::keyer_t::external_keyer;
 845         else if (contains_param(L"EXTERNAL_SEPARATE_DEVICE_KEY", params))
 846                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 847         else
 848                 config.keyer = configuration::keyer_t::default_keyer;
 849
 850         if (contains_param(L"LOW_LATENCY", params))
 851                 config.latency = configuration::latency_t::low_latency;
 852
 853         config.embedded_audio   = contains_param(L"EMBEDDED_AUDIO", params);
 854         config.key_only                 = contains_param(L"KEY_ONLY", params);
 855
 856         auto channel_layout = get_param(L"CHANNEL_LAYOUT", params);
 857
 858         if (!channel_layout.empty())
 859         {
 860                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(channel_layout);
 861
 862                 if (!found_layout)
 863                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + channel_layout + L" not found."));
 864
 865                 config.out_channel_layout = *found_layout;
 866         }
 867
 868         bool old_configuration_api = get_driver_version() < get_new_configuration_api_version();
 869
 870         if (old_configuration_api)
 871                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration_v10_2>>(config);
 872         else
 873                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration>>(config);
 874 }
 875
 876 spl::shared_ptr<core::frame_consumer> create_preconfigured_consumer(
 877                 const boost::property_tree::wptree& ptree, core::interaction_sink*, std::vector<spl::shared_ptr<core::video_channel>> channels)
 878 {
 879         configuration config;
 880
 881         auto keyer = ptree.get(L"keyer", L"default");
 882         if(keyer == L"external")
 883                 config.keyer = configuration::keyer_t::external_keyer;
 884         else if(keyer == L"internal")
 885                 config.keyer = configuration::keyer_t::internal_keyer;
 886         else if (keyer == L"external_separate_device")
 887                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 888
 889         auto latency = ptree.get(L"latency", L"default");
 890         if(latency == L"low")
 891                 config.latency = configuration::latency_t::low_latency;
 892         else if(latency == L"normal")
 893                 config.latency = configuration::latency_t::normal_latency;
 894
 895         auto channel_layout = ptree.get_optional<std::wstring>(L"channel-layout");
 896
 897         if (channel_layout)
 898         {
 899                 CASPAR_SCOPED_CONTEXT_MSG("/channel-layout")
 900
 901                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(*channel_layout);
 902
 903                 if (!found_layout)
 904                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + *channel_layout + L" not found."));
 905
 906                 config.out_channel_layout = *found_layout;
 907         }
 908
 909         config.key_only                         = ptree.get(L"key-only",                config.key_only);
 910         config.device_index                     = ptree.get(L"device",                  config.device_index);
 911         config.key_device_idx           = ptree.get(L"key-device",              config.key_device_idx);
 912         config.embedded_audio           = ptree.get(L"embedded-audio",  config.embedded_audio);
 913         config.base_buffer_depth        = ptree.get(L"buffer-depth",    config.base_buffer_depth);
 914
 915         bool old_configuration_api = get_driver_version() < get_new_configuration_api_version();
 916
 917         if (old_configuration_api)
 918                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration_v10_2>>(config);
 919         else
 920                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration>>(config);
 921 }
 922
 923 }}
 924
 925 /*
 926 ##############################################################################
 927 Pre-rolling
 928
 929 Mail: 2011-05-09
 930
 931 Yoshan
 932 BMD Developer Support
 933 developer@blackmagic-design.com
 934
 935 -----------------------------------------------------------------------------
 936
 937 Thanks for your inquiry. The minimum number of frames that you can preroll
 938 for scheduled playback is three frames for video and four frames for audio.
 939 As you mentioned if you preroll less frames then playback will not start or
 940 playback will be very sporadic. From our experience with Media Express, we
 941 recommended that at least seven frames are prerolled for smooth playback.
 942
 943 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
 944 There can be around 3 frames worth of latency on scheduled output.
 945 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
 946 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync()
 947 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
 948 guarantee that the provided frame will be output as soon the previous
 949 frame output has been completed.
 950 ################################################################################
 951 */
 952
 953 /*
 954 ##############################################################################
 955 Async DMA Transfer without redundant copying
 956
 957 Mail: 2011-05-10
 958
 959 Yoshan
 960 BMD Developer Support
 961 developer@blackmagic-design.com
 962
 963 -----------------------------------------------------------------------------
 964
 965 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
 966 and providing a pointer to your video buffer when GetBytes() is called.
 967 This may help to keep copying to a minimum. Please ensure that the pixel
 968 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
 969 have to colourspace convert which may result in additional copying.
 970 ################################################################################
 971 */