git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Robert Nagy, ronag89@gmail.com
  20 */
  21
  22 #include "../StdAfx.h"
  23
  24 #include "decklink_consumer.h"
  25
  26 #include "../util/util.h"
  27 #include "../decklink.h"
  28
  29 #include "../decklink_api.h"
  30
  31 #include <core/frame/frame.h>
  32 #include <core/frame/audio_channel_layout.h>
  33 #include <core/mixer/audio/audio_mixer.h>
  34 #include <core/consumer/frame_consumer.h>
  35 #include <core/diagnostics/call_context.h>
  36 #include <core/help/help_sink.h>
  37 #include <core/help/help_repository.h>
  38
  39 #include <common/executor.h>
  40 #include <common/lock.h>
  41 #include <common/diagnostics/graph.h>
  42 #include <common/except.h>
  43 #include <common/memshfl.h>
  44 #include <common/memcpy.h>
  45 #include <common/no_init_proxy.h>
  46 #include <common/array.h>
  47 #include <common/future.h>
  48 #include <common/cache_aligned_vector.h>
  49 #include <common/timer.h>
  50 #include <common/param.h>
  51 #include <common/software_version.h>
  52
  53 #include <tbb/concurrent_queue.h>
  54
  55 #include <common/assert.h>
  56 #include <boost/lexical_cast.hpp>
  57 #include <boost/circular_buffer.hpp>
  58 #include <boost/property_tree/ptree.hpp>
  59 #include <boost/thread/mutex.hpp>
  60
  61 #include <future>
  62
  63 namespace caspar { namespace decklink {
  64
  65 struct configuration
  66 {
  67         enum class keyer_t
  68         {
  69                 internal_keyer,
  70                 external_keyer,
  71                 external_separate_device_keyer,
  72                 default_keyer                                   = external_keyer
  73         };
  74
  75         enum class latency_t
  76         {
  77                 low_latency,
  78                 normal_latency,
  79                 default_latency = normal_latency
  80         };
  81
  82         int                                                     device_index            = 1;
  83         int                                                     key_device_idx          = 0;
  84         bool                                            embedded_audio          = true;
  85         keyer_t                                         keyer                           = keyer_t::default_keyer;
  86         latency_t                                       latency                         = latency_t::default_latency;
  87         bool                                            key_only                        = false;
  88         int                                                     base_buffer_depth       = 3;
  89         core::audio_channel_layout      out_channel_layout      = core::audio_channel_layout::invalid();
  90
  91         int buffer_depth() const
  92         {
  93                 return base_buffer_depth + (latency == latency_t::low_latency ? 0 : 1);
  94         }
  95
  96         int key_device_index() const
  97         {
  98                 return key_device_idx == 0 ? device_index + 1 : key_device_idx;
  99         }
 100
 101         core::audio_channel_layout get_adjusted_layout(const core::audio_channel_layout& in_layout) const
 102         {
 103                 auto adjusted = out_channel_layout == core::audio_channel_layout::invalid() ? in_layout : out_channel_layout;
 104
 105                 if (adjusted.num_channels == 1) // Duplicate mono-signal into both left and right.
 106                 {
 107                         adjusted.num_channels = 2;
 108                         adjusted.channel_order.push_back(adjusted.channel_order.at(0)); // Usually FC -> FC FC
 109                 }
 110                 else if (adjusted.num_channels == 2)
 111                 {
 112                         adjusted.num_channels = 2;
 113                 }
 114                 else if (adjusted.num_channels <= 8)
 115                 {
 116                         adjusted.num_channels = 8;
 117                 }
 118                 else // Over 8 always pad to 16 or drop >16
 119                 {
 120                         adjusted.num_channels = 16;
 121                 }
 122
 123                 return adjusted;
 124         }
 125 };
 126
 127 template <typename Configuration>
 128 void set_latency(
 129                 const com_iface_ptr<Configuration>& config,
 130                 configuration::latency_t latency,
 131                 const std::wstring& print)
 132 {
 133         if (latency == configuration::latency_t::low_latency)
 134         {
 135                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
 136                 CASPAR_LOG(info) << print << L" Enabled low-latency mode.";
 137         }
 138         else if (latency == configuration::latency_t::normal_latency)
 139         {
 140                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
 141                 CASPAR_LOG(info) << print << L" Disabled low-latency mode.";
 142         }
 143 }
 144
 145 void set_keyer(
 146                 const com_iface_ptr<IDeckLinkAttributes>& attributes,
 147                 const com_iface_ptr<IDeckLinkKeyer>& decklink_keyer,
 148                 configuration::keyer_t keyer,
 149                 const std::wstring& print)
 150 {
 151         if (keyer == configuration::keyer_t::internal_keyer)
 152         {
 153                 BOOL value = true;
 154                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
 155                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 156                 else if (FAILED(decklink_keyer->Enable(FALSE)))
 157                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 158                 else if (FAILED(decklink_keyer->SetLevel(255)))
 159                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 160                 else
 161                         CASPAR_LOG(info) << print << L" Enabled internal keyer.";
 162         }
 163         else if (keyer == configuration::keyer_t::external_keyer)
 164         {
 165                 BOOL value = true;
 166                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
 167                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 168                 else if (FAILED(decklink_keyer->Enable(TRUE)))
 169                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 170                 else if (FAILED(decklink_keyer->SetLevel(255)))
 171                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 172                 else
 173                         CASPAR_LOG(info) << print << L" Enabled external keyer.";
 174         }
 175 }
 176
 177 class decklink_frame : public IDeckLinkVideoFrame
 178 {
 179         tbb::atomic<int>                                                                ref_count_;
 180         core::const_frame                                                               frame_;
 181         const core::video_format_desc                                   format_desc_;
 182
 183         const bool                                                                              key_only_;
 184         bool                                                                                    needs_to_copy_;
 185         cache_aligned_vector<no_init_proxy<uint8_t>>    data_;
 186 public:
 187         decklink_frame(core::const_frame frame, const core::video_format_desc& format_desc, bool key_only, bool will_attempt_dma)
 188                 : frame_(frame)
 189                 , format_desc_(format_desc)
 190                 , key_only_(key_only)
 191         {
 192                 ref_count_ = 0;
 193
 194                 bool dma_transfer_from_gl_buffer_impossible;
 195
 196 #if !defined(_MSC_VER)
 197                 // On Linux Decklink cannot DMA transfer from memory returned by glMapBuffer (at least on nvidia)
 198                 dma_transfer_from_gl_buffer_impossible = true;
 199 #else
 200                 // On Windows it is possible.
 201                 dma_transfer_from_gl_buffer_impossible = false;
 202 #endif
 203
 204                 needs_to_copy_ = will_attempt_dma && dma_transfer_from_gl_buffer_impossible;
 205         }
 206
 207         // IUnknown
 208
 209         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)
 210         {
 211                 return E_NOINTERFACE;
 212         }
 213
 214         virtual ULONG STDMETHODCALLTYPE AddRef()
 215         {
 216                 return ++ref_count_;
 217         }
 218
 219         virtual ULONG STDMETHODCALLTYPE Release()
 220         {
 221                 if(--ref_count_ == 0)
 222                         delete this;
 223                 return ref_count_;
 224         }
 225
 226         // IDecklinkVideoFrame
 227
 228         virtual long STDMETHODCALLTYPE GetWidth()                   {return static_cast<long>(format_desc_.width);}
 229         virtual long STDMETHODCALLTYPE GetHeight()                  {return static_cast<long>(format_desc_.height);}
 230         virtual long STDMETHODCALLTYPE GetRowBytes()                {return static_cast<long>(format_desc_.width*4);}
 231         virtual BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat()   {return bmdFormat8BitBGRA;}
 232         virtual BMDFrameFlags STDMETHODCALLTYPE GetFlags()                      {return bmdFrameFlagDefault;}
 233
 234         virtual HRESULT STDMETHODCALLTYPE GetBytes(void** buffer)
 235         {
 236                 try
 237                 {
 238                         if(static_cast<int>(frame_.image_data().size()) != format_desc_.size)
 239                         {
 240                                 data_.resize(format_desc_.size);
 241                                 *buffer = data_.data();
 242                         }
 243                         else if(key_only_)
 244                         {
 245                                 if(data_.empty())
 246                                 {
 247                                         data_.resize(frame_.image_data().size());
 248                                         aligned_memshfl(data_.data(), frame_.image_data().begin(), frame_.image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
 249                                 }
 250                                 *buffer = data_.data();
 251                         }
 252                         else
 253                         {
 254                                 *buffer = const_cast<uint8_t*>(frame_.image_data().begin());
 255
 256                                 if (needs_to_copy_)
 257                                 {
 258                                         data_.resize(frame_.image_data().size());
 259                                         fast_memcpy(data_.data(), *buffer, frame_.image_data().size());
 260                                         *buffer = data_.data();
 261                                 }
 262                         }
 263                 }
 264                 catch(...)
 265                 {
 266                         CASPAR_LOG_CURRENT_EXCEPTION();
 267                         return E_FAIL;
 268                 }
 269
 270                 return S_OK;
 271         }
 272
 273         virtual HRESULT STDMETHODCALLTYPE GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode) {return S_FALSE;}
 274         virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)              {return S_FALSE;}
 275
 276         // decklink_frame
 277
 278         const core::audio_buffer& audio_data()
 279         {
 280                 return frame_.audio_data();
 281         }
 282
 283         int64_t get_age_millis() const
 284         {
 285                 return frame_.get_age_millis();
 286         }
 287 };
 288
 289 template <typename Configuration>
 290 struct key_video_context : public IDeckLinkVideoOutputCallback, boost::noncopyable
 291 {
 292         const configuration                                     config_;
 293         com_ptr<IDeckLink>                                      decklink_                                       = get_device(config_.key_device_index());
 294         com_iface_ptr<IDeckLinkOutput>          output_                                         = iface_cast<IDeckLinkOutput>(decklink_);
 295         com_iface_ptr<IDeckLinkKeyer>           keyer_                                          = iface_cast<IDeckLinkKeyer>(decklink_, true);
 296         com_iface_ptr<IDeckLinkAttributes>      attributes_                                     = iface_cast<IDeckLinkAttributes>(decklink_);
 297         com_iface_ptr<Configuration>            configuration_                          = iface_cast<Configuration>(decklink_);
 298         tbb::atomic<int64_t>                            current_presentation_delay_;
 299         tbb::atomic<int64_t>                            scheduled_frames_completed_;
 300
 301         key_video_context(const configuration& config, const std::wstring& print)
 302                 : config_(config)
 303         {
 304                 current_presentation_delay_ = 0;
 305                 scheduled_frames_completed_ = 0;
 306
 307                 set_latency(configuration_, config.latency, print);
 308                 set_keyer(attributes_, keyer_, config.keyer, print);
 309
 310                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 311                         CASPAR_THROW_EXCEPTION(caspar_exception()
 312                                         << msg_info(print + L" Failed to set key playback completion callback.")
 313                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 314         }
 315
 316         template<typename Print>
 317         void enable_video(BMDDisplayMode display_mode, const Print& print)
 318         {
 319                 if (FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 320                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable key video output."));
 321
 322                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 323                         CASPAR_THROW_EXCEPTION(caspar_exception()
 324                                         << msg_info(print() + L" Failed to set key playback completion callback.")
 325                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 326         }
 327
 328         virtual ~key_video_context()
 329         {
 330                 if (output_)
 331                 {
 332                         output_->StopScheduledPlayback(0, nullptr, 0);
 333                         output_->DisableVideoOutput();
 334                 }
 335         }
 336
 337         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 338         virtual ULONG STDMETHODCALLTYPE AddRef()                                                        {return 1;}
 339         virtual ULONG STDMETHODCALLTYPE Release()                                                       {return 1;}
 340
 341         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 342         {
 343                 return S_OK;
 344         }
 345
 346         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(
 347                         IDeckLinkVideoFrame* completed_frame,
 348                         BMDOutputFrameCompletionResult result)
 349         {
 350                 auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 351                 current_presentation_delay_ = dframe->get_age_millis();
 352                 ++scheduled_frames_completed_;
 353
 354                 // Let the fill callback keep the pace, so no scheduling here.
 355
 356                 return S_OK;
 357         }
 358 };
 359
 360 template <typename Configuration>
 361 struct decklink_consumer : public IDeckLinkVideoOutputCallback, boost::noncopyable
 362 {
 363         const int                                                                                       channel_index_;
 364         const configuration                                                                     config_;
 365
 366         com_ptr<IDeckLink>                                                                      decklink_                               = get_device(config_.device_index);
 367         com_iface_ptr<IDeckLinkOutput>                                          output_                                 = iface_cast<IDeckLinkOutput>(decklink_);
 368         com_iface_ptr<Configuration>                                            configuration_                  = iface_cast<Configuration>(decklink_);
 369         com_iface_ptr<IDeckLinkKeyer>                                           keyer_                                  = iface_cast<IDeckLinkKeyer>(decklink_, true);
 370         com_iface_ptr<IDeckLinkAttributes>                                      attributes_                             = iface_cast<IDeckLinkAttributes>(decklink_);
 371
 372         tbb::spin_mutex                                                                         exception_mutex_;
 373         std::exception_ptr                                                                      exception_;
 374
 375         tbb::atomic<bool>                                                                       is_running_;
 376
 377         const std::wstring                                                                      model_name_                             = get_model_name(decklink_);
 378         bool                                                                                            will_attempt_dma_;
 379         const core::video_format_desc                                           format_desc_;
 380         const core::audio_channel_layout                                        in_channel_layout_;
 381         const core::audio_channel_layout                                        out_channel_layout_             = config_.get_adjusted_layout(in_channel_layout_);
 382         core::audio_channel_remapper                                            channel_remapper_               { in_channel_layout_, out_channel_layout_ };
 383         const int                                                                                       buffer_size_                    = config_.buffer_depth(); // Minimum buffer-size 3.
 384
 385         long long                                                                                       video_scheduled_                = 0;
 386         long long                                                                                       audio_scheduled_                = 0;
 387
 388         int                                                                                                     preroll_count_                  = 0;
 389
 390         boost::circular_buffer<std::vector<int32_t>>            audio_container_                { buffer_size_ + 1 };
 391
 392         tbb::concurrent_bounded_queue<core::const_frame>        frame_buffer_;
 393
 394         spl::shared_ptr<diagnostics::graph>                                     graph_;
 395         caspar::timer                                                                           tick_timer_;
 396         boost::mutex                                                                            send_completion_mutex_;
 397         std::packaged_task<bool ()>                                                     send_completion_;
 398         reference_signal_detector                                                       reference_signal_detector_      { output_ };
 399         tbb::atomic<int64_t>                                                            current_presentation_delay_;
 400         tbb::atomic<int64_t>                                                            scheduled_frames_completed_;
 401         std::unique_ptr<key_video_context<Configuration>>       key_context_;
 402
 403 public:
 404         decklink_consumer(
 405                         const configuration& config,
 406                         const core::video_format_desc& format_desc,
 407                         const core::audio_channel_layout& in_channel_layout,
 408                         int channel_index)
 409                 : channel_index_(channel_index)
 410                 , config_(config)
 411                 , format_desc_(format_desc)
 412                 , in_channel_layout_(in_channel_layout)
 413         {
 414                 is_running_ = true;
 415                 current_presentation_delay_ = 0;
 416                 scheduled_frames_completed_ = 0;
 417
 418                 frame_buffer_.set_capacity(1);
 419
 420                 if (config.keyer == configuration::keyer_t::external_separate_device_keyer)
 421                         key_context_.reset(new key_video_context<Configuration>(config, print()));
 422
 423                 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
 424                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
 425                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
 426                 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
 427                 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
 428                 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
 429
 430                 if (key_context_)
 431                 {
 432                         graph_->set_color("key-offset", diagnostics::color(1.0f, 0.0f, 0.0f));
 433                 }
 434
 435                 graph_->set_text(print());
 436                 diagnostics::register_graph(graph_);
 437
 438                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault, will_attempt_dma_));
 439
 440                 if(config.embedded_audio)
 441                         enable_audio();
 442
 443                 set_latency(configuration_, config.latency, print());
 444                 set_keyer(attributes_, keyer_, config.keyer, print());
 445
 446                 if(config.embedded_audio)
 447                         output_->BeginAudioPreroll();
 448
 449                 for (int n = 0; n < buffer_size_; ++n)
 450                 {
 451                         if (config.embedded_audio)
 452                                 schedule_next_audio(core::mutable_audio_buffer(format_desc_.audio_cadence[n % format_desc_.audio_cadence.size()] * out_channel_layout_.num_channels, 0));
 453
 454                         schedule_next_video(core::const_frame::empty());
 455                 }
 456
 457                 if (config.embedded_audio)
 458                 {
 459                         // Preroll one extra frame worth of audio
 460                         schedule_next_audio(core::mutable_audio_buffer(format_desc_.audio_cadence[buffer_size_ % format_desc_.audio_cadence.size()] * out_channel_layout_.num_channels, 0));
 461                         output_->EndAudioPreroll();
 462                 }
 463
 464                 start_playback();
 465         }
 466
 467         ~decklink_consumer()
 468         {
 469                 is_running_ = false;
 470                 frame_buffer_.try_push(core::const_frame::empty());
 471
 472                 if(output_ != nullptr)
 473                 {
 474                         output_->StopScheduledPlayback(0, nullptr, 0);
 475                         if(config_.embedded_audio)
 476                                 output_->DisableAudioOutput();
 477                         output_->DisableVideoOutput();
 478                 }
 479         }
 480
 481         void enable_audio()
 482         {
 483                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, out_channel_layout_.num_channels, bmdAudioOutputStreamTimestamped)))
 484                                 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable audio output."));
 485
 486                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
 487         }
 488
 489         void enable_video(BMDDisplayMode display_mode)
 490         {
 491                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 492                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable fill video output."));
 493
 494                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 495                         CASPAR_THROW_EXCEPTION(caspar_exception()
 496                                                                         << msg_info(print() + L" Failed to set fill playback completion callback.")
 497                                                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 498
 499                 if (key_context_)
 500                         key_context_->enable_video(display_mode, [this]() { return print(); });
 501         }
 502
 503         void start_playback()
 504         {
 505                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 506                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule fill playback."));
 507
 508                 if (key_context_ && FAILED(key_context_->output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 509                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule key playback."));
 510         }
 511
 512         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 513         virtual ULONG STDMETHODCALLTYPE AddRef()                                        {return 1;}
 514         virtual ULONG STDMETHODCALLTYPE Release()                               {return 1;}
 515
 516         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 517         {
 518                 is_running_ = false;
 519                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
 520                 return S_OK;
 521         }
 522
 523         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result)
 524         {
 525                 if(!is_running_)
 526                         return E_FAIL;
 527
 528                 try
 529                 {
 530                         auto tick_time = tick_timer_.elapsed()*format_desc_.fps * 0.5;
 531                         graph_->set_value("tick-time", tick_time);
 532                         tick_timer_.restart();
 533
 534                         reference_signal_detector_.detect_change([this]() { return print(); });
 535
 536                         auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 537                         current_presentation_delay_ = dframe->get_age_millis();
 538                         ++scheduled_frames_completed_;
 539
 540                         if (key_context_)
 541                                 graph_->set_value(
 542                                                 "key-offset",
 543                                                 static_cast<double>(
 544                                                                 scheduled_frames_completed_
 545                                                                 - key_context_->scheduled_frames_completed_)
 546                                                 * 0.1 + 0.5);
 547
 548                         if(result == bmdOutputFrameDisplayedLate)
 549                         {
 550                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "late-frame");
 551                                 video_scheduled_ += format_desc_.duration;
 552                                 audio_scheduled_ += dframe->audio_data().size() / in_channel_layout_.num_channels;
 553                         }
 554                         else if(result == bmdOutputFrameDropped)
 555                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "dropped-frame");
 556                         else if(result == bmdOutputFrameFlushed)
 557                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "flushed-frame");
 558
 559                         UINT32 buffered;
 560                         output_->GetBufferedVideoFrameCount(&buffered);
 561                         graph_->set_value("buffered-video", static_cast<double>(buffered) / (config_.buffer_depth()));
 562
 563                         if (config_.embedded_audio)
 564                         {
 565                                 output_->GetBufferedAudioSampleFrameCount(&buffered);
 566                                 graph_->set_value("buffered-audio", static_cast<double>(buffered) / (format_desc_.audio_cadence[0] * config_.buffer_depth()));
 567                         }
 568
 569                         auto frame = core::const_frame::empty();
 570
 571                         frame_buffer_.pop(frame);
 572
 573                         {
 574                                 boost::lock_guard<boost::mutex> lock(send_completion_mutex_);
 575
 576                                 if (send_completion_.valid())
 577                                 {
 578                                         send_completion_();
 579                                         send_completion_ = std::packaged_task<bool()>();
 580                                 }
 581                         }
 582
 583                         if (config_.embedded_audio)
 584                                 schedule_next_audio(channel_remapper_.mix_and_rearrange(frame.audio_data()));
 585
 586                         schedule_next_video(frame);
 587                 }
 588                 catch(...)
 589                 {
 590                         lock(exception_mutex_, [&]
 591                         {
 592                                 exception_ = std::current_exception();
 593                         });
 594                         return E_FAIL;
 595                 }
 596
 597                 return S_OK;
 598         }
 599
 600         template<typename T>
 601         void schedule_next_audio(const T& audio_data)
 602         {
 603                 auto sample_frame_count = static_cast<int>(audio_data.size()/out_channel_layout_.num_channels);
 604
 605                 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
 606
 607                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
 608                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
 609
 610                 audio_scheduled_ += sample_frame_count;
 611         }
 612
 613         void schedule_next_video(core::const_frame frame)
 614         {
 615                 if (key_context_)
 616                 {
 617                         auto key_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, true, will_attempt_dma_));
 618                         if (FAILED(key_context_->output_->ScheduleVideoFrame(get_raw(key_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 619                                 CASPAR_LOG(error) << print() << L" Failed to schedule key video.";
 620                 }
 621
 622                 auto fill_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, config_.key_only, will_attempt_dma_));
 623                 if (FAILED(output_->ScheduleVideoFrame(get_raw(fill_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 624                         CASPAR_LOG(error) << print() << L" Failed to schedule fill video.";
 625
 626                 video_scheduled_ += format_desc_.duration;
 627         }
 628
 629         std::future<bool> send(core::const_frame frame)
 630         {
 631                 auto exception = lock(exception_mutex_, [&]
 632                 {
 633                         return exception_;
 634                 });
 635
 636                 if(exception != nullptr)
 637                         std::rethrow_exception(exception);
 638
 639                 if(!is_running_)
 640                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Is not running."));
 641
 642                 if (frame_buffer_.try_push(frame))
 643                         return make_ready_future(true);
 644
 645                 boost::lock_guard<boost::mutex> lock(send_completion_mutex_);
 646
 647                 send_completion_ = std::packaged_task<bool ()>([frame, this] () mutable -> bool
 648                 {
 649                         frame_buffer_.push(frame);
 650
 651                         return true;
 652                 });
 653
 654                 return send_completion_.get_future();
 655         }
 656
 657         std::wstring print() const
 658         {
 659                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 660                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 661                                 boost::lexical_cast<std::wstring>(config_.device_index) +
 662                                 L"&&" +
 663                                 boost::lexical_cast<std::wstring>(config_.key_device_index()) +
 664                                 L"|" +
 665                                 format_desc_.name + L"]";
 666                 else
 667                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 668                                 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
 669         }
 670 };
 671
 672 template <typename Configuration>
 673 struct decklink_consumer_proxy : public core::frame_consumer
 674 {
 675         core::monitor::subject                                                          monitor_subject_;
 676         const configuration                                                                     config_;
 677         std::unique_ptr<decklink_consumer<Configuration>>       consumer_;
 678         core::video_format_desc                                                         format_desc_;
 679         executor                                                                                        executor_;
 680 public:
 681
 682         decklink_consumer_proxy(const configuration& config)
 683                 : config_(config)
 684                 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
 685         {
 686                 auto ctx = core::diagnostics::call_context::for_thread();
 687                 executor_.begin_invoke([=]
 688                 {
 689                         core::diagnostics::call_context::for_thread() = ctx;
 690                         com_initialize();
 691                 });
 692         }
 693
 694         ~decklink_consumer_proxy()
 695         {
 696                 executor_.invoke([=]
 697                 {
 698                         consumer_.reset();
 699                         com_uninitialize();
 700                 });
 701         }
 702
 703         // frame_consumer
 704
 705         void initialize(const core::video_format_desc& format_desc, const core::audio_channel_layout& channel_layout, int channel_index) override
 706         {
 707                 format_desc_ = format_desc;
 708                 executor_.invoke([=]
 709                 {
 710                         consumer_.reset();
 711                         consumer_.reset(new decklink_consumer<Configuration>(config_, format_desc, channel_layout, channel_index));
 712                 });
 713         }
 714
 715         std::future<bool> send(core::const_frame frame) override
 716         {
 717                 return consumer_->send(frame);
 718         }
 719
 720         std::wstring print() const override
 721         {
 722                 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
 723         }
 724
 725         std::wstring name() const override
 726         {
 727                 return L"decklink";
 728         }
 729
 730         boost::property_tree::wptree info() const override
 731         {
 732                 boost::property_tree::wptree info;
 733                 info.add(L"type", L"decklink");
 734                 info.add(L"key-only", config_.key_only);
 735                 info.add(L"device", config_.device_index);
 736
 737                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 738                 {
 739                         info.add(L"key-device", config_.key_device_index());
 740                 }
 741
 742                 info.add(L"low-latency", config_.latency == configuration::latency_t::low_latency);
 743                 info.add(L"embedded-audio", config_.embedded_audio);
 744                 info.add(L"presentation-frame-age", presentation_frame_age_millis());
 745                 //info.add(L"internal-key", config_.internal_key);
 746                 return info;
 747         }
 748
 749         int buffer_depth() const override
 750         {
 751                 return config_.buffer_depth() + 2;
 752         }
 753
 754         int index() const override
 755         {
 756                 return 300 + config_.device_index;
 757         }
 758
 759         int64_t presentation_frame_age_millis() const override
 760         {
 761                 return consumer_ ? static_cast<int64_t>(consumer_->current_presentation_delay_) : 0;
 762         }
 763
 764         core::monitor::subject& monitor_output()
 765         {
 766                 return monitor_subject_;
 767         }
 768 };
 769
 770 const software_version<3>& get_driver_version()
 771 {
 772         static software_version<3> version(u8(get_version()));
 773
 774         return version;
 775 }
 776
 777 const software_version<3> get_new_configuration_api_version()
 778 {
 779         static software_version<3> NEW_CONFIGURATION_API("10.2");
 780
 781         return NEW_CONFIGURATION_API;
 782 }
 783
 784 void describe_consumer(core::help_sink& sink, const core::help_repository& repo)
 785 {
 786         sink.short_description(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 787         sink.syntax(L"DECKLINK "
 788                                 L"{[device_index:int]|1} "
 789                                 L"{[keyer:INTERNAL_KEY,EXTERNAL_KEY,EXTERNAL_SEPARATE_DEVICE_KEY]} "
 790                                 L"{[low_latency:LOW_LATENCY]} "
 791                                 L"{[embedded_audio:EMBEDDED_AUDIO]} "
 792                                 L"{[key_only:KEY_ONLY]} "
 793                                 L"{CHANNEL_LAYOUT [channel_layout:string]}");
 794         sink.para()->text(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 795         sink.definitions()
 796                 ->item(L"device_index", L"The Blackmagic video card to use (See Blackmagic control panel for card order). Default is 1.")
 797                 ->item(L"keyer",
 798                                 L"If given tries to enable either internal or external keying. Not all Blackmagic cards supports this. "
 799                                 L"There is also a third experimental option (EXTERNAL_SEPARATE_DEVICE_KEY) which allocates device_index + 1 for synhronized key output.")
 800                 ->item(L"low_latency", L"Tries to enable low latency if given.")
 801                 ->item(L"embedded_audio", L"Embeds the audio into the SDI signal if given.")
 802                 ->item(L"key_only",
 803                                 L" will extract only the alpha channel from the "
 804                                 L"channel. This is useful when you have two SDI video cards, and neither has native support "
 805                                 L"for separate fill/key output")
 806                 ->item(L"channel_layout", L"If specified, overrides the audio channel layout used by the channel.");
 807         sink.para()->text(L"Examples:");
 808         sink.example(L">> ADD 1 DECKLINK", L"for using the default device_index of 1.");
 809         sink.example(L">> ADD 1 DECKLINK 2", L"uses device_index 2.");
 810         sink.example(L">> ADD 1 DECKLINK 1 EXTERNAL_KEY EMBEDDED_AUDIO");
 811         sink.example(
 812                         L">> ADD 1 DECKLINK 1 EMBEDDED_AUDIO\n"
 813                         L">> ADD 1 DECKLINK 2 KEY_ONLY", L"uses device with index 1 as fill output with audio and device with index 2 as key output.");
 814         sink.example(
 815                         L">> ADD 1 DECKLINK 1 EXTERNAL_SEPARATE_DEVICE_KEY EMBEDDED_AUDIO",
 816                         L"Uses device 2 for key output. May give better sync between key and fill than the previous method.");
 817 }
 818
 819 spl::shared_ptr<core::frame_consumer> create_consumer(
 820                 const std::vector<std::wstring>& params, core::interaction_sink*)
 821 {
 822         if (params.size() < 1 || !boost::iequals(params.at(0), L"DECKLINK"))
 823                 return core::frame_consumer::empty();
 824
 825         configuration config;
 826
 827         if (params.size() > 1)
 828                 config.device_index = boost::lexical_cast<int>(params.at(1));
 829
 830         if (contains_param(L"INTERNAL_KEY", params))
 831                 config.keyer = configuration::keyer_t::internal_keyer;
 832         else if (contains_param(L"EXTERNAL_KEY", params))
 833                 config.keyer = configuration::keyer_t::external_keyer;
 834         else if (contains_param(L"EXTERNAL_SEPARATE_DEVICE_KEY", params))
 835                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 836         else
 837                 config.keyer = configuration::keyer_t::default_keyer;
 838
 839         if (contains_param(L"LOW_LATENCY", params))
 840                 config.latency = configuration::latency_t::low_latency;
 841
 842         config.embedded_audio   = contains_param(L"EMBEDDED_AUDIO", params);
 843         config.key_only                 = contains_param(L"KEY_ONLY", params);
 844
 845         auto channel_layout = get_param(L"CHANNEL_LAYOUT", params);
 846
 847         if (!channel_layout.empty())
 848         {
 849                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(channel_layout);
 850
 851                 if (!found_layout)
 852                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + channel_layout + L" not found."));
 853
 854                 config.out_channel_layout = *found_layout;
 855         }
 856
 857         bool old_configuration_api = get_driver_version() < get_new_configuration_api_version();
 858
 859         if (old_configuration_api)
 860                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration_v10_2>>(config);
 861         else
 862                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration>>(config);
 863 }
 864
 865 spl::shared_ptr<core::frame_consumer> create_preconfigured_consumer(
 866                 const boost::property_tree::wptree& ptree, core::interaction_sink*)
 867 {
 868         configuration config;
 869
 870         auto keyer = ptree.get(L"keyer", L"default");
 871         if(keyer == L"external")
 872                 config.keyer = configuration::keyer_t::external_keyer;
 873         else if(keyer == L"internal")
 874                 config.keyer = configuration::keyer_t::internal_keyer;
 875         else if (keyer == L"external_separate_device")
 876                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 877
 878         auto latency = ptree.get(L"latency", L"default");
 879         if(latency == L"low")
 880                 config.latency = configuration::latency_t::low_latency;
 881         else if(latency == L"normal")
 882                 config.latency = configuration::latency_t::normal_latency;
 883
 884         auto channel_layout = ptree.get_optional<std::wstring>(L"channel-layout");
 885
 886         if (channel_layout)
 887         {
 888                 CASPAR_SCOPED_CONTEXT_MSG("/channel-layout")
 889
 890                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(*channel_layout);
 891
 892                 if (!found_layout)
 893                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + *channel_layout + L" not found."));
 894
 895                 config.out_channel_layout = *found_layout;
 896         }
 897
 898         config.key_only                         = ptree.get(L"key-only",                config.key_only);
 899         config.device_index                     = ptree.get(L"device",                  config.device_index);
 900         config.key_device_idx           = ptree.get(L"key-device",              config.key_device_idx);
 901         config.embedded_audio           = ptree.get(L"embedded-audio",  config.embedded_audio);
 902         config.base_buffer_depth        = ptree.get(L"buffer-depth",    config.base_buffer_depth);
 903
 904         bool old_configuration_api = get_driver_version() < get_new_configuration_api_version();
 905
 906         if (old_configuration_api)
 907                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration_v10_2>>(config);
 908         else
 909                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration>>(config);
 910 }
 911
 912 }}
 913
 914 /*
 915 ##############################################################################
 916 Pre-rolling
 917
 918 Mail: 2011-05-09
 919
 920 Yoshan
 921 BMD Developer Support
 922 developer@blackmagic-design.com
 923
 924 -----------------------------------------------------------------------------
 925
 926 Thanks for your inquiry. The minimum number of frames that you can preroll
 927 for scheduled playback is three frames for video and four frames for audio.
 928 As you mentioned if you preroll less frames then playback will not start or
 929 playback will be very sporadic. From our experience with Media Express, we
 930 recommended that at least seven frames are prerolled for smooth playback.
 931
 932 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
 933 There can be around 3 frames worth of latency on scheduled output.
 934 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
 935 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync()
 936 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
 937 guarantee that the provided frame will be output as soon the previous
 938 frame output has been completed.
 939 ################################################################################
 940 */
 941
 942 /*
 943 ##############################################################################
 944 Async DMA Transfer without redundant copying
 945
 946 Mail: 2011-05-10
 947
 948 Yoshan
 949 BMD Developer Support
 950 developer@blackmagic-design.com
 951
 952 -----------------------------------------------------------------------------
 953
 954 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
 955 and providing a pointer to your video buffer when GetBytes() is called.
 956 This may help to keep copying to a minimum. Please ensure that the pixel
 957 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
 958 have to colourspace convert which may result in additional copying.
 959 ################################################################################
 960 */