git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Robert Nagy, ronag89@gmail.com
  20 */
  21
  22 #include "../StdAfx.h"
  23
  24 #include "decklink_consumer.h"
  25
  26 #include "../util/util.h"
  27 #include "../decklink.h"
  28
  29 #include "../decklink_api.h"
  30
  31 #include <core/frame/frame.h>
  32 #include <core/frame/audio_channel_layout.h>
  33 #include <core/mixer/audio/audio_mixer.h>
  34 #include <core/consumer/frame_consumer.h>
  35 #include <core/diagnostics/call_context.h>
  36 #include <core/help/help_sink.h>
  37 #include <core/help/help_repository.h>
  38
  39 #include <common/executor.h>
  40 #include <common/lock.h>
  41 #include <common/diagnostics/graph.h>
  42 #include <common/except.h>
  43 #include <common/memshfl.h>
  44 #include <common/memcpy.h>
  45 #include <common/no_init_proxy.h>
  46 #include <common/array.h>
  47 #include <common/future.h>
  48 #include <common/cache_aligned_vector.h>
  49 #include <common/timer.h>
  50 #include <common/param.h>
  51 #include <common/software_version.h>
  52
  53 #include <tbb/concurrent_queue.h>
  54
  55 #include <common/assert.h>
  56 #include <boost/lexical_cast.hpp>
  57 #include <boost/circular_buffer.hpp>
  58 #include <boost/property_tree/ptree.hpp>
  59
  60 #include <future>
  61
  62 namespace caspar { namespace decklink {
  63
  64 struct configuration
  65 {
  66         enum class keyer_t
  67         {
  68                 internal_keyer,
  69                 external_keyer,
  70                 external_separate_device_keyer,
  71                 default_keyer                                   = external_keyer
  72         };
  73
  74         enum class latency_t
  75         {
  76                 low_latency,
  77                 normal_latency,
  78                 default_latency = normal_latency
  79         };
  80
  81         int                                                     device_index            = 1;
  82         int                                                     key_device_idx          = 0;
  83         bool                                            embedded_audio          = true;
  84         keyer_t                                         keyer                           = keyer_t::default_keyer;
  85         latency_t                                       latency                         = latency_t::default_latency;
  86         bool                                            key_only                        = false;
  87         int                                                     base_buffer_depth       = 3;
  88         core::audio_channel_layout      out_channel_layout      = core::audio_channel_layout::invalid();
  89
  90         int buffer_depth() const
  91         {
  92                 return base_buffer_depth + (latency == latency_t::low_latency ? 0 : 1);
  93         }
  94
  95         int key_device_index() const
  96         {
  97                 return key_device_idx == 0 ? device_index + 1 : key_device_idx;
  98         }
  99
 100         core::audio_channel_layout get_adjusted_layout(const core::audio_channel_layout& in_layout) const
 101         {
 102                 auto adjusted = out_channel_layout == core::audio_channel_layout::invalid() ? in_layout : out_channel_layout;
 103
 104                 if (adjusted.num_channels == 1) // Duplicate mono-signal into both left and right.
 105                 {
 106                         adjusted.num_channels = 2;
 107                         adjusted.channel_order.push_back(adjusted.channel_order.at(0)); // Usually FC -> FC FC
 108                 }
 109                 else if (adjusted.num_channels == 2)
 110                 {
 111                         adjusted.num_channels = 2;
 112                 }
 113                 else if (adjusted.num_channels <= 8)
 114                 {
 115                         adjusted.num_channels = 8;
 116                 }
 117                 else // Over 8 always pad to 16 or drop >16
 118                 {
 119                         adjusted.num_channels = 16;
 120                 }
 121
 122                 return adjusted;
 123         }
 124 };
 125
 126 template <typename Configuration>
 127 void set_latency(
 128                 const com_iface_ptr<Configuration>& config,
 129                 configuration::latency_t latency,
 130                 const std::wstring& print)
 131 {
 132         if (latency == configuration::latency_t::low_latency)
 133         {
 134                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
 135                 CASPAR_LOG(info) << print << L" Enabled low-latency mode.";
 136         }
 137         else if (latency == configuration::latency_t::normal_latency)
 138         {
 139                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
 140                 CASPAR_LOG(info) << print << L" Disabled low-latency mode.";
 141         }
 142 }
 143
 144 void set_keyer(
 145                 const com_iface_ptr<IDeckLinkAttributes>& attributes,
 146                 const com_iface_ptr<IDeckLinkKeyer>& decklink_keyer,
 147                 configuration::keyer_t keyer,
 148                 const std::wstring& print)
 149 {
 150         if (keyer == configuration::keyer_t::internal_keyer)
 151         {
 152                 BOOL value = true;
 153                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
 154                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 155                 else if (FAILED(decklink_keyer->Enable(FALSE)))
 156                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 157                 else if (FAILED(decklink_keyer->SetLevel(255)))
 158                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 159                 else
 160                         CASPAR_LOG(info) << print << L" Enabled internal keyer.";
 161         }
 162         else if (keyer == configuration::keyer_t::external_keyer)
 163         {
 164                 BOOL value = true;
 165                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
 166                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 167                 else if (FAILED(decklink_keyer->Enable(TRUE)))
 168                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 169                 else if (FAILED(decklink_keyer->SetLevel(255)))
 170                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 171                 else
 172                         CASPAR_LOG(info) << print << L" Enabled external keyer.";
 173         }
 174 }
 175
 176 class decklink_frame : public IDeckLinkVideoFrame
 177 {
 178         tbb::atomic<int>                                                                ref_count_;
 179         core::const_frame                                                               frame_;
 180         const core::video_format_desc                                   format_desc_;
 181
 182         const bool                                                                              key_only_;
 183         bool                                                                                    needs_to_copy_;
 184         cache_aligned_vector<no_init_proxy<uint8_t>>    data_;
 185 public:
 186         decklink_frame(core::const_frame frame, const core::video_format_desc& format_desc, bool key_only, bool will_attempt_dma)
 187                 : frame_(frame)
 188                 , format_desc_(format_desc)
 189                 , key_only_(key_only)
 190         {
 191                 ref_count_ = 0;
 192
 193                 bool dma_transfer_from_gl_buffer_impossible;
 194
 195 #if !defined(_MSC_VER)
 196                 // On Linux Decklink cannot DMA transfer from memory returned by glMapBuffer (at least on nvidia)
 197                 dma_transfer_from_gl_buffer_impossible = true;
 198 #else
 199                 // On Windows it is possible.
 200                 dma_transfer_from_gl_buffer_impossible = false;
 201 #endif
 202
 203                 needs_to_copy_ = will_attempt_dma && dma_transfer_from_gl_buffer_impossible;
 204         }
 205
 206         // IUnknown
 207
 208         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)
 209         {
 210                 return E_NOINTERFACE;
 211         }
 212
 213         virtual ULONG STDMETHODCALLTYPE AddRef()
 214         {
 215                 return ++ref_count_;
 216         }
 217
 218         virtual ULONG STDMETHODCALLTYPE Release()
 219         {
 220                 if(--ref_count_ == 0)
 221                         delete this;
 222                 return ref_count_;
 223         }
 224
 225         // IDecklinkVideoFrame
 226
 227         virtual long STDMETHODCALLTYPE GetWidth()                   {return static_cast<long>(format_desc_.width);}
 228         virtual long STDMETHODCALLTYPE GetHeight()                  {return static_cast<long>(format_desc_.height);}
 229         virtual long STDMETHODCALLTYPE GetRowBytes()                {return static_cast<long>(format_desc_.width*4);}
 230         virtual BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat()   {return bmdFormat8BitBGRA;}
 231         virtual BMDFrameFlags STDMETHODCALLTYPE GetFlags()                      {return bmdFrameFlagDefault;}
 232
 233         virtual HRESULT STDMETHODCALLTYPE GetBytes(void** buffer)
 234         {
 235                 try
 236                 {
 237                         if(static_cast<int>(frame_.image_data().size()) != format_desc_.size)
 238                         {
 239                                 data_.resize(format_desc_.size);
 240                                 *buffer = data_.data();
 241                         }
 242                         else if(key_only_)
 243                         {
 244                                 if(data_.empty())
 245                                 {
 246                                         data_.resize(frame_.image_data().size());
 247                                         aligned_memshfl(data_.data(), frame_.image_data().begin(), frame_.image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
 248                                 }
 249                                 *buffer = data_.data();
 250                         }
 251                         else
 252                         {
 253                                 *buffer = const_cast<uint8_t*>(frame_.image_data().begin());
 254
 255                                 if (needs_to_copy_)
 256                                 {
 257                                         data_.resize(frame_.image_data().size());
 258                                         fast_memcpy(data_.data(), *buffer, frame_.image_data().size());
 259                                         *buffer = data_.data();
 260                                 }
 261                         }
 262                 }
 263                 catch(...)
 264                 {
 265                         CASPAR_LOG_CURRENT_EXCEPTION();
 266                         return E_FAIL;
 267                 }
 268
 269                 return S_OK;
 270         }
 271
 272         virtual HRESULT STDMETHODCALLTYPE GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode) {return S_FALSE;}
 273         virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)              {return S_FALSE;}
 274
 275         // decklink_frame
 276
 277         const core::audio_buffer& audio_data()
 278         {
 279                 return frame_.audio_data();
 280         }
 281
 282         int64_t get_age_millis() const
 283         {
 284                 return frame_.get_age_millis();
 285         }
 286 };
 287
 288 template <typename Configuration>
 289 struct key_video_context : public IDeckLinkVideoOutputCallback, boost::noncopyable
 290 {
 291         const configuration                                     config_;
 292         com_ptr<IDeckLink>                                      decklink_                                       = get_device(config_.key_device_index());
 293         com_iface_ptr<IDeckLinkOutput>          output_                                         = iface_cast<IDeckLinkOutput>(decklink_);
 294         com_iface_ptr<IDeckLinkKeyer>           keyer_                                          = iface_cast<IDeckLinkKeyer>(decklink_, true);
 295         com_iface_ptr<IDeckLinkAttributes>      attributes_                                     = iface_cast<IDeckLinkAttributes>(decklink_);
 296         com_iface_ptr<Configuration>            configuration_                          = iface_cast<Configuration>(decklink_);
 297         tbb::atomic<int64_t>                            current_presentation_delay_;
 298         tbb::atomic<int64_t>                            scheduled_frames_completed_;
 299
 300         key_video_context(const configuration& config, const std::wstring& print)
 301                 : config_(config)
 302         {
 303                 current_presentation_delay_ = 0;
 304                 scheduled_frames_completed_ = 0;
 305
 306                 set_latency(configuration_, config.latency, print);
 307                 set_keyer(attributes_, keyer_, config.keyer, print);
 308
 309                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 310                         CASPAR_THROW_EXCEPTION(caspar_exception()
 311                                         << msg_info(print + L" Failed to set key playback completion callback.")
 312                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 313         }
 314
 315         template<typename Print>
 316         void enable_video(BMDDisplayMode display_mode, const Print& print)
 317         {
 318                 if (FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 319                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable key video output."));
 320
 321                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 322                         CASPAR_THROW_EXCEPTION(caspar_exception()
 323                                         << msg_info(print() + L" Failed to set key playback completion callback.")
 324                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 325         }
 326
 327         virtual ~key_video_context()
 328         {
 329                 if (output_)
 330                 {
 331                         output_->StopScheduledPlayback(0, nullptr, 0);
 332                         output_->DisableVideoOutput();
 333                 }
 334         }
 335
 336         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 337         virtual ULONG STDMETHODCALLTYPE AddRef()                                                        {return 1;}
 338         virtual ULONG STDMETHODCALLTYPE Release()                                                       {return 1;}
 339
 340         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 341         {
 342                 return S_OK;
 343         }
 344
 345         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(
 346                         IDeckLinkVideoFrame* completed_frame,
 347                         BMDOutputFrameCompletionResult result)
 348         {
 349                 auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 350                 current_presentation_delay_ = dframe->get_age_millis();
 351                 ++scheduled_frames_completed_;
 352
 353                 // Let the fill callback keep the pace, so no scheduling here.
 354
 355                 return S_OK;
 356         }
 357 };
 358
 359 template <typename Configuration>
 360 struct decklink_consumer : public IDeckLinkVideoOutputCallback, boost::noncopyable
 361 {
 362         const int                                                                                       channel_index_;
 363         const configuration                                                                     config_;
 364
 365         com_ptr<IDeckLink>                                                                      decklink_                               = get_device(config_.device_index);
 366         com_iface_ptr<IDeckLinkOutput>                                          output_                                 = iface_cast<IDeckLinkOutput>(decklink_);
 367         com_iface_ptr<Configuration>                                            configuration_                  = iface_cast<Configuration>(decklink_);
 368         com_iface_ptr<IDeckLinkKeyer>                                           keyer_                                  = iface_cast<IDeckLinkKeyer>(decklink_, true);
 369         com_iface_ptr<IDeckLinkAttributes>                                      attributes_                             = iface_cast<IDeckLinkAttributes>(decklink_);
 370
 371         tbb::spin_mutex                                                                         exception_mutex_;
 372         std::exception_ptr                                                                      exception_;
 373
 374         tbb::atomic<bool>                                                                       is_running_;
 375
 376         const std::wstring                                                                      model_name_                             = get_model_name(decklink_);
 377         bool                                                                                            will_attempt_dma_;
 378         const core::video_format_desc                                           format_desc_;
 379         const core::audio_channel_layout                                        in_channel_layout_;
 380         const core::audio_channel_layout                                        out_channel_layout_             = config_.get_adjusted_layout(in_channel_layout_);
 381         core::audio_channel_remapper                                            channel_remapper_               { in_channel_layout_, out_channel_layout_ };
 382         const int                                                                                       buffer_size_                    = config_.buffer_depth(); // Minimum buffer-size 3.
 383
 384         long long                                                                                       video_scheduled_                = 0;
 385         long long                                                                                       audio_scheduled_                = 0;
 386
 387         int                                                                                                     preroll_count_                  = 0;
 388
 389         boost::circular_buffer<std::vector<int32_t>>            audio_container_                { buffer_size_ + 1 };
 390
 391         tbb::concurrent_bounded_queue<core::const_frame>        frame_buffer_;
 392
 393         spl::shared_ptr<diagnostics::graph>                                     graph_;
 394         caspar::timer                                                                           tick_timer_;
 395         std::packaged_task<bool ()>                                                     send_completion_;
 396         reference_signal_detector                                                       reference_signal_detector_      { output_ };
 397         tbb::atomic<int64_t>                                                            current_presentation_delay_;
 398         tbb::atomic<int64_t>                                                            scheduled_frames_completed_;
 399         std::unique_ptr<key_video_context<Configuration>>       key_context_;
 400
 401 public:
 402         decklink_consumer(
 403                         const configuration& config,
 404                         const core::video_format_desc& format_desc,
 405                         const core::audio_channel_layout& in_channel_layout,
 406                         int channel_index)
 407                 : channel_index_(channel_index)
 408                 , config_(config)
 409                 , format_desc_(format_desc)
 410                 , in_channel_layout_(in_channel_layout)
 411         {
 412                 is_running_ = true;
 413                 current_presentation_delay_ = 0;
 414                 scheduled_frames_completed_ = 0;
 415
 416                 frame_buffer_.set_capacity(1);
 417
 418                 if (config.keyer == configuration::keyer_t::external_separate_device_keyer)
 419                         key_context_.reset(new key_video_context<Configuration>(config, print()));
 420
 421                 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
 422                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
 423                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
 424                 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
 425                 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
 426                 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
 427
 428                 if (key_context_)
 429                 {
 430                         graph_->set_color("key-offset", diagnostics::color(1.0f, 0.0f, 0.0f));
 431                 }
 432
 433                 graph_->set_text(print());
 434                 diagnostics::register_graph(graph_);
 435
 436                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault, will_attempt_dma_));
 437
 438                 if(config.embedded_audio)
 439                         enable_audio();
 440
 441                 set_latency(configuration_, config.latency, print());
 442                 set_keyer(attributes_, keyer_, config.keyer, print());
 443
 444                 if(config.embedded_audio)
 445                         output_->BeginAudioPreroll();
 446
 447                 for (int n = 0; n < buffer_size_; ++n)
 448                 {
 449                         if (config.embedded_audio)
 450                                 schedule_next_audio(core::mutable_audio_buffer(format_desc_.audio_cadence[n % format_desc_.audio_cadence.size()] * out_channel_layout_.num_channels, 0));
 451
 452                         schedule_next_video(core::const_frame::empty());
 453                 }
 454
 455                 if (config.embedded_audio)
 456                 {
 457                         // Preroll one extra frame worth of audio
 458                         schedule_next_audio(core::mutable_audio_buffer(format_desc_.audio_cadence[buffer_size_ % format_desc_.audio_cadence.size()] * out_channel_layout_.num_channels, 0));
 459                         output_->EndAudioPreroll();
 460                 }
 461
 462                 start_playback();
 463         }
 464
 465         ~decklink_consumer()
 466         {
 467                 is_running_ = false;
 468                 frame_buffer_.try_push(core::const_frame::empty());
 469
 470                 if(output_ != nullptr)
 471                 {
 472                         output_->StopScheduledPlayback(0, nullptr, 0);
 473                         if(config_.embedded_audio)
 474                                 output_->DisableAudioOutput();
 475                         output_->DisableVideoOutput();
 476                 }
 477         }
 478
 479         void enable_audio()
 480         {
 481                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, out_channel_layout_.num_channels, bmdAudioOutputStreamTimestamped)))
 482                                 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable audio output."));
 483
 484                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
 485         }
 486
 487         void enable_video(BMDDisplayMode display_mode)
 488         {
 489                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 490                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable fill video output."));
 491
 492                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 493                         CASPAR_THROW_EXCEPTION(caspar_exception()
 494                                                                         << msg_info(print() + L" Failed to set fill playback completion callback.")
 495                                                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 496
 497                 if (key_context_)
 498                         key_context_->enable_video(display_mode, [this]() { return print(); });
 499         }
 500
 501         void start_playback()
 502         {
 503                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 504                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule fill playback."));
 505
 506                 if (key_context_ && FAILED(key_context_->output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 507                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule key playback."));
 508         }
 509
 510         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 511         virtual ULONG STDMETHODCALLTYPE AddRef()                                        {return 1;}
 512         virtual ULONG STDMETHODCALLTYPE Release()                               {return 1;}
 513
 514         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 515         {
 516                 is_running_ = false;
 517                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
 518                 return S_OK;
 519         }
 520
 521         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result)
 522         {
 523                 if(!is_running_)
 524                         return E_FAIL;
 525
 526                 try
 527                 {
 528                         auto tick_time = tick_timer_.elapsed()*format_desc_.fps * 0.5;
 529                         graph_->set_value("tick-time", tick_time);
 530                         tick_timer_.restart();
 531
 532                         reference_signal_detector_.detect_change([this]() { return print(); });
 533
 534                         auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 535                         current_presentation_delay_ = dframe->get_age_millis();
 536                         ++scheduled_frames_completed_;
 537
 538                         if (key_context_)
 539                                 graph_->set_value(
 540                                                 "key-offset",
 541                                                 static_cast<double>(
 542                                                                 scheduled_frames_completed_
 543                                                                 - key_context_->scheduled_frames_completed_)
 544                                                 * 0.1 + 0.5);
 545
 546                         if(result == bmdOutputFrameDisplayedLate)
 547                         {
 548                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "late-frame");
 549                                 video_scheduled_ += format_desc_.duration;
 550                                 audio_scheduled_ += dframe->audio_data().size() / in_channel_layout_.num_channels;
 551                         }
 552                         else if(result == bmdOutputFrameDropped)
 553                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "dropped-frame");
 554                         else if(result == bmdOutputFrameFlushed)
 555                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "flushed-frame");
 556
 557                         UINT32 buffered;
 558                         output_->GetBufferedVideoFrameCount(&buffered);
 559                         graph_->set_value("buffered-video", static_cast<double>(buffered) / (config_.buffer_depth()));
 560
 561                         if (config_.embedded_audio)
 562                         {
 563                                 output_->GetBufferedAudioSampleFrameCount(&buffered);
 564                                 graph_->set_value("buffered-audio", static_cast<double>(buffered) / (format_desc_.audio_cadence[0] * config_.buffer_depth()));
 565                         }
 566
 567                         auto frame = core::const_frame::empty();
 568
 569                         frame_buffer_.pop(frame);
 570
 571                         if (send_completion_.valid())
 572                                 send_completion_();
 573
 574                         if (config_.embedded_audio)
 575                                 schedule_next_audio(channel_remapper_.mix_and_rearrange(frame.audio_data()));
 576
 577                         schedule_next_video(frame);
 578                 }
 579                 catch(...)
 580                 {
 581                         lock(exception_mutex_, [&]
 582                         {
 583                                 exception_ = std::current_exception();
 584                         });
 585                         return E_FAIL;
 586                 }
 587
 588                 return S_OK;
 589         }
 590
 591         template<typename T>
 592         void schedule_next_audio(const T& audio_data)
 593         {
 594                 auto sample_frame_count = static_cast<int>(audio_data.size()/out_channel_layout_.num_channels);
 595
 596                 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
 597
 598                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
 599                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
 600
 601                 audio_scheduled_ += sample_frame_count;
 602         }
 603
 604         void schedule_next_video(core::const_frame frame)
 605         {
 606                 if (key_context_)
 607                 {
 608                         auto key_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, true, will_attempt_dma_));
 609                         if (FAILED(key_context_->output_->ScheduleVideoFrame(get_raw(key_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 610                                 CASPAR_LOG(error) << print() << L" Failed to schedule key video.";
 611                 }
 612
 613                 auto fill_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, config_.key_only, will_attempt_dma_));
 614                 if (FAILED(output_->ScheduleVideoFrame(get_raw(fill_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 615                         CASPAR_LOG(error) << print() << L" Failed to schedule fill video.";
 616
 617                 video_scheduled_ += format_desc_.duration;
 618         }
 619
 620         std::future<bool> send(core::const_frame frame)
 621         {
 622                 auto exception = lock(exception_mutex_, [&]
 623                 {
 624                         return exception_;
 625                 });
 626
 627                 if(exception != nullptr)
 628                         std::rethrow_exception(exception);
 629
 630                 if(!is_running_)
 631                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Is not running."));
 632
 633                 if (frame_buffer_.try_push(frame))
 634                         return make_ready_future(true);
 635
 636                 send_completion_ = std::packaged_task<bool ()>([frame, this] () mutable -> bool
 637                 {
 638                         frame_buffer_.push(frame);
 639
 640                         return true;
 641                 });
 642
 643                 return send_completion_.get_future();
 644         }
 645
 646         std::wstring print() const
 647         {
 648                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 649                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 650                                 boost::lexical_cast<std::wstring>(config_.device_index) +
 651                                 L"&&" +
 652                                 boost::lexical_cast<std::wstring>(config_.key_device_index()) +
 653                                 L"|" +
 654                                 format_desc_.name + L"]";
 655                 else
 656                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 657                                 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
 658         }
 659 };
 660
 661 template <typename Configuration>
 662 struct decklink_consumer_proxy : public core::frame_consumer
 663 {
 664         core::monitor::subject                                                          monitor_subject_;
 665         const configuration                                                                     config_;
 666         std::unique_ptr<decklink_consumer<Configuration>>       consumer_;
 667         core::video_format_desc                                                         format_desc_;
 668         executor                                                                                        executor_;
 669 public:
 670
 671         decklink_consumer_proxy(const configuration& config)
 672                 : config_(config)
 673                 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
 674         {
 675                 auto ctx = core::diagnostics::call_context::for_thread();
 676                 executor_.begin_invoke([=]
 677                 {
 678                         core::diagnostics::call_context::for_thread() = ctx;
 679                         com_initialize();
 680                 });
 681         }
 682
 683         ~decklink_consumer_proxy()
 684         {
 685                 executor_.invoke([=]
 686                 {
 687                         consumer_.reset();
 688                         com_uninitialize();
 689                 });
 690         }
 691
 692         // frame_consumer
 693
 694         void initialize(const core::video_format_desc& format_desc, const core::audio_channel_layout& channel_layout, int channel_index) override
 695         {
 696                 format_desc_ = format_desc;
 697                 executor_.invoke([=]
 698                 {
 699                         consumer_.reset();
 700                         consumer_.reset(new decklink_consumer<Configuration>(config_, format_desc, channel_layout, channel_index));
 701                 });
 702         }
 703
 704         std::future<bool> send(core::const_frame frame) override
 705         {
 706                 return consumer_->send(frame);
 707         }
 708
 709         std::wstring print() const override
 710         {
 711                 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
 712         }
 713
 714         std::wstring name() const override
 715         {
 716                 return L"decklink";
 717         }
 718
 719         boost::property_tree::wptree info() const override
 720         {
 721                 boost::property_tree::wptree info;
 722                 info.add(L"type", L"decklink");
 723                 info.add(L"key-only", config_.key_only);
 724                 info.add(L"device", config_.device_index);
 725
 726                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 727                 {
 728                         info.add(L"key-device", config_.key_device_index());
 729                 }
 730
 731                 info.add(L"low-latency", config_.latency == configuration::latency_t::low_latency);
 732                 info.add(L"embedded-audio", config_.embedded_audio);
 733                 info.add(L"presentation-frame-age", presentation_frame_age_millis());
 734                 //info.add(L"internal-key", config_.internal_key);
 735                 return info;
 736         }
 737
 738         int buffer_depth() const override
 739         {
 740                 return config_.buffer_depth() + 2;
 741         }
 742
 743         int index() const override
 744         {
 745                 return 300 + config_.device_index;
 746         }
 747
 748         int64_t presentation_frame_age_millis() const override
 749         {
 750                 return consumer_ ? static_cast<int64_t>(consumer_->current_presentation_delay_) : 0;
 751         }
 752
 753         core::monitor::subject& monitor_output()
 754         {
 755                 return monitor_subject_;
 756         }
 757 };
 758
 759 const software_version<3>& get_driver_version()
 760 {
 761         static software_version<3> version(u8(get_version()));
 762
 763         return version;
 764 }
 765
 766 const software_version<3> get_new_configuration_api_version()
 767 {
 768         static software_version<3> NEW_CONFIGURATION_API("10.2");
 769
 770         return NEW_CONFIGURATION_API;
 771 }
 772
 773 void describe_consumer(core::help_sink& sink, const core::help_repository& repo)
 774 {
 775         sink.short_description(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 776         sink.syntax(L"DECKLINK "
 777                                 L"{[device_index:int]|1} "
 778                                 L"{[keyer:INTERNAL_KEY,EXTERNAL_KEY,EXTERNAL_SEPARATE_DEVICE_KEY]} "
 779                                 L"{[low_latency:LOW_LATENCY]} "
 780                                 L"{[embedded_audio:EMBEDDED_AUDIO]} "
 781                                 L"{[key_only:KEY_ONLY]} "
 782                                 L"{CHANNEL_LAYOUT [channel_layout:string]}");
 783         sink.para()->text(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 784         sink.definitions()
 785                 ->item(L"device_index", L"The Blackmagic video card to use (See Blackmagic control panel for card order). Default is 1.")
 786                 ->item(L"keyer",
 787                                 L"If given tries to enable either internal or external keying. Not all Blackmagic cards supports this. "
 788                                 L"There is also a third experimental option (EXTERNAL_SEPARATE_DEVICE_KEY) which allocates device_index + 1 for synhronized key output.")
 789                 ->item(L"low_latency", L"Tries to enable low latency if given.")
 790                 ->item(L"embedded_audio", L"Embeds the audio into the SDI signal if given.")
 791                 ->item(L"key_only",
 792                                 L" will extract only the alpha channel from the "
 793                                 L"channel. This is useful when you have two SDI video cards, and neither has native support "
 794                                 L"for separate fill/key output")
 795                 ->item(L"channel_layout", L"If specified, overrides the audio channel layout used by the channel.");
 796         sink.para()->text(L"Examples:");
 797         sink.example(L">> ADD 1 DECKLINK", L"for using the default device_index of 1.");
 798         sink.example(L">> ADD 1 DECKLINK 2", L"uses device_index 2.");
 799         sink.example(L">> ADD 1 DECKLINK 1 EXTERNAL_KEY EMBEDDED_AUDIO");
 800         sink.example(
 801                         L">> ADD 1 DECKLINK 1 EMBEDDED_AUDIO\n"
 802                         L">> ADD 1 DECKLINK 2 KEY_ONLY", L"uses device with index 1 as fill output with audio and device with index 2 as key output.");
 803         sink.example(
 804                         L">> ADD 1 DECKLINK 1 EXTERNAL_SEPARATE_DEVICE_KEY EMBEDDED_AUDIO",
 805                         L"Uses device 2 for key output. May give better sync between key and fill than the previous method.");
 806 }
 807
 808 spl::shared_ptr<core::frame_consumer> create_consumer(
 809                 const std::vector<std::wstring>& params, core::interaction_sink*)
 810 {
 811         if (params.size() < 1 || !boost::iequals(params.at(0), L"DECKLINK"))
 812                 return core::frame_consumer::empty();
 813
 814         configuration config;
 815
 816         if (params.size() > 1)
 817                 config.device_index = boost::lexical_cast<int>(params.at(1));
 818
 819         if (contains_param(L"INTERNAL_KEY", params))
 820                 config.keyer = configuration::keyer_t::internal_keyer;
 821         else if (contains_param(L"EXTERNAL_KEY", params))
 822                 config.keyer = configuration::keyer_t::external_keyer;
 823         else if (contains_param(L"EXTERNAL_SEPARATE_DEVICE_KEY", params))
 824                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 825         else
 826                 config.keyer = configuration::keyer_t::default_keyer;
 827
 828         if (contains_param(L"LOW_LATENCY", params))
 829                 config.latency = configuration::latency_t::low_latency;
 830
 831         config.embedded_audio   = contains_param(L"EMBEDDED_AUDIO", params);
 832         config.key_only                 = contains_param(L"KEY_ONLY", params);
 833
 834         auto channel_layout = get_param(L"CHANNEL_LAYOUT", params);
 835
 836         if (!channel_layout.empty())
 837         {
 838                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(channel_layout);
 839
 840                 if (!found_layout)
 841                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + channel_layout + L" not found."));
 842
 843                 config.out_channel_layout = *found_layout;
 844         }
 845
 846         bool old_configuration_api = get_driver_version() < get_new_configuration_api_version();
 847
 848         if (old_configuration_api)
 849                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration_v10_2>>(config);
 850         else
 851                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration>>(config);
 852 }
 853
 854 spl::shared_ptr<core::frame_consumer> create_preconfigured_consumer(
 855                 const boost::property_tree::wptree& ptree, core::interaction_sink*)
 856 {
 857         configuration config;
 858
 859         auto keyer = ptree.get(L"keyer", L"default");
 860         if(keyer == L"external")
 861                 config.keyer = configuration::keyer_t::external_keyer;
 862         else if(keyer == L"internal")
 863                 config.keyer = configuration::keyer_t::internal_keyer;
 864         else if (keyer == L"external_separate_device")
 865                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 866
 867         auto latency = ptree.get(L"latency", L"default");
 868         if(latency == L"low")
 869                 config.latency = configuration::latency_t::low_latency;
 870         else if(latency == L"normal")
 871                 config.latency = configuration::latency_t::normal_latency;
 872
 873         auto channel_layout = ptree.get_optional<std::wstring>(L"channel-layout");
 874
 875         if (channel_layout)
 876         {
 877                 CASPAR_SCOPED_CONTEXT_MSG("/channel-layout")
 878
 879                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(*channel_layout);
 880
 881                 if (!found_layout)
 882                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + *channel_layout + L" not found."));
 883
 884                 config.out_channel_layout = *found_layout;
 885         }
 886
 887         config.key_only                         = ptree.get(L"key-only",                config.key_only);
 888         config.device_index                     = ptree.get(L"device",                  config.device_index);
 889         config.key_device_idx           = ptree.get(L"key-device",              config.key_device_idx);
 890         config.embedded_audio           = ptree.get(L"embedded-audio",  config.embedded_audio);
 891         config.base_buffer_depth        = ptree.get(L"buffer-depth",    config.base_buffer_depth);
 892
 893         bool old_configuration_api = get_driver_version() < get_new_configuration_api_version();
 894
 895         if (old_configuration_api)
 896                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration_v10_2>>(config);
 897         else
 898                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration>>(config);
 899 }
 900
 901 }}
 902
 903 /*
 904 ##############################################################################
 905 Pre-rolling
 906
 907 Mail: 2011-05-09
 908
 909 Yoshan
 910 BMD Developer Support
 911 developer@blackmagic-design.com
 912
 913 -----------------------------------------------------------------------------
 914
 915 Thanks for your inquiry. The minimum number of frames that you can preroll
 916 for scheduled playback is three frames for video and four frames for audio.
 917 As you mentioned if you preroll less frames then playback will not start or
 918 playback will be very sporadic. From our experience with Media Express, we
 919 recommended that at least seven frames are prerolled for smooth playback.
 920
 921 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
 922 There can be around 3 frames worth of latency on scheduled output.
 923 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
 924 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync()
 925 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
 926 guarantee that the provided frame will be output as soon the previous
 927 frame output has been completed.
 928 ################################################################################
 929 */
 930
 931 /*
 932 ##############################################################################
 933 Async DMA Transfer without redundant copying
 934
 935 Mail: 2011-05-10
 936
 937 Yoshan
 938 BMD Developer Support
 939 developer@blackmagic-design.com
 940
 941 -----------------------------------------------------------------------------
 942
 943 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
 944 and providing a pointer to your video buffer when GetBytes() is called.
 945 This may help to keep copying to a minimum. Please ensure that the pixel
 946 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
 947 have to colourspace convert which may result in additional copying.
 948 ################################################################################
 949 */