git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Robert Nagy, ronag89@gmail.com
  20 */
  21
  22 #include "../StdAfx.h"
  23
  24 #include "decklink_consumer.h"
  25
  26 #include "../util/util.h"
  27 #include "../decklink.h"
  28
  29 #include "../decklink_api.h"
  30
  31 #include <core/frame/frame.h>
  32 #include <core/frame/audio_channel_layout.h>
  33 #include <core/mixer/audio/audio_mixer.h>
  34 #include <core/consumer/frame_consumer.h>
  35 #include <core/diagnostics/call_context.h>
  36 #include <core/help/help_sink.h>
  37 #include <core/help/help_repository.h>
  38
  39 #include <common/executor.h>
  40 #include <common/lock.h>
  41 #include <common/diagnostics/graph.h>
  42 #include <common/except.h>
  43 #include <common/memshfl.h>
  44 #include <common/memcpy.h>
  45 #include <common/no_init_proxy.h>
  46 #include <common/array.h>
  47 #include <common/future.h>
  48 #include <common/cache_aligned_vector.h>
  49 #include <common/timer.h>
  50 #include <common/param.h>
  51 #include <common/software_version.h>
  52
  53 #include <tbb/concurrent_queue.h>
  54
  55 #include <common/assert.h>
  56 #include <boost/lexical_cast.hpp>
  57 #include <boost/circular_buffer.hpp>
  58 #include <boost/property_tree/ptree.hpp>
  59
  60 #include <future>
  61
  62 namespace caspar { namespace decklink {
  63
  64 struct configuration
  65 {
  66         enum class keyer_t
  67         {
  68                 internal_keyer,
  69                 external_keyer,
  70                 external_separate_device_keyer,
  71                 default_keyer                                   = external_keyer
  72         };
  73
  74         enum class latency_t
  75         {
  76                 low_latency,
  77                 normal_latency,
  78                 default_latency = normal_latency
  79         };
  80
  81         int                                                     device_index            = 1;
  82         int                                                     key_device_idx          = 0;
  83         bool                                            embedded_audio          = true;
  84         keyer_t                                         keyer                           = keyer_t::default_keyer;
  85         latency_t                                       latency                         = latency_t::default_latency;
  86         bool                                            key_only                        = false;
  87         int                                                     base_buffer_depth       = 3;
  88         core::audio_channel_layout      out_channel_layout      = core::audio_channel_layout::invalid();
  89
  90         int buffer_depth() const
  91         {
  92                 return base_buffer_depth + (latency == latency_t::low_latency ? 0 : 1);
  93         }
  94
  95         int key_device_index() const
  96         {
  97                 return key_device_idx == 0 ? device_index + 1 : key_device_idx;
  98         }
  99
 100         core::audio_channel_layout get_adjusted_layout(const core::audio_channel_layout& in_layout) const
 101         {
 102                 auto adjusted = out_channel_layout == core::audio_channel_layout::invalid() ? in_layout : out_channel_layout;
 103
 104                 if (adjusted.num_channels == 1) // Duplicate mono-signal into both left and right.
 105                 {
 106                         adjusted.num_channels = 2;
 107                         adjusted.channel_order.push_back(adjusted.channel_order.at(0)); // Usually FC -> FC FC
 108                 }
 109                 else if (adjusted.num_channels == 2)
 110                 {
 111                         adjusted.num_channels = 2;
 112                 }
 113                 else if (adjusted.num_channels <= 8)
 114                 {
 115                         adjusted.num_channels = 8;
 116                 }
 117                 else // Over 8 always pad to 16 or drop >16
 118                 {
 119                         adjusted.num_channels = 16;
 120                 }
 121
 122                 return adjusted;
 123         }
 124 };
 125
 126 template <typename Configuration>
 127 void set_latency(
 128                 const com_iface_ptr<Configuration>& config,
 129                 configuration::latency_t latency,
 130                 const std::wstring& print)
 131 {
 132         if (latency == configuration::latency_t::low_latency)
 133         {
 134                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
 135                 CASPAR_LOG(info) << print << L" Enabled low-latency mode.";
 136         }
 137         else if (latency == configuration::latency_t::normal_latency)
 138         {
 139                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
 140                 CASPAR_LOG(info) << print << L" Disabled low-latency mode.";
 141         }
 142 }
 143
 144 void set_keyer(
 145                 const com_iface_ptr<IDeckLinkAttributes>& attributes,
 146                 const com_iface_ptr<IDeckLinkKeyer>& decklink_keyer,
 147                 configuration::keyer_t keyer,
 148                 const std::wstring& print)
 149 {
 150         if (keyer == configuration::keyer_t::internal_keyer)
 151         {
 152                 BOOL value = true;
 153                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
 154                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 155                 else if (FAILED(decklink_keyer->Enable(FALSE)))
 156                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 157                 else if (FAILED(decklink_keyer->SetLevel(255)))
 158                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 159                 else
 160                         CASPAR_LOG(info) << print << L" Enabled internal keyer.";
 161         }
 162         else if (keyer == configuration::keyer_t::external_keyer)
 163         {
 164                 BOOL value = true;
 165                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
 166                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 167                 else if (FAILED(decklink_keyer->Enable(TRUE)))
 168                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 169                 else if (FAILED(decklink_keyer->SetLevel(255)))
 170                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 171                 else
 172                         CASPAR_LOG(info) << print << L" Enabled external keyer.";
 173         }
 174 }
 175
 176 class decklink_frame : public IDeckLinkVideoFrame
 177 {
 178         tbb::atomic<int>                                                                ref_count_;
 179         core::const_frame                                                               frame_;
 180         const core::video_format_desc                                   format_desc_;
 181
 182         const bool                                                                              key_only_;
 183         bool                                                                                    needs_to_copy_;
 184         cache_aligned_vector<no_init_proxy<uint8_t>>    data_;
 185 public:
 186         decklink_frame(core::const_frame frame, const core::video_format_desc& format_desc, bool key_only, bool will_attempt_dma)
 187                 : frame_(frame)
 188                 , format_desc_(format_desc)
 189                 , key_only_(key_only)
 190         {
 191                 ref_count_ = 0;
 192
 193                 bool dma_transfer_from_gl_buffer_impossible;
 194
 195 #if !defined(_MSC_VER)
 196                 // On Linux Decklink cannot DMA transfer from memory returned by glMapBuffer (at least on nvidia)
 197                 dma_transfer_from_gl_buffer_impossible = true;
 198 #else
 199                 // On Windows it is possible.
 200                 dma_transfer_from_gl_buffer_impossible = false;
 201 #endif
 202
 203                 needs_to_copy_ = will_attempt_dma && dma_transfer_from_gl_buffer_impossible;
 204         }
 205
 206         // IUnknown
 207
 208         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)
 209         {
 210                 return E_NOINTERFACE;
 211         }
 212
 213         virtual ULONG STDMETHODCALLTYPE AddRef()
 214         {
 215                 return ++ref_count_;
 216         }
 217
 218         virtual ULONG STDMETHODCALLTYPE Release()
 219         {
 220                 if(--ref_count_ == 0)
 221                         delete this;
 222                 return ref_count_;
 223         }
 224
 225         // IDecklinkVideoFrame
 226
 227         virtual long STDMETHODCALLTYPE GetWidth()                   {return static_cast<long>(format_desc_.width);}
 228         virtual long STDMETHODCALLTYPE GetHeight()                  {return static_cast<long>(format_desc_.height);}
 229         virtual long STDMETHODCALLTYPE GetRowBytes()                {return static_cast<long>(format_desc_.width*4);}
 230         virtual BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat()   {return bmdFormat8BitBGRA;}
 231         virtual BMDFrameFlags STDMETHODCALLTYPE GetFlags()                      {return bmdFrameFlagDefault;}
 232
 233         virtual HRESULT STDMETHODCALLTYPE GetBytes(void** buffer)
 234         {
 235                 try
 236                 {
 237                         if(static_cast<int>(frame_.image_data().size()) != format_desc_.size)
 238                         {
 239                                 data_.resize(format_desc_.size);
 240                                 *buffer = data_.data();
 241                         }
 242                         else if(key_only_)
 243                         {
 244                                 if(data_.empty())
 245                                 {
 246                                         data_.resize(frame_.image_data().size());
 247                                         aligned_memshfl(data_.data(), frame_.image_data().begin(), frame_.image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
 248                                 }
 249                                 *buffer = data_.data();
 250                         }
 251                         else
 252                         {
 253                                 *buffer = const_cast<uint8_t*>(frame_.image_data().begin());
 254
 255                                 if (needs_to_copy_)
 256                                 {
 257                                         data_.resize(frame_.image_data().size());
 258                                         fast_memcpy(data_.data(), *buffer, frame_.image_data().size());
 259                                         *buffer = data_.data();
 260                                 }
 261                         }
 262                 }
 263                 catch(...)
 264                 {
 265                         CASPAR_LOG_CURRENT_EXCEPTION();
 266                         return E_FAIL;
 267                 }
 268
 269                 return S_OK;
 270         }
 271
 272         virtual HRESULT STDMETHODCALLTYPE GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode) {return S_FALSE;}
 273         virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)              {return S_FALSE;}
 274
 275         // decklink_frame
 276
 277         const core::audio_buffer& audio_data()
 278         {
 279                 return frame_.audio_data();
 280         }
 281
 282         int64_t get_age_millis() const
 283         {
 284                 return frame_.get_age_millis();
 285         }
 286 };
 287
 288 template <typename Configuration>
 289 struct key_video_context : public IDeckLinkVideoOutputCallback, boost::noncopyable
 290 {
 291         const configuration                                     config_;
 292         com_ptr<IDeckLink>                                      decklink_                                       = get_device(config_.key_device_index());
 293         com_iface_ptr<IDeckLinkOutput>          output_                                         = iface_cast<IDeckLinkOutput>(decklink_);
 294         com_iface_ptr<IDeckLinkKeyer>           keyer_                                          = iface_cast<IDeckLinkKeyer>(decklink_, true);
 295         com_iface_ptr<IDeckLinkAttributes>      attributes_                                     = iface_cast<IDeckLinkAttributes>(decklink_);
 296         com_iface_ptr<Configuration>            configuration_                          = iface_cast<Configuration>(decklink_);
 297         tbb::atomic<int64_t>                            current_presentation_delay_;
 298         tbb::atomic<int64_t>                            scheduled_frames_completed_;
 299
 300         key_video_context(const configuration& config, const std::wstring& print)
 301                 : config_(config)
 302         {
 303                 current_presentation_delay_ = 0;
 304                 scheduled_frames_completed_ = 0;
 305
 306                 set_latency(configuration_, config.latency, print);
 307                 set_keyer(attributes_, keyer_, config.keyer, print);
 308
 309                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 310                         CASPAR_THROW_EXCEPTION(caspar_exception()
 311                                         << msg_info(print + L" Failed to set key playback completion callback.")
 312                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 313         }
 314
 315         template<typename Print>
 316         void enable_video(BMDDisplayMode display_mode, const Print& print)
 317         {
 318                 if (FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 319                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable key video output."));
 320
 321                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 322                         CASPAR_THROW_EXCEPTION(caspar_exception()
 323                                         << msg_info(print() + L" Failed to set key playback completion callback.")
 324                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 325         }
 326
 327         virtual ~key_video_context()
 328         {
 329                 if (output_)
 330                 {
 331                         output_->StopScheduledPlayback(0, nullptr, 0);
 332                         output_->DisableVideoOutput();
 333                 }
 334         }
 335
 336         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 337         virtual ULONG STDMETHODCALLTYPE AddRef()                                                        {return 1;}
 338         virtual ULONG STDMETHODCALLTYPE Release()                                                       {return 1;}
 339
 340         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 341         {
 342                 return S_OK;
 343         }
 344
 345         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(
 346                         IDeckLinkVideoFrame* completed_frame,
 347                         BMDOutputFrameCompletionResult result)
 348         {
 349                 auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 350                 current_presentation_delay_ = dframe->get_age_millis();
 351                 ++scheduled_frames_completed_;
 352
 353                 // Let the fill callback keep the pace, so no scheduling here.
 354
 355                 return S_OK;
 356         }
 357 };
 358
 359 template <typename Configuration>
 360 struct decklink_consumer : public IDeckLinkVideoOutputCallback, boost::noncopyable
 361 {
 362         const int                                                                                       channel_index_;
 363         const configuration                                                                     config_;
 364
 365         com_ptr<IDeckLink>                                                                      decklink_                               = get_device(config_.device_index);
 366         com_iface_ptr<IDeckLinkOutput>                                          output_                                 = iface_cast<IDeckLinkOutput>(decklink_);
 367         com_iface_ptr<Configuration>                                            configuration_                  = iface_cast<Configuration>(decklink_);
 368         com_iface_ptr<IDeckLinkKeyer>                                           keyer_                                  = iface_cast<IDeckLinkKeyer>(decklink_, true);
 369         com_iface_ptr<IDeckLinkAttributes>                                      attributes_                             = iface_cast<IDeckLinkAttributes>(decklink_);
 370
 371         tbb::spin_mutex                                                                         exception_mutex_;
 372         std::exception_ptr                                                                      exception_;
 373
 374         tbb::atomic<bool>                                                                       is_running_;
 375
 376         const std::wstring                                                                      model_name_                             = get_model_name(decklink_);
 377         bool                                                                                            will_attempt_dma_;
 378         const core::video_format_desc                                           format_desc_;
 379         const core::audio_channel_layout                                        in_channel_layout_;
 380         const core::audio_channel_layout                                        out_channel_layout_             = config_.get_adjusted_layout(in_channel_layout_);
 381         core::audio_channel_remapper                                            channel_remapper_               { in_channel_layout_, out_channel_layout_ };
 382         const int                                                                                       buffer_size_                    = config_.buffer_depth(); // Minimum buffer-size 3.
 383
 384         long long                                                                                       video_scheduled_                = 0;
 385         long long                                                                                       audio_scheduled_                = 0;
 386
 387         int                                                                                                     preroll_count_                  = 0;
 388
 389         boost::circular_buffer<std::vector<int32_t>>            audio_container_                { buffer_size_ + 1 };
 390
 391         tbb::concurrent_bounded_queue<core::const_frame>        frame_buffer_;
 392
 393         spl::shared_ptr<diagnostics::graph>                                     graph_;
 394         caspar::timer                                                                           tick_timer_;
 395         std::packaged_task<bool ()>                                                     send_completion_;
 396         reference_signal_detector                                                       reference_signal_detector_      { output_ };
 397         tbb::atomic<int64_t>                                                            current_presentation_delay_;
 398         tbb::atomic<int64_t>                                                            scheduled_frames_completed_;
 399         std::unique_ptr<key_video_context<Configuration>>       key_context_;
 400
 401 public:
 402         decklink_consumer(
 403                         const configuration& config,
 404                         const core::video_format_desc& format_desc,
 405                         const core::audio_channel_layout& in_channel_layout,
 406                         int channel_index)
 407                 : channel_index_(channel_index)
 408                 , config_(config)
 409                 , format_desc_(format_desc)
 410                 , in_channel_layout_(in_channel_layout)
 411         {
 412                 is_running_ = true;
 413                 current_presentation_delay_ = 0;
 414                 scheduled_frames_completed_ = 0;
 415
 416                 frame_buffer_.set_capacity(1);
 417
 418                 if (config.keyer == configuration::keyer_t::external_separate_device_keyer)
 419                         key_context_.reset(new key_video_context<Configuration>(config, print()));
 420
 421                 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
 422                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
 423                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
 424                 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
 425                 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
 426                 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
 427
 428                 if (key_context_)
 429                 {
 430                         graph_->set_color("key-offset", diagnostics::color(1.0f, 0.0f, 0.0f));
 431                 }
 432
 433                 graph_->set_text(print());
 434                 diagnostics::register_graph(graph_);
 435
 436                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault, will_attempt_dma_));
 437
 438                 if(config.embedded_audio)
 439                         enable_audio();
 440
 441                 set_latency(configuration_, config.latency, print());
 442                 set_keyer(attributes_, keyer_, config.keyer, print());
 443
 444                 if(config.embedded_audio)
 445                         output_->BeginAudioPreroll();
 446
 447                 for (int n = 0; n < buffer_size_; ++n)
 448                 {
 449                         if (config.embedded_audio)
 450                                 schedule_next_audio(core::mutable_audio_buffer(format_desc_.audio_cadence[n % format_desc_.audio_cadence.size()] * out_channel_layout_.num_channels, 0));
 451
 452                         schedule_next_video(core::const_frame::empty());
 453                 }
 454
 455                 if (config.embedded_audio)
 456                 {
 457                         // Preroll one extra frame worth of audio
 458                         schedule_next_audio(core::mutable_audio_buffer(format_desc_.audio_cadence[buffer_size_ % format_desc_.audio_cadence.size()] * out_channel_layout_.num_channels, 0));
 459                         output_->EndAudioPreroll();
 460                 }
 461
 462                 start_playback();
 463         }
 464
 465         ~decklink_consumer()
 466         {
 467                 is_running_ = false;
 468                 frame_buffer_.try_push(core::const_frame::empty());
 469
 470                 if(output_ != nullptr)
 471                 {
 472                         output_->StopScheduledPlayback(0, nullptr, 0);
 473                         if(config_.embedded_audio)
 474                                 output_->DisableAudioOutput();
 475                         output_->DisableVideoOutput();
 476                 }
 477         }
 478
 479         void enable_audio()
 480         {
 481                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, out_channel_layout_.num_channels, bmdAudioOutputStreamTimestamped)))
 482                                 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable audio output."));
 483
 484                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
 485         }
 486
 487         void enable_video(BMDDisplayMode display_mode)
 488         {
 489                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 490                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable fill video output."));
 491
 492                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 493                         CASPAR_THROW_EXCEPTION(caspar_exception()
 494                                                                         << msg_info(print() + L" Failed to set fill playback completion callback.")
 495                                                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 496
 497                 if (key_context_)
 498                         key_context_->enable_video(display_mode, [this]() { return print(); });
 499         }
 500
 501         void start_playback()
 502         {
 503                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 504                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule fill playback."));
 505
 506                 if (key_context_ && FAILED(key_context_->output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 507                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule key playback."));
 508         }
 509
 510         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 511         virtual ULONG STDMETHODCALLTYPE AddRef()                                        {return 1;}
 512         virtual ULONG STDMETHODCALLTYPE Release()                               {return 1;}
 513
 514         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 515         {
 516                 is_running_ = false;
 517                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
 518                 return S_OK;
 519         }
 520
 521         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result)
 522         {
 523                 if(!is_running_)
 524                         return E_FAIL;
 525
 526                 try
 527                 {
 528                         auto tick_time = tick_timer_.elapsed()*format_desc_.fps * 0.5;
 529                         graph_->set_value("tick-time", tick_time);
 530                         tick_timer_.restart();
 531
 532                         reference_signal_detector_.detect_change([this]() { return print(); });
 533
 534                         auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 535                         current_presentation_delay_ = dframe->get_age_millis();
 536                         ++scheduled_frames_completed_;
 537
 538                         if (key_context_)
 539                                 graph_->set_value(
 540                                                 "key-offset",
 541                                                 static_cast<double>(
 542                                                                 scheduled_frames_completed_
 543                                                                 - key_context_->scheduled_frames_completed_)
 544                                                 * 0.1 + 0.5);
 545
 546                         if(result == bmdOutputFrameDisplayedLate)
 547                         {
 548                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "late-frame");
 549                                 video_scheduled_ += format_desc_.duration;
 550                                 audio_scheduled_ += dframe->audio_data().size() / in_channel_layout_.num_channels;
 551                         }
 552                         else if(result == bmdOutputFrameDropped)
 553                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "dropped-frame");
 554                         else if(result == bmdOutputFrameFlushed)
 555                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "flushed-frame");
 556
 557                         UINT32 buffered;
 558                         output_->GetBufferedVideoFrameCount(&buffered);
 559                         graph_->set_value("buffered-video", static_cast<double>(buffered) / (config_.buffer_depth()));
 560
 561                         if (config_.embedded_audio)
 562                         {
 563                                 output_->GetBufferedAudioSampleFrameCount(&buffered);
 564                                 graph_->set_value("buffered-audio", static_cast<double>(buffered) / (format_desc_.audio_cadence[0] * config_.buffer_depth()));
 565                         }
 566
 567                         auto frame = core::const_frame::empty();
 568
 569                         frame_buffer_.pop(frame);
 570
 571                         if (send_completion_.valid())
 572                         {
 573                                 send_completion_();
 574                                 send_completion_ = std::packaged_task<bool ()>();
 575                         }
 576
 577                         if (config_.embedded_audio)
 578                                 schedule_next_audio(channel_remapper_.mix_and_rearrange(frame.audio_data()));
 579
 580                         schedule_next_video(frame);
 581                 }
 582                 catch(...)
 583                 {
 584                         lock(exception_mutex_, [&]
 585                         {
 586                                 exception_ = std::current_exception();
 587                         });
 588                         return E_FAIL;
 589                 }
 590
 591                 return S_OK;
 592         }
 593
 594         template<typename T>
 595         void schedule_next_audio(const T& audio_data)
 596         {
 597                 auto sample_frame_count = static_cast<int>(audio_data.size()/out_channel_layout_.num_channels);
 598
 599                 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
 600
 601                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
 602                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
 603
 604                 audio_scheduled_ += sample_frame_count;
 605         }
 606
 607         void schedule_next_video(core::const_frame frame)
 608         {
 609                 if (key_context_)
 610                 {
 611                         auto key_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, true, will_attempt_dma_));
 612                         if (FAILED(key_context_->output_->ScheduleVideoFrame(get_raw(key_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 613                                 CASPAR_LOG(error) << print() << L" Failed to schedule key video.";
 614                 }
 615
 616                 auto fill_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, config_.key_only, will_attempt_dma_));
 617                 if (FAILED(output_->ScheduleVideoFrame(get_raw(fill_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 618                         CASPAR_LOG(error) << print() << L" Failed to schedule fill video.";
 619
 620                 video_scheduled_ += format_desc_.duration;
 621         }
 622
 623         std::future<bool> send(core::const_frame frame)
 624         {
 625                 auto exception = lock(exception_mutex_, [&]
 626                 {
 627                         return exception_;
 628                 });
 629
 630                 if(exception != nullptr)
 631                         std::rethrow_exception(exception);
 632
 633                 if(!is_running_)
 634                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Is not running."));
 635
 636                 if (frame_buffer_.try_push(frame))
 637                         return make_ready_future(true);
 638
 639                 send_completion_ = std::packaged_task<bool ()>([frame, this] () mutable -> bool
 640                 {
 641                         frame_buffer_.push(frame);
 642
 643                         return true;
 644                 });
 645
 646                 return send_completion_.get_future();
 647         }
 648
 649         std::wstring print() const
 650         {
 651                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 652                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 653                                 boost::lexical_cast<std::wstring>(config_.device_index) +
 654                                 L"&&" +
 655                                 boost::lexical_cast<std::wstring>(config_.key_device_index()) +
 656                                 L"|" +
 657                                 format_desc_.name + L"]";
 658                 else
 659                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 660                                 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
 661         }
 662 };
 663
 664 template <typename Configuration>
 665 struct decklink_consumer_proxy : public core::frame_consumer
 666 {
 667         core::monitor::subject                                                          monitor_subject_;
 668         const configuration                                                                     config_;
 669         std::unique_ptr<decklink_consumer<Configuration>>       consumer_;
 670         core::video_format_desc                                                         format_desc_;
 671         executor                                                                                        executor_;
 672 public:
 673
 674         decklink_consumer_proxy(const configuration& config)
 675                 : config_(config)
 676                 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
 677         {
 678                 auto ctx = core::diagnostics::call_context::for_thread();
 679                 executor_.begin_invoke([=]
 680                 {
 681                         core::diagnostics::call_context::for_thread() = ctx;
 682                         com_initialize();
 683                 });
 684         }
 685
 686         ~decklink_consumer_proxy()
 687         {
 688                 executor_.invoke([=]
 689                 {
 690                         consumer_.reset();
 691                         com_uninitialize();
 692                 });
 693         }
 694
 695         // frame_consumer
 696
 697         void initialize(const core::video_format_desc& format_desc, const core::audio_channel_layout& channel_layout, int channel_index) override
 698         {
 699                 format_desc_ = format_desc;
 700                 executor_.invoke([=]
 701                 {
 702                         consumer_.reset();
 703                         consumer_.reset(new decklink_consumer<Configuration>(config_, format_desc, channel_layout, channel_index));
 704                 });
 705         }
 706
 707         std::future<bool> send(core::const_frame frame) override
 708         {
 709                 return consumer_->send(frame);
 710         }
 711
 712         std::wstring print() const override
 713         {
 714                 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
 715         }
 716
 717         std::wstring name() const override
 718         {
 719                 return L"decklink";
 720         }
 721
 722         boost::property_tree::wptree info() const override
 723         {
 724                 boost::property_tree::wptree info;
 725                 info.add(L"type", L"decklink");
 726                 info.add(L"key-only", config_.key_only);
 727                 info.add(L"device", config_.device_index);
 728
 729                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 730                 {
 731                         info.add(L"key-device", config_.key_device_index());
 732                 }
 733
 734                 info.add(L"low-latency", config_.latency == configuration::latency_t::low_latency);
 735                 info.add(L"embedded-audio", config_.embedded_audio);
 736                 info.add(L"presentation-frame-age", presentation_frame_age_millis());
 737                 //info.add(L"internal-key", config_.internal_key);
 738                 return info;
 739         }
 740
 741         int buffer_depth() const override
 742         {
 743                 return config_.buffer_depth() + 2;
 744         }
 745
 746         int index() const override
 747         {
 748                 return 300 + config_.device_index;
 749         }
 750
 751         int64_t presentation_frame_age_millis() const override
 752         {
 753                 return consumer_ ? static_cast<int64_t>(consumer_->current_presentation_delay_) : 0;
 754         }
 755
 756         core::monitor::subject& monitor_output()
 757         {
 758                 return monitor_subject_;
 759         }
 760 };
 761
 762 const software_version<3>& get_driver_version()
 763 {
 764         static software_version<3> version(u8(get_version()));
 765
 766         return version;
 767 }
 768
 769 const software_version<3> get_new_configuration_api_version()
 770 {
 771         static software_version<3> NEW_CONFIGURATION_API("10.2");
 772
 773         return NEW_CONFIGURATION_API;
 774 }
 775
 776 void describe_consumer(core::help_sink& sink, const core::help_repository& repo)
 777 {
 778         sink.short_description(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 779         sink.syntax(L"DECKLINK "
 780                                 L"{[device_index:int]|1} "
 781                                 L"{[keyer:INTERNAL_KEY,EXTERNAL_KEY,EXTERNAL_SEPARATE_DEVICE_KEY]} "
 782                                 L"{[low_latency:LOW_LATENCY]} "
 783                                 L"{[embedded_audio:EMBEDDED_AUDIO]} "
 784                                 L"{[key_only:KEY_ONLY]} "
 785                                 L"{CHANNEL_LAYOUT [channel_layout:string]}");
 786         sink.para()->text(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 787         sink.definitions()
 788                 ->item(L"device_index", L"The Blackmagic video card to use (See Blackmagic control panel for card order). Default is 1.")
 789                 ->item(L"keyer",
 790                                 L"If given tries to enable either internal or external keying. Not all Blackmagic cards supports this. "
 791                                 L"There is also a third experimental option (EXTERNAL_SEPARATE_DEVICE_KEY) which allocates device_index + 1 for synhronized key output.")
 792                 ->item(L"low_latency", L"Tries to enable low latency if given.")
 793                 ->item(L"embedded_audio", L"Embeds the audio into the SDI signal if given.")
 794                 ->item(L"key_only",
 795                                 L" will extract only the alpha channel from the "
 796                                 L"channel. This is useful when you have two SDI video cards, and neither has native support "
 797                                 L"for separate fill/key output")
 798                 ->item(L"channel_layout", L"If specified, overrides the audio channel layout used by the channel.");
 799         sink.para()->text(L"Examples:");
 800         sink.example(L">> ADD 1 DECKLINK", L"for using the default device_index of 1.");
 801         sink.example(L">> ADD 1 DECKLINK 2", L"uses device_index 2.");
 802         sink.example(L">> ADD 1 DECKLINK 1 EXTERNAL_KEY EMBEDDED_AUDIO");
 803         sink.example(
 804                         L">> ADD 1 DECKLINK 1 EMBEDDED_AUDIO\n"
 805                         L">> ADD 1 DECKLINK 2 KEY_ONLY", L"uses device with index 1 as fill output with audio and device with index 2 as key output.");
 806         sink.example(
 807                         L">> ADD 1 DECKLINK 1 EXTERNAL_SEPARATE_DEVICE_KEY EMBEDDED_AUDIO",
 808                         L"Uses device 2 for key output. May give better sync between key and fill than the previous method.");
 809 }
 810
 811 spl::shared_ptr<core::frame_consumer> create_consumer(
 812                 const std::vector<std::wstring>& params, core::interaction_sink*)
 813 {
 814         if (params.size() < 1 || !boost::iequals(params.at(0), L"DECKLINK"))
 815                 return core::frame_consumer::empty();
 816
 817         configuration config;
 818
 819         if (params.size() > 1)
 820                 config.device_index = boost::lexical_cast<int>(params.at(1));
 821
 822         if (contains_param(L"INTERNAL_KEY", params))
 823                 config.keyer = configuration::keyer_t::internal_keyer;
 824         else if (contains_param(L"EXTERNAL_KEY", params))
 825                 config.keyer = configuration::keyer_t::external_keyer;
 826         else if (contains_param(L"EXTERNAL_SEPARATE_DEVICE_KEY", params))
 827                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 828         else
 829                 config.keyer = configuration::keyer_t::default_keyer;
 830
 831         if (contains_param(L"LOW_LATENCY", params))
 832                 config.latency = configuration::latency_t::low_latency;
 833
 834         config.embedded_audio   = contains_param(L"EMBEDDED_AUDIO", params);
 835         config.key_only                 = contains_param(L"KEY_ONLY", params);
 836
 837         auto channel_layout = get_param(L"CHANNEL_LAYOUT", params);
 838
 839         if (!channel_layout.empty())
 840         {
 841                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(channel_layout);
 842
 843                 if (!found_layout)
 844                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + channel_layout + L" not found."));
 845
 846                 config.out_channel_layout = *found_layout;
 847         }
 848
 849         bool old_configuration_api = get_driver_version() < get_new_configuration_api_version();
 850
 851         if (old_configuration_api)
 852                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration_v10_2>>(config);
 853         else
 854                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration>>(config);
 855 }
 856
 857 spl::shared_ptr<core::frame_consumer> create_preconfigured_consumer(
 858                 const boost::property_tree::wptree& ptree, core::interaction_sink*)
 859 {
 860         configuration config;
 861
 862         auto keyer = ptree.get(L"keyer", L"default");
 863         if(keyer == L"external")
 864                 config.keyer = configuration::keyer_t::external_keyer;
 865         else if(keyer == L"internal")
 866                 config.keyer = configuration::keyer_t::internal_keyer;
 867         else if (keyer == L"external_separate_device")
 868                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 869
 870         auto latency = ptree.get(L"latency", L"default");
 871         if(latency == L"low")
 872                 config.latency = configuration::latency_t::low_latency;
 873         else if(latency == L"normal")
 874                 config.latency = configuration::latency_t::normal_latency;
 875
 876         auto channel_layout = ptree.get_optional<std::wstring>(L"channel-layout");
 877
 878         if (channel_layout)
 879         {
 880                 CASPAR_SCOPED_CONTEXT_MSG("/channel-layout")
 881
 882                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(*channel_layout);
 883
 884                 if (!found_layout)
 885                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + *channel_layout + L" not found."));
 886
 887                 config.out_channel_layout = *found_layout;
 888         }
 889
 890         config.key_only                         = ptree.get(L"key-only",                config.key_only);
 891         config.device_index                     = ptree.get(L"device",                  config.device_index);
 892         config.key_device_idx           = ptree.get(L"key-device",              config.key_device_idx);
 893         config.embedded_audio           = ptree.get(L"embedded-audio",  config.embedded_audio);
 894         config.base_buffer_depth        = ptree.get(L"buffer-depth",    config.base_buffer_depth);
 895
 896         bool old_configuration_api = get_driver_version() < get_new_configuration_api_version();
 897
 898         if (old_configuration_api)
 899                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration_v10_2>>(config);
 900         else
 901                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration>>(config);
 902 }
 903
 904 }}
 905
 906 /*
 907 ##############################################################################
 908 Pre-rolling
 909
 910 Mail: 2011-05-09
 911
 912 Yoshan
 913 BMD Developer Support
 914 developer@blackmagic-design.com
 915
 916 -----------------------------------------------------------------------------
 917
 918 Thanks for your inquiry. The minimum number of frames that you can preroll
 919 for scheduled playback is three frames for video and four frames for audio.
 920 As you mentioned if you preroll less frames then playback will not start or
 921 playback will be very sporadic. From our experience with Media Express, we
 922 recommended that at least seven frames are prerolled for smooth playback.
 923
 924 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
 925 There can be around 3 frames worth of latency on scheduled output.
 926 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
 927 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync()
 928 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
 929 guarantee that the provided frame will be output as soon the previous
 930 frame output has been completed.
 931 ################################################################################
 932 */
 933
 934 /*
 935 ##############################################################################
 936 Async DMA Transfer without redundant copying
 937
 938 Mail: 2011-05-10
 939
 940 Yoshan
 941 BMD Developer Support
 942 developer@blackmagic-design.com
 943
 944 -----------------------------------------------------------------------------
 945
 946 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
 947 and providing a pointer to your video buffer when GetBytes() is called.
 948 This may help to keep copying to a minimum. Please ensure that the pixel
 949 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
 950 have to colourspace convert which may result in additional copying.
 951 ################################################################################
 952 */