git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Robert Nagy, ronag89@gmail.com
  20 */
  21
  22 #include "../StdAfx.h"
  23
  24 #include "decklink_consumer.h"
  25
  26 #include "../util/util.h"
  27 #include "../decklink.h"
  28
  29 #include "../decklink_api.h"
  30
  31 #include <core/frame/frame.h>
  32 #include <core/frame/audio_channel_layout.h>
  33 #include <core/mixer/audio/audio_mixer.h>
  34 #include <core/consumer/frame_consumer.h>
  35 #include <core/diagnostics/call_context.h>
  36 #include <core/help/help_sink.h>
  37 #include <core/help/help_repository.h>
  38
  39 #include <common/executor.h>
  40 #include <common/lock.h>
  41 #include <common/diagnostics/graph.h>
  42 #include <common/except.h>
  43 #include <common/memshfl.h>
  44 #include <common/memcpy.h>
  45 #include <common/no_init_proxy.h>
  46 #include <common/array.h>
  47 #include <common/future.h>
  48 #include <common/cache_aligned_vector.h>
  49 #include <common/timer.h>
  50 #include <common/param.h>
  51 #include <common/software_version.h>
  52
  53 #include <tbb/concurrent_queue.h>
  54
  55 #include <common/assert.h>
  56 #include <boost/lexical_cast.hpp>
  57 #include <boost/circular_buffer.hpp>
  58 #include <boost/property_tree/ptree.hpp>
  59
  60 namespace caspar { namespace decklink {
  61
  62 struct configuration
  63 {
  64         enum class keyer_t
  65         {
  66                 internal_keyer,
  67                 external_keyer,
  68                 external_separate_device_keyer,
  69                 default_keyer                                   = external_keyer
  70         };
  71
  72         enum class latency_t
  73         {
  74                 low_latency,
  75                 normal_latency,
  76                 default_latency = normal_latency
  77         };
  78
  79         int                                                     device_index            = 1;
  80         int                                                     key_device_idx          = 0;
  81         bool                                            embedded_audio          = true;
  82         keyer_t                                         keyer                           = keyer_t::default_keyer;
  83         latency_t                                       latency                         = latency_t::default_latency;
  84         bool                                            key_only                        = false;
  85         int                                                     base_buffer_depth       = 3;
  86         core::audio_channel_layout      out_channel_layout      = core::audio_channel_layout::invalid();
  87
  88         int buffer_depth() const
  89         {
  90                 return base_buffer_depth + (latency == latency_t::low_latency ? 0 : 1) + (embedded_audio ? 1 : 0);
  91         }
  92
  93         int key_device_index() const
  94         {
  95                 return key_device_idx == 0 ? device_index + 1 : key_device_idx;
  96         }
  97
  98         core::audio_channel_layout get_adjusted_layout(const core::audio_channel_layout& in_layout) const
  99         {
 100                 auto adjusted = out_channel_layout == core::audio_channel_layout::invalid() ? in_layout : out_channel_layout;
 101
 102                 if (adjusted.num_channels == 1) // Duplicate mono-signal into both left and right.
 103                 {
 104                         adjusted.num_channels = 2;
 105                         adjusted.channel_order.push_back(adjusted.channel_order.at(0)); // Usually FC -> FC FC
 106                 }
 107                 else if (adjusted.num_channels == 2)
 108                 {
 109                         adjusted.num_channels = 2;
 110                 }
 111                 else if (adjusted.num_channels <= 8)
 112                 {
 113                         adjusted.num_channels = 8;
 114                 }
 115                 else // Over 8 always pad to 16 or drop >16
 116                 {
 117                         adjusted.num_channels = 16;
 118                 }
 119
 120                 return adjusted;
 121         }
 122 };
 123
 124 template <typename Configuration>
 125 void set_latency(
 126                 const com_iface_ptr<Configuration>& config,
 127                 configuration::latency_t latency,
 128                 const std::wstring& print)
 129 {
 130         if (latency == configuration::latency_t::low_latency)
 131         {
 132                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
 133                 CASPAR_LOG(info) << print << L" Enabled low-latency mode.";
 134         }
 135         else if (latency == configuration::latency_t::normal_latency)
 136         {
 137                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
 138                 CASPAR_LOG(info) << print << L" Disabled low-latency mode.";
 139         }
 140 }
 141
 142 void set_keyer(
 143                 const com_iface_ptr<IDeckLinkAttributes>& attributes,
 144                 const com_iface_ptr<IDeckLinkKeyer>& decklink_keyer,
 145                 configuration::keyer_t keyer,
 146                 const std::wstring& print)
 147 {
 148         if (keyer == configuration::keyer_t::internal_keyer)
 149         {
 150                 BOOL value = true;
 151                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
 152                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 153                 else if (FAILED(decklink_keyer->Enable(FALSE)))
 154                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 155                 else if (FAILED(decklink_keyer->SetLevel(255)))
 156                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 157                 else
 158                         CASPAR_LOG(info) << print << L" Enabled internal keyer.";
 159         }
 160         else if (keyer == configuration::keyer_t::external_keyer)
 161         {
 162                 BOOL value = true;
 163                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
 164                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 165                 else if (FAILED(decklink_keyer->Enable(TRUE)))
 166                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 167                 else if (FAILED(decklink_keyer->SetLevel(255)))
 168                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 169                 else
 170                         CASPAR_LOG(info) << print << L" Enabled external keyer.";
 171         }
 172 }
 173
 174 class decklink_frame : public IDeckLinkVideoFrame
 175 {
 176         tbb::atomic<int>                                                                ref_count_;
 177         core::const_frame                                                               frame_;
 178         const core::video_format_desc                                   format_desc_;
 179
 180         const bool                                                                              key_only_;
 181         bool                                                                                    needs_to_copy_;
 182         cache_aligned_vector<no_init_proxy<uint8_t>>    data_;
 183 public:
 184         decklink_frame(core::const_frame frame, const core::video_format_desc& format_desc, bool key_only, bool will_attempt_dma)
 185                 : frame_(frame)
 186                 , format_desc_(format_desc)
 187                 , key_only_(key_only)
 188         {
 189                 ref_count_ = 0;
 190
 191                 bool dma_transfer_from_gl_buffer_impossible;
 192
 193 #if !defined(_MSC_VER)
 194                 // On Linux Decklink cannot DMA transfer from memory returned by glMapBuffer (at least on nvidia)
 195                 dma_transfer_from_gl_buffer_impossible = true;
 196 #else
 197                 // On Windows it is possible.
 198                 dma_transfer_from_gl_buffer_impossible = false;
 199 #endif
 200
 201                 needs_to_copy_ = will_attempt_dma && dma_transfer_from_gl_buffer_impossible;
 202         }
 203
 204         // IUnknown
 205
 206         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)
 207         {
 208                 return E_NOINTERFACE;
 209         }
 210
 211         virtual ULONG STDMETHODCALLTYPE AddRef()
 212         {
 213                 return ++ref_count_;
 214         }
 215
 216         virtual ULONG STDMETHODCALLTYPE Release()
 217         {
 218                 if(--ref_count_ == 0)
 219                         delete this;
 220                 return ref_count_;
 221         }
 222
 223         // IDecklinkVideoFrame
 224
 225         virtual long STDMETHODCALLTYPE GetWidth()                   {return static_cast<long>(format_desc_.width);}
 226         virtual long STDMETHODCALLTYPE GetHeight()                  {return static_cast<long>(format_desc_.height);}
 227         virtual long STDMETHODCALLTYPE GetRowBytes()                {return static_cast<long>(format_desc_.width*4);}
 228         virtual BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat()   {return bmdFormat8BitBGRA;}
 229         virtual BMDFrameFlags STDMETHODCALLTYPE GetFlags()                      {return bmdFrameFlagDefault;}
 230
 231         virtual HRESULT STDMETHODCALLTYPE GetBytes(void** buffer)
 232         {
 233                 try
 234                 {
 235                         if(static_cast<int>(frame_.image_data().size()) != format_desc_.size)
 236                         {
 237                                 data_.resize(format_desc_.size);
 238                                 *buffer = data_.data();
 239                         }
 240                         else if(key_only_)
 241                         {
 242                                 if(data_.empty())
 243                                 {
 244                                         data_.resize(frame_.image_data().size());
 245                                         aligned_memshfl(data_.data(), frame_.image_data().begin(), frame_.image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
 246                                 }
 247                                 *buffer = data_.data();
 248                         }
 249                         else
 250                         {
 251                                 *buffer = const_cast<uint8_t*>(frame_.image_data().begin());
 252
 253                                 if (needs_to_copy_)
 254                                 {
 255                                         data_.resize(frame_.image_data().size());
 256                                         fast_memcpy(data_.data(), *buffer, frame_.image_data().size());
 257                                         *buffer = data_.data();
 258                                 }
 259                         }
 260                 }
 261                 catch(...)
 262                 {
 263                         CASPAR_LOG_CURRENT_EXCEPTION();
 264                         return E_FAIL;
 265                 }
 266
 267                 return S_OK;
 268         }
 269
 270         virtual HRESULT STDMETHODCALLTYPE GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode) {return S_FALSE;}
 271         virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)              {return S_FALSE;}
 272
 273         // decklink_frame
 274
 275         const core::audio_buffer& audio_data()
 276         {
 277                 return frame_.audio_data();
 278         }
 279
 280         int64_t get_age_millis() const
 281         {
 282                 return frame_.get_age_millis();
 283         }
 284 };
 285
 286 template <typename Configuration>
 287 struct key_video_context : public IDeckLinkVideoOutputCallback, boost::noncopyable
 288 {
 289         const configuration                                     config_;
 290         com_ptr<IDeckLink>                                      decklink_                                       = get_device(config_.key_device_index());
 291         com_iface_ptr<IDeckLinkOutput>          output_                                         = iface_cast<IDeckLinkOutput>(decklink_);
 292         com_iface_ptr<IDeckLinkKeyer>           keyer_                                          = iface_cast<IDeckLinkKeyer>(decklink_, true);
 293         com_iface_ptr<IDeckLinkAttributes>      attributes_                                     = iface_cast<IDeckLinkAttributes>(decklink_);
 294         com_iface_ptr<Configuration>            configuration_                          = iface_cast<Configuration>(decklink_);
 295         tbb::atomic<int64_t>                            current_presentation_delay_;
 296         tbb::atomic<int64_t>                            scheduled_frames_completed_;
 297
 298         key_video_context(const configuration& config, const std::wstring& print)
 299                 : config_(config)
 300         {
 301                 current_presentation_delay_ = 0;
 302                 scheduled_frames_completed_ = 0;
 303
 304                 set_latency(configuration_, config.latency, print);
 305                 set_keyer(attributes_, keyer_, config.keyer, print);
 306
 307                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 308                         CASPAR_THROW_EXCEPTION(caspar_exception()
 309                                         << msg_info(print + L" Failed to set key playback completion callback.")
 310                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 311         }
 312
 313         template<typename Print>
 314         void enable_video(BMDDisplayMode display_mode, const Print& print)
 315         {
 316                 if (FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 317                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable key video output."));
 318
 319                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 320                         CASPAR_THROW_EXCEPTION(caspar_exception()
 321                                         << msg_info(print() + L" Failed to set key playback completion callback.")
 322                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 323         }
 324
 325         virtual ~key_video_context()
 326         {
 327                 if (output_)
 328                 {
 329                         output_->StopScheduledPlayback(0, nullptr, 0);
 330                         output_->DisableVideoOutput();
 331                 }
 332         }
 333
 334         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 335         virtual ULONG STDMETHODCALLTYPE AddRef()                                                        {return 1;}
 336         virtual ULONG STDMETHODCALLTYPE Release()                                                       {return 1;}
 337
 338         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 339         {
 340                 return S_OK;
 341         }
 342
 343         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(
 344                         IDeckLinkVideoFrame* completed_frame,
 345                         BMDOutputFrameCompletionResult result)
 346         {
 347                 auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 348                 current_presentation_delay_ = dframe->get_age_millis();
 349                 ++scheduled_frames_completed_;
 350
 351                 // Let the fill callback keep the pace, so no scheduling here.
 352
 353                 return S_OK;
 354         }
 355 };
 356
 357 template <typename Configuration>
 358 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
 359 {
 360         const int                                                                                       channel_index_;
 361         const configuration                                                                     config_;
 362
 363         com_ptr<IDeckLink>                                                                      decklink_                               = get_device(config_.device_index);
 364         com_iface_ptr<IDeckLinkOutput>                                          output_                                 = iface_cast<IDeckLinkOutput>(decklink_);
 365         com_iface_ptr<Configuration>                                            configuration_                  = iface_cast<Configuration>(decklink_);
 366         com_iface_ptr<IDeckLinkKeyer>                                           keyer_                                  = iface_cast<IDeckLinkKeyer>(decklink_, true);
 367         com_iface_ptr<IDeckLinkAttributes>                                      attributes_                             = iface_cast<IDeckLinkAttributes>(decklink_);
 368
 369         tbb::spin_mutex                                                                         exception_mutex_;
 370         std::exception_ptr                                                                      exception_;
 371
 372         tbb::atomic<bool>                                                                       is_running_;
 373
 374         const std::wstring                                                                      model_name_                             = get_model_name(decklink_);
 375         bool                                                                                            will_attempt_dma_;
 376         const core::video_format_desc                                           format_desc_;
 377         const core::audio_channel_layout                                        in_channel_layout_;
 378         const core::audio_channel_layout                                        out_channel_layout_             = config_.get_adjusted_layout(in_channel_layout_);
 379         core::audio_channel_remapper                                            channel_remapper_               { in_channel_layout_, out_channel_layout_ };
 380         const int                                                                                       buffer_size_                    = config_.buffer_depth(); // Minimum buffer-size 3.
 381
 382         long long                                                                                       video_scheduled_                = 0;
 383         long long                                                                                       audio_scheduled_                = 0;
 384
 385         int                                                                                                     preroll_count_                  = 0;
 386
 387         boost::circular_buffer<std::vector<int32_t>>            audio_container_                { buffer_size_ + 1 };
 388
 389         tbb::concurrent_bounded_queue<core::const_frame>        video_frame_buffer_;
 390         tbb::concurrent_bounded_queue<core::const_frame>        audio_frame_buffer_;
 391
 392         spl::shared_ptr<diagnostics::graph>                                     graph_;
 393         caspar::timer                                                                           tick_timer_;
 394         retry_task<bool>                                                                        send_completion_;
 395         reference_signal_detector                                                       reference_signal_detector_      { output_ };
 396         tbb::atomic<int64_t>                                                            current_presentation_delay_;
 397         tbb::atomic<int64_t>                                                            scheduled_frames_completed_;
 398         std::unique_ptr<key_video_context<Configuration>>       key_context_;
 399
 400 public:
 401         decklink_consumer(
 402                         const configuration& config,
 403                         const core::video_format_desc& format_desc,
 404                         const core::audio_channel_layout& in_channel_layout,
 405                         int channel_index)
 406                 : channel_index_(channel_index)
 407                 , config_(config)
 408                 , format_desc_(format_desc)
 409                 , in_channel_layout_(in_channel_layout)
 410         {
 411                 is_running_ = true;
 412                 current_presentation_delay_ = 0;
 413                 scheduled_frames_completed_ = 0;
 414
 415                 video_frame_buffer_.set_capacity(1);
 416
 417                 // Blackmagic calls RenderAudioSamples() 50 times per second
 418                 // regardless of video mode so we sometimes need to give them
 419                 // samples from 2 frames in order to keep up
 420                 audio_frame_buffer_.set_capacity((format_desc.fps > 50.0) ? 2 : 1);
 421
 422                 if (config.keyer == configuration::keyer_t::external_separate_device_keyer)
 423                         key_context_.reset(new key_video_context<Configuration>(config, print()));
 424
 425                 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
 426                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
 427                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
 428                 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
 429                 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
 430                 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
 431
 432                 if (key_context_)
 433                 {
 434                         graph_->set_color("key-offset", diagnostics::color(1.0f, 0.0f, 0.0f));
 435                 }
 436
 437                 graph_->set_text(print());
 438                 diagnostics::register_graph(graph_);
 439
 440                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault, will_attempt_dma_));
 441
 442                 if(config.embedded_audio)
 443                         enable_audio();
 444
 445                 set_latency(configuration_, config.latency, print());
 446                 set_keyer(attributes_, keyer_, config.keyer, print());
 447
 448                 if(config.embedded_audio)
 449                         output_->BeginAudioPreroll();
 450
 451                 for(int n = 0; n < buffer_size_; ++n)
 452                         schedule_next_video(core::const_frame::empty());
 453
 454                 if(!config.embedded_audio)
 455                         start_playback();
 456         }
 457
 458         ~decklink_consumer()
 459         {
 460                 is_running_ = false;
 461                 video_frame_buffer_.try_push(core::const_frame::empty());
 462                 audio_frame_buffer_.try_push(core::const_frame::empty());
 463
 464                 if(output_ != nullptr)
 465                 {
 466                         output_->StopScheduledPlayback(0, nullptr, 0);
 467                         if(config_.embedded_audio)
 468                                 output_->DisableAudioOutput();
 469                         output_->DisableVideoOutput();
 470                 }
 471         }
 472
 473         void enable_audio()
 474         {
 475                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, out_channel_layout_.num_channels, bmdAudioOutputStreamTimestamped)))
 476                                 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable audio output."));
 477
 478                 if(FAILED(output_->SetAudioCallback(this)))
 479                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not set audio callback."));
 480
 481                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
 482         }
 483
 484         void enable_video(BMDDisplayMode display_mode)
 485         {
 486                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 487                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable fill video output."));
 488
 489                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 490                         CASPAR_THROW_EXCEPTION(caspar_exception()
 491                                                                         << msg_info(print() + L" Failed to set fill playback completion callback.")
 492                                                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 493
 494                 if (key_context_)
 495                         key_context_->enable_video(display_mode, [this]() { return print(); });
 496         }
 497
 498         void start_playback()
 499         {
 500                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 501                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule fill playback."));
 502
 503                 if (key_context_ && FAILED(key_context_->output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 504                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule key playback."));
 505         }
 506
 507         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 508         virtual ULONG STDMETHODCALLTYPE AddRef()                                        {return 1;}
 509         virtual ULONG STDMETHODCALLTYPE Release()                               {return 1;}
 510
 511         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 512         {
 513                 is_running_ = false;
 514                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
 515                 return S_OK;
 516         }
 517
 518         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result)
 519         {
 520                 if(!is_running_)
 521                         return E_FAIL;
 522
 523                 try
 524                 {
 525                         auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 526                         current_presentation_delay_ = dframe->get_age_millis();
 527                         ++scheduled_frames_completed_;
 528
 529                         if (key_context_)
 530                                 graph_->set_value(
 531                                                 "key-offset",
 532                                                 static_cast<double>(
 533                                                                 scheduled_frames_completed_
 534                                                                 - key_context_->scheduled_frames_completed_)
 535                                                 * 0.1 + 0.5);
 536
 537                         if(result == bmdOutputFrameDisplayedLate)
 538                         {
 539                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "late-frame");
 540                                 video_scheduled_ += format_desc_.duration;
 541                                 audio_scheduled_ += dframe->audio_data().size() / out_channel_layout_.num_channels;
 542                                 //++video_scheduled_;
 543                                 //audio_scheduled_ += format_desc_.audio_cadence[0];
 544                                 //++audio_scheduled_;
 545                         }
 546                         else if(result == bmdOutputFrameDropped)
 547                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "dropped-frame");
 548                         else if(result == bmdOutputFrameFlushed)
 549                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "flushed-frame");
 550
 551                         UINT32 buffered;
 552                         output_->GetBufferedVideoFrameCount(&buffered);
 553                         graph_->set_value("buffered-video", static_cast<double>(buffered) / (config_.buffer_depth()));
 554
 555                         auto frame = core::const_frame::empty();
 556                         video_frame_buffer_.pop(frame);
 557                         send_completion_.try_completion();
 558                         schedule_next_video(frame);
 559                 }
 560                 catch(...)
 561                 {
 562                         lock(exception_mutex_, [&]
 563                         {
 564                                 exception_ = std::current_exception();
 565                         });
 566                         return E_FAIL;
 567                 }
 568
 569                 return S_OK;
 570         }
 571
 572         virtual HRESULT STDMETHODCALLTYPE RenderAudioSamples(BOOL preroll)
 573         {
 574                 if(!is_running_)
 575                         return E_FAIL;
 576
 577                 try
 578                 {
 579                         if(preroll)
 580                         {
 581                                 if(++preroll_count_ >= buffer_size_)
 582                                 {
 583                                         output_->EndAudioPreroll();
 584                                         start_playback();
 585                                 }
 586                                 else
 587                                 {
 588                                         schedule_next_audio(core::mutable_audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()] * out_channel_layout_.num_channels, 0));
 589                                 }
 590                         }
 591                         else
 592                         {
 593                                 auto frame = core::const_frame::empty();
 594
 595                                 while(audio_frame_buffer_.try_pop(frame))
 596                                 {
 597                                         UINT32 buffered;
 598                                         output_->GetBufferedAudioSampleFrameCount(&buffered);
 599                                         graph_->set_value("buffered-audio", static_cast<double>(buffered) / (format_desc_.audio_cadence[0] * config_.buffer_depth()));
 600
 601                                         send_completion_.try_completion();
 602                                         schedule_next_audio(channel_remapper_.mix_and_rearrange(frame.audio_data()));
 603                                 }
 604                         }
 605                 }
 606                 catch(...)
 607                 {
 608                         tbb::spin_mutex::scoped_lock lock(exception_mutex_);
 609                         exception_ = std::current_exception();
 610                         return E_FAIL;
 611                 }
 612
 613                 return S_OK;
 614         }
 615
 616         template<typename T>
 617         void schedule_next_audio(const T& audio_data)
 618         {
 619                 auto sample_frame_count = static_cast<int>(audio_data.size()/out_channel_layout_.num_channels);
 620
 621                 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
 622
 623                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
 624                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
 625
 626                 audio_scheduled_ += sample_frame_count;
 627         }
 628
 629         void schedule_next_video(core::const_frame frame)
 630         {
 631                 if (key_context_)
 632                 {
 633                         auto key_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, true, will_attempt_dma_));
 634                         if (FAILED(key_context_->output_->ScheduleVideoFrame(get_raw(key_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 635                                 CASPAR_LOG(error) << print() << L" Failed to schedule key video.";
 636                 }
 637
 638                 auto fill_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, config_.key_only, will_attempt_dma_));
 639                 if (FAILED(output_->ScheduleVideoFrame(get_raw(fill_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 640                         CASPAR_LOG(error) << print() << L" Failed to schedule fill video.";
 641
 642                 video_scheduled_ += format_desc_.duration;
 643
 644                 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
 645                 tick_timer_.restart();
 646
 647                 reference_signal_detector_.detect_change([this]() { return print(); });
 648         }
 649
 650         std::future<bool> send(core::const_frame frame)
 651         {
 652                 auto exception = lock(exception_mutex_, [&]
 653                 {
 654                         return exception_;
 655                 });
 656
 657                 if(exception != nullptr)
 658                         std::rethrow_exception(exception);
 659
 660                 if(!is_running_)
 661                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Is not running."));
 662
 663                 bool audio_ready = !config_.embedded_audio;
 664                 bool video_ready = false;
 665
 666                 auto enqueue_task = [audio_ready, video_ready, frame, this]() mutable -> boost::optional<bool>
 667                 {
 668                         if (!audio_ready)
 669                                 audio_ready = audio_frame_buffer_.try_push(frame);
 670
 671                         if (!video_ready)
 672                                 video_ready = video_frame_buffer_.try_push(frame);
 673
 674                         if (audio_ready && video_ready)
 675                                 return true;
 676                         else
 677                                 return boost::optional<bool>();
 678                 };
 679
 680                 if (enqueue_task())
 681                         return make_ready_future(true);
 682
 683                 send_completion_.set_task(enqueue_task);
 684
 685                 return send_completion_.get_future();
 686         }
 687
 688         std::wstring print() const
 689         {
 690                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 691                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 692                                 boost::lexical_cast<std::wstring>(config_.device_index) +
 693                                 L"&&" +
 694                                 boost::lexical_cast<std::wstring>(config_.key_device_index()) +
 695                                 L"|" +
 696                                 format_desc_.name + L"]";
 697                 else
 698                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 699                                 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
 700         }
 701 };
 702
 703 template <typename Configuration>
 704 struct decklink_consumer_proxy : public core::frame_consumer
 705 {
 706         core::monitor::subject                                                          monitor_subject_;
 707         const configuration                                                                     config_;
 708         std::unique_ptr<decklink_consumer<Configuration>>       consumer_;
 709         core::video_format_desc                                                         format_desc_;
 710         executor                                                                                        executor_;
 711 public:
 712
 713         decklink_consumer_proxy(const configuration& config)
 714                 : config_(config)
 715                 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
 716         {
 717                 auto ctx = core::diagnostics::call_context::for_thread();
 718                 executor_.begin_invoke([=]
 719                 {
 720                         core::diagnostics::call_context::for_thread() = ctx;
 721                         com_initialize();
 722                 });
 723         }
 724
 725         ~decklink_consumer_proxy()
 726         {
 727                 executor_.invoke([=]
 728                 {
 729                         consumer_.reset();
 730                         com_uninitialize();
 731                 });
 732         }
 733
 734         // frame_consumer
 735
 736         void initialize(const core::video_format_desc& format_desc, const core::audio_channel_layout& channel_layout, int channel_index) override
 737         {
 738                 format_desc_ = format_desc;
 739                 executor_.invoke([=]
 740                 {
 741                         consumer_.reset();
 742                         consumer_.reset(new decklink_consumer<Configuration>(config_, format_desc, channel_layout, channel_index));
 743                 });
 744         }
 745
 746         std::future<bool> send(core::const_frame frame) override
 747         {
 748                 return consumer_->send(frame);
 749         }
 750
 751         std::wstring print() const override
 752         {
 753                 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
 754         }
 755
 756         std::wstring name() const override
 757         {
 758                 return L"decklink";
 759         }
 760
 761         boost::property_tree::wptree info() const override
 762         {
 763                 boost::property_tree::wptree info;
 764                 info.add(L"type", L"decklink");
 765                 info.add(L"key-only", config_.key_only);
 766                 info.add(L"device", config_.device_index);
 767
 768                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 769                 {
 770                         info.add(L"key-device", config_.key_device_index());
 771                 }
 772
 773                 info.add(L"low-latency", config_.latency == configuration::latency_t::low_latency);
 774                 info.add(L"embedded-audio", config_.embedded_audio);
 775                 info.add(L"presentation-frame-age", presentation_frame_age_millis());
 776                 //info.add(L"internal-key", config_.internal_key);
 777                 return info;
 778         }
 779
 780         int buffer_depth() const override
 781         {
 782                 return config_.buffer_depth() + 2;
 783         }
 784
 785         int index() const override
 786         {
 787                 return 300 + config_.device_index;
 788         }
 789
 790         int64_t presentation_frame_age_millis() const override
 791         {
 792                 return consumer_ ? static_cast<int64_t>(consumer_->current_presentation_delay_) : 0;
 793         }
 794
 795         core::monitor::subject& monitor_output()
 796         {
 797                 return monitor_subject_;
 798         }
 799 };
 800
 801 const software_version<3>& get_driver_version()
 802 {
 803         static software_version<3> version(u8(get_version()));
 804
 805         return version;
 806 }
 807
 808 const software_version<3> get_new_configuration_api_version()
 809 {
 810         static software_version<3> NEW_CONFIGURATION_API("10.2");
 811
 812         return NEW_CONFIGURATION_API;
 813 }
 814
 815 void describe_consumer(core::help_sink& sink, const core::help_repository& repo)
 816 {
 817         sink.short_description(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 818         sink.syntax(L"DECKLINK "
 819                                 L"{[device_index:int]|1} "
 820                                 L"{[keyer:INTERNAL_KEY,EXTERNAL_KEY,EXTERNAL_SEPARATE_DEVICE_KEY]} "
 821                                 L"{[low_latency:LOW_LATENCY]} "
 822                                 L"{[embedded_audio:EMBEDDED_AUDIO]} "
 823                                 L"{[key_only:KEY_ONLY]} "
 824                                 L"{CHANNEL_LAYOUT [channel_layout:string]}");
 825         sink.para()->text(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 826         sink.definitions()
 827                 ->item(L"device_index", L"The Blackmagic video card to use (See Blackmagic control panel for card order). Default is 1.")
 828                 ->item(L"keyer",
 829                                 L"If given tries to enable either internal or external keying. Not all Blackmagic cards supports this. "
 830                                 L"There is also a third experimental option (EXTERNAL_SEPARATE_DEVICE_KEY) which allocates device_index + 1 for synhronized key output.")
 831                 ->item(L"low_latency", L"Tries to enable low latency if given.")
 832                 ->item(L"embedded_audio", L"Embeds the audio into the SDI signal if given.")
 833                 ->item(L"key_only",
 834                                 L" will extract only the alpha channel from the "
 835                                 L"channel. This is useful when you have two SDI video cards, and neither has native support "
 836                                 L"for separate fill/key output")
 837                 ->item(L"channel_layout", L"If specified, overrides the audio channel layout used by the channel.");
 838         sink.para()->text(L"Examples:");
 839         sink.example(L">> ADD 1 DECKLINK", L"for using the default device_index of 1.");
 840         sink.example(L">> ADD 1 DECKLINK 2", L"uses device_index 2.");
 841         sink.example(L">> ADD 1 DECKLINK 1 EXTERNAL_KEY EMBEDDED_AUDIO");
 842         sink.example(
 843                         L">> ADD 1 DECKLINK 1 EMBEDDED_AUDIO\n"
 844                         L">> ADD 1 DECKLINK 2 KEY_ONLY", L"uses device with index 1 as fill output with audio and device with index 2 as key output.");
 845         sink.example(
 846                         L">> ADD 1 DECKLINK 1 EXTERNAL_SEPARATE_DEVICE_KEY EMBEDDED_AUDIO",
 847                         L"Uses device 2 for key output. May give better sync between key and fill than the previous method.");
 848 }
 849
 850 spl::shared_ptr<core::frame_consumer> create_consumer(
 851                 const std::vector<std::wstring>& params, core::interaction_sink*)
 852 {
 853         if (params.size() < 1 || !boost::iequals(params.at(0), L"DECKLINK"))
 854                 return core::frame_consumer::empty();
 855
 856         configuration config;
 857
 858         if (params.size() > 1)
 859                 config.device_index = boost::lexical_cast<int>(params.at(1));
 860
 861         if (contains_param(L"INTERNAL_KEY", params))
 862                 config.keyer = configuration::keyer_t::internal_keyer;
 863         else if (contains_param(L"EXTERNAL_KEY", params))
 864                 config.keyer = configuration::keyer_t::external_keyer;
 865         else if (contains_param(L"EXTERNAL_SEPARATE_DEVICE_KEY", params))
 866                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 867         else
 868                 config.keyer = configuration::keyer_t::default_keyer;
 869
 870         if (contains_param(L"LOW_LATENCY", params))
 871                 config.latency = configuration::latency_t::low_latency;
 872
 873         config.embedded_audio   = contains_param(L"EMBEDDED_AUDIO", params);
 874         config.key_only                 = contains_param(L"KEY_ONLY", params);
 875
 876         auto channel_layout = get_param(L"CHANNEL_LAYOUT", params);
 877
 878         if (!channel_layout.empty())
 879         {
 880                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(channel_layout);
 881
 882                 if (!found_layout)
 883                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + channel_layout + L" not found."));
 884
 885                 config.out_channel_layout = *found_layout;
 886         }
 887
 888         bool old_configuration_api = get_driver_version() < get_new_configuration_api_version();
 889
 890         if (old_configuration_api)
 891                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration_v10_2>>(config);
 892         else
 893                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration>>(config);
 894 }
 895
 896 spl::shared_ptr<core::frame_consumer> create_preconfigured_consumer(
 897                 const boost::property_tree::wptree& ptree, core::interaction_sink*)
 898 {
 899         configuration config;
 900
 901         auto keyer = ptree.get(L"keyer", L"default");
 902         if(keyer == L"external")
 903                 config.keyer = configuration::keyer_t::external_keyer;
 904         else if(keyer == L"internal")
 905                 config.keyer = configuration::keyer_t::internal_keyer;
 906         else if (keyer == L"external_separate_device")
 907                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 908
 909         auto latency = ptree.get(L"latency", L"default");
 910         if(latency == L"low")
 911                 config.latency = configuration::latency_t::low_latency;
 912         else if(latency == L"normal")
 913                 config.latency = configuration::latency_t::normal_latency;
 914
 915         auto channel_layout = ptree.get_optional<std::wstring>(L"channel-layout");
 916
 917         if (channel_layout)
 918         {
 919                 CASPAR_SCOPED_CONTEXT_MSG("/channel-layout")
 920
 921                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(*channel_layout);
 922
 923                 if (!found_layout)
 924                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + *channel_layout + L" not found."));
 925
 926                 config.out_channel_layout = *found_layout;
 927         }
 928
 929         config.key_only                         = ptree.get(L"key-only",                config.key_only);
 930         config.device_index                     = ptree.get(L"device",                  config.device_index);
 931         config.key_device_idx           = ptree.get(L"key-device",              config.key_device_idx);
 932         config.embedded_audio           = ptree.get(L"embedded-audio",  config.embedded_audio);
 933         config.base_buffer_depth        = ptree.get(L"buffer-depth",    config.base_buffer_depth);
 934
 935         bool old_configuration_api = get_driver_version() < get_new_configuration_api_version();
 936
 937         if (old_configuration_api)
 938                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration_v10_2>>(config);
 939         else
 940                 return spl::make_shared<decklink_consumer_proxy<IDeckLinkConfiguration>>(config);
 941 }
 942
 943 }}
 944
 945 /*
 946 ##############################################################################
 947 Pre-rolling
 948
 949 Mail: 2011-05-09
 950
 951 Yoshan
 952 BMD Developer Support
 953 developer@blackmagic-design.com
 954
 955 -----------------------------------------------------------------------------
 956
 957 Thanks for your inquiry. The minimum number of frames that you can preroll
 958 for scheduled playback is three frames for video and four frames for audio.
 959 As you mentioned if you preroll less frames then playback will not start or
 960 playback will be very sporadic. From our experience with Media Express, we
 961 recommended that at least seven frames are prerolled for smooth playback.
 962
 963 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
 964 There can be around 3 frames worth of latency on scheduled output.
 965 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
 966 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync()
 967 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
 968 guarantee that the provided frame will be output as soon the previous
 969 frame output has been completed.
 970 ################################################################################
 971 */
 972
 973 /*
 974 ##############################################################################
 975 Async DMA Transfer without redundant copying
 976
 977 Mail: 2011-05-10
 978
 979 Yoshan
 980 BMD Developer Support
 981 developer@blackmagic-design.com
 982
 983 -----------------------------------------------------------------------------
 984
 985 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
 986 and providing a pointer to your video buffer when GetBytes() is called.
 987 This may help to keep copying to a minimum. Please ensure that the pixel
 988 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
 989 have to colourspace convert which may result in additional copying.
 990 ################################################################################
 991 */