git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Robert Nagy, ronag89@gmail.com
  20 */
  21
  22 #include "../StdAfx.h"
  23
  24 #include "decklink_consumer.h"
  25
  26 #include "../util/util.h"
  27
  28 #include "../decklink_api.h"
  29
  30 #include <core/frame/frame.h>
  31 #include <core/frame/audio_channel_layout.h>
  32 #include <core/mixer/audio/audio_mixer.h>
  33 #include <core/consumer/frame_consumer.h>
  34 #include <core/diagnostics/call_context.h>
  35 #include <core/help/help_sink.h>
  36 #include <core/help/help_repository.h>
  37
  38 #include <common/executor.h>
  39 #include <common/lock.h>
  40 #include <common/diagnostics/graph.h>
  41 #include <common/except.h>
  42 #include <common/memshfl.h>
  43 #include <common/memcpy.h>
  44 #include <common/no_init_proxy.h>
  45 #include <common/array.h>
  46 #include <common/future.h>
  47 #include <common/cache_aligned_vector.h>
  48 #include <common/timer.h>
  49 #include <common/param.h>
  50
  51 #include <tbb/concurrent_queue.h>
  52
  53 #include <common/assert.h>
  54 #include <boost/lexical_cast.hpp>
  55 #include <boost/circular_buffer.hpp>
  56 #include <boost/property_tree/ptree.hpp>
  57
  58 namespace caspar { namespace decklink {
  59
  60 struct configuration
  61 {
  62         enum class keyer_t
  63         {
  64                 internal_keyer,
  65                 external_keyer,
  66                 external_separate_device_keyer,
  67                 default_keyer                                   = external_keyer
  68         };
  69
  70         enum class latency_t
  71         {
  72                 low_latency,
  73                 normal_latency,
  74                 default_latency = normal_latency
  75         };
  76
  77         int                                                     device_index            = 1;
  78         int                                                     key_device_idx          = 0;
  79         bool                                            embedded_audio          = true;
  80         keyer_t                                         keyer                           = keyer_t::default_keyer;
  81         latency_t                                       latency                         = latency_t::default_latency;
  82         bool                                            key_only                        = false;
  83         int                                                     base_buffer_depth       = 3;
  84         core::audio_channel_layout      out_channel_layout      = core::audio_channel_layout::invalid();
  85
  86         int buffer_depth() const
  87         {
  88                 return base_buffer_depth + (latency == latency_t::low_latency ? 0 : 1) + (embedded_audio ? 1 : 0);
  89         }
  90
  91         int key_device_index() const
  92         {
  93                 return key_device_idx == 0 ? device_index + 1 : key_device_idx;
  94         }
  95
  96         core::audio_channel_layout get_adjusted_layout(const core::audio_channel_layout& in_layout) const
  97         {
  98                 auto adjusted = out_channel_layout == core::audio_channel_layout::invalid() ? in_layout : out_channel_layout;
  99
 100                 if (adjusted.num_channels == 1) // Duplicate mono-signal into both left and right.
 101                 {
 102                         adjusted.num_channels = 2;
 103                         adjusted.channel_order.push_back(adjusted.channel_order.at(0)); // Usually FC -> FC FC
 104                 }
 105                 else if (adjusted.num_channels == 2)
 106                 {
 107                         adjusted.num_channels = 2;
 108                 }
 109                 else if (adjusted.num_channels <= 8)
 110                 {
 111                         adjusted.num_channels = 8;
 112                 }
 113                 else // Over 8 always pad to 16 or drop >16
 114                 {
 115                         adjusted.num_channels = 16;
 116                 }
 117
 118                 return adjusted;
 119         }
 120 };
 121
 122 void set_latency(
 123                 const com_iface_ptr<IDeckLinkConfiguration_v10_2>& config,
 124                 configuration::latency_t latency,
 125                 const std::wstring& print)
 126 {
 127         if (latency == configuration::latency_t::low_latency)
 128         {
 129                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
 130                 CASPAR_LOG(info) << print << L" Enabled low-latency mode.";
 131         }
 132         else if (latency == configuration::latency_t::normal_latency)
 133         {
 134                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
 135                 CASPAR_LOG(info) << print << L" Disabled low-latency mode.";
 136         }
 137 }
 138
 139 void set_keyer(
 140                 const com_iface_ptr<IDeckLinkAttributes>& attributes,
 141                 const com_iface_ptr<IDeckLinkKeyer>& decklink_keyer,
 142                 configuration::keyer_t keyer,
 143                 const std::wstring& print)
 144 {
 145         if (keyer == configuration::keyer_t::internal_keyer)
 146         {
 147                 BOOL value = true;
 148                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
 149                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 150                 else if (FAILED(decklink_keyer->Enable(FALSE)))
 151                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 152                 else if (FAILED(decklink_keyer->SetLevel(255)))
 153                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 154                 else
 155                         CASPAR_LOG(info) << print << L" Enabled internal keyer.";
 156         }
 157         else if (keyer == configuration::keyer_t::external_keyer)
 158         {
 159                 BOOL value = true;
 160                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
 161                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 162                 else if (FAILED(decklink_keyer->Enable(TRUE)))
 163                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 164                 else if (FAILED(decklink_keyer->SetLevel(255)))
 165                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 166                 else
 167                         CASPAR_LOG(info) << print << L" Enabled external keyer.";
 168         }
 169 }
 170
 171 class decklink_frame : public IDeckLinkVideoFrame
 172 {
 173         tbb::atomic<int>                                                                ref_count_;
 174         core::const_frame                                                               frame_;
 175         const core::video_format_desc                                   format_desc_;
 176
 177         const bool                                                                              key_only_;
 178         bool                                                                                    needs_to_copy_;
 179         cache_aligned_vector<no_init_proxy<uint8_t>>    data_;
 180 public:
 181         decklink_frame(core::const_frame frame, const core::video_format_desc& format_desc, bool key_only, bool will_attempt_dma)
 182                 : frame_(frame)
 183                 , format_desc_(format_desc)
 184                 , key_only_(key_only)
 185         {
 186                 ref_count_ = 0;
 187
 188                 bool dma_transfer_from_gl_buffer_impossible;
 189
 190 #if !defined(_MSC_VER)
 191                 // On Linux Decklink cannot DMA transfer from memory returned by glMapBuffer (at least on nvidia)
 192                 dma_transfer_from_gl_buffer_impossible = true;
 193 #else
 194                 // On Windows it is possible.
 195                 dma_transfer_from_gl_buffer_impossible = false;
 196 #endif
 197
 198                 needs_to_copy_ = will_attempt_dma && dma_transfer_from_gl_buffer_impossible;
 199         }
 200
 201         // IUnknown
 202
 203         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)
 204         {
 205                 return E_NOINTERFACE;
 206         }
 207
 208         virtual ULONG STDMETHODCALLTYPE AddRef()
 209         {
 210                 return ++ref_count_;
 211         }
 212
 213         virtual ULONG STDMETHODCALLTYPE Release()
 214         {
 215                 if(--ref_count_ == 0)
 216                         delete this;
 217                 return ref_count_;
 218         }
 219
 220         // IDecklinkVideoFrame
 221
 222         virtual long STDMETHODCALLTYPE GetWidth()                   {return static_cast<long>(format_desc_.width);}
 223         virtual long STDMETHODCALLTYPE GetHeight()                  {return static_cast<long>(format_desc_.height);}
 224         virtual long STDMETHODCALLTYPE GetRowBytes()                {return static_cast<long>(format_desc_.width*4);}
 225         virtual BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat()   {return bmdFormat8BitBGRA;}
 226         virtual BMDFrameFlags STDMETHODCALLTYPE GetFlags()                      {return bmdFrameFlagDefault;}
 227
 228         virtual HRESULT STDMETHODCALLTYPE GetBytes(void** buffer)
 229         {
 230                 try
 231                 {
 232                         if(static_cast<int>(frame_.image_data().size()) != format_desc_.size)
 233                         {
 234                                 data_.resize(format_desc_.size);
 235                                 *buffer = data_.data();
 236                         }
 237                         else if(key_only_)
 238                         {
 239                                 if(data_.empty())
 240                                 {
 241                                         data_.resize(frame_.image_data().size());
 242                                         aligned_memshfl(data_.data(), frame_.image_data().begin(), frame_.image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
 243                                 }
 244                                 *buffer = data_.data();
 245                         }
 246                         else
 247                         {
 248                                 *buffer = const_cast<uint8_t*>(frame_.image_data().begin());
 249
 250                                 if (needs_to_copy_)
 251                                 {
 252                                         data_.resize(frame_.image_data().size());
 253                                         fast_memcpy(data_.data(), *buffer, frame_.image_data().size());
 254                                         *buffer = data_.data();
 255                                 }
 256                         }
 257                 }
 258                 catch(...)
 259                 {
 260                         CASPAR_LOG_CURRENT_EXCEPTION();
 261                         return E_FAIL;
 262                 }
 263
 264                 return S_OK;
 265         }
 266
 267         virtual HRESULT STDMETHODCALLTYPE GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode) {return S_FALSE;}
 268         virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)              {return S_FALSE;}
 269
 270         // decklink_frame
 271
 272         const core::audio_buffer& audio_data()
 273         {
 274                 return frame_.audio_data();
 275         }
 276
 277         int64_t get_age_millis() const
 278         {
 279                 return frame_.get_age_millis();
 280         }
 281 };
 282
 283 struct key_video_context : public IDeckLinkVideoOutputCallback, boost::noncopyable
 284 {
 285         const configuration                                                                             config_;
 286         com_ptr<IDeckLink>                                                                              decklink_                                       = get_device(config_.key_device_index());
 287         com_iface_ptr<IDeckLinkOutput>                                                  output_                                         = iface_cast<IDeckLinkOutput>(decklink_);
 288         com_iface_ptr<IDeckLinkKeyer>                                                   keyer_                                          = iface_cast<IDeckLinkKeyer>(decklink_);
 289         com_iface_ptr<IDeckLinkAttributes>                                              attributes_                                     = iface_cast<IDeckLinkAttributes>(decklink_);
 290         com_iface_ptr<IDeckLinkConfiguration_v10_2>                             configuration_                          = iface_cast<IDeckLinkConfiguration_v10_2>(decklink_);
 291         tbb::atomic<int64_t>                                                                    current_presentation_delay_;
 292         tbb::atomic<int64_t>                                                                    scheduled_frames_completed_;
 293
 294         key_video_context(const configuration& config, const std::wstring& print)
 295                 : config_(config)
 296         {
 297                 current_presentation_delay_ = 0;
 298                 scheduled_frames_completed_ = 0;
 299
 300                 set_latency(configuration_, config.latency, print);
 301                 set_keyer(attributes_, keyer_, config.keyer, print);
 302
 303                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 304                         CASPAR_THROW_EXCEPTION(caspar_exception()
 305                                         << msg_info(print + L" Failed to set key playback completion callback.")
 306                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 307         }
 308
 309         template<typename Print>
 310         void enable_video(BMDDisplayMode display_mode, const Print& print)
 311         {
 312                 if (FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 313                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable key video output."));
 314
 315                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 316                         CASPAR_THROW_EXCEPTION(caspar_exception()
 317                                         << msg_info(print() + L" Failed to set key playback completion callback.")
 318                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 319         }
 320
 321         virtual ~key_video_context()
 322         {
 323                 if (output_)
 324                 {
 325                         output_->StopScheduledPlayback(0, nullptr, 0);
 326                         output_->DisableVideoOutput();
 327                 }
 328         }
 329
 330         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 331         virtual ULONG STDMETHODCALLTYPE AddRef()                                                        {return 1;}
 332         virtual ULONG STDMETHODCALLTYPE Release()                                                       {return 1;}
 333
 334         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 335         {
 336                 return S_OK;
 337         }
 338
 339         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(
 340                         IDeckLinkVideoFrame* completed_frame,
 341                         BMDOutputFrameCompletionResult result)
 342         {
 343                 auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 344                 current_presentation_delay_ = dframe->get_age_millis();
 345                 ++scheduled_frames_completed_;
 346
 347                 // Let the fill callback keep the pace, so no scheduling here.
 348
 349                 return S_OK;
 350         }
 351 };
 352
 353 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
 354 {
 355         const int                                                                                       channel_index_;
 356         const configuration                                                                     config_;
 357
 358         com_ptr<IDeckLink>                                                                      decklink_                               = get_device(config_.device_index);
 359         com_iface_ptr<IDeckLinkOutput>                                          output_                                 = iface_cast<IDeckLinkOutput>(decklink_);
 360         com_iface_ptr<IDeckLinkConfiguration_v10_2>                     configuration_                  = iface_cast<IDeckLinkConfiguration_v10_2>(decklink_);
 361         com_iface_ptr<IDeckLinkKeyer>                                           keyer_                                  = iface_cast<IDeckLinkKeyer>(decklink_);
 362         com_iface_ptr<IDeckLinkAttributes>                                      attributes_                             = iface_cast<IDeckLinkAttributes>(decklink_);
 363
 364         tbb::spin_mutex                                                                         exception_mutex_;
 365         std::exception_ptr                                                                      exception_;
 366
 367         tbb::atomic<bool>                                                                       is_running_;
 368
 369         const std::wstring                                                                      model_name_                             = get_model_name(decklink_);
 370         bool                                                                                            will_attempt_dma_;
 371         const core::video_format_desc                                           format_desc_;
 372         const core::audio_channel_layout                                        in_channel_layout_;
 373         const core::audio_channel_layout                                        out_channel_layout_             = config_.get_adjusted_layout(in_channel_layout_);
 374         core::audio_channel_remapper                                            channel_remapper_               { in_channel_layout_, out_channel_layout_ };
 375         const int                                                                                       buffer_size_                    = config_.buffer_depth(); // Minimum buffer-size 3.
 376
 377         long long                                                                                       video_scheduled_                = 0;
 378         long long                                                                                       audio_scheduled_                = 0;
 379
 380         int                                                                                                     preroll_count_                  = 0;
 381
 382         boost::circular_buffer<std::vector<int32_t>>            audio_container_                { buffer_size_ + 1 };
 383
 384         tbb::concurrent_bounded_queue<core::const_frame>        video_frame_buffer_;
 385         tbb::concurrent_bounded_queue<core::const_frame>        audio_frame_buffer_;
 386
 387         spl::shared_ptr<diagnostics::graph>                                     graph_;
 388         caspar::timer                                                                           tick_timer_;
 389         retry_task<bool>                                                                        send_completion_;
 390         reference_signal_detector                                                       reference_signal_detector_      { output_ };
 391         tbb::atomic<int64_t>                                                            current_presentation_delay_;
 392         tbb::atomic<int64_t>                                                            scheduled_frames_completed_;
 393         std::unique_ptr<key_video_context>                                      key_context_;
 394
 395 public:
 396         decklink_consumer(
 397                         const configuration& config,
 398                         const core::video_format_desc& format_desc,
 399                         const core::audio_channel_layout& in_channel_layout,
 400                         int channel_index)
 401                 : channel_index_(channel_index)
 402                 , config_(config)
 403                 , format_desc_(format_desc)
 404                 , in_channel_layout_(in_channel_layout)
 405         {
 406                 is_running_ = true;
 407                 current_presentation_delay_ = 0;
 408                 scheduled_frames_completed_ = 0;
 409
 410                 video_frame_buffer_.set_capacity(1);
 411
 412                 // Blackmagic calls RenderAudioSamples() 50 times per second
 413                 // regardless of video mode so we sometimes need to give them
 414                 // samples from 2 frames in order to keep up
 415                 audio_frame_buffer_.set_capacity((format_desc.fps > 50.0) ? 2 : 1);
 416
 417                 if (config.keyer == configuration::keyer_t::external_separate_device_keyer)
 418                         key_context_.reset(new key_video_context(config, print()));
 419
 420                 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
 421                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
 422                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
 423                 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
 424                 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
 425                 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
 426
 427                 if (key_context_)
 428                 {
 429                         graph_->set_color("key-offset", diagnostics::color(1.0f, 0.0f, 0.0f));
 430                 }
 431
 432                 graph_->set_text(print());
 433                 diagnostics::register_graph(graph_);
 434
 435                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault, will_attempt_dma_));
 436
 437                 if(config.embedded_audio)
 438                         enable_audio();
 439
 440                 set_latency(configuration_, config.latency, print());
 441                 set_keyer(attributes_, keyer_, config.keyer, print());
 442
 443                 if(config.embedded_audio)
 444                         output_->BeginAudioPreroll();
 445
 446                 for(int n = 0; n < buffer_size_; ++n)
 447                         schedule_next_video(core::const_frame::empty());
 448
 449                 if(!config.embedded_audio)
 450                         start_playback();
 451         }
 452
 453         ~decklink_consumer()
 454         {
 455                 is_running_ = false;
 456                 video_frame_buffer_.try_push(core::const_frame::empty());
 457                 audio_frame_buffer_.try_push(core::const_frame::empty());
 458
 459                 if(output_ != nullptr)
 460                 {
 461                         output_->StopScheduledPlayback(0, nullptr, 0);
 462                         if(config_.embedded_audio)
 463                                 output_->DisableAudioOutput();
 464                         output_->DisableVideoOutput();
 465                 }
 466         }
 467
 468         void enable_audio()
 469         {
 470                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, out_channel_layout_.num_channels, bmdAudioOutputStreamTimestamped)))
 471                                 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable audio output."));
 472
 473                 if(FAILED(output_->SetAudioCallback(this)))
 474                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not set audio callback."));
 475
 476                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
 477         }
 478
 479         void enable_video(BMDDisplayMode display_mode)
 480         {
 481                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 482                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable fill video output."));
 483
 484                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 485                         CASPAR_THROW_EXCEPTION(caspar_exception()
 486                                                                         << msg_info(print() + L" Failed to set fill playback completion callback.")
 487                                                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 488
 489                 if (key_context_)
 490                         key_context_->enable_video(display_mode, [this]() { return print(); });
 491         }
 492
 493         void start_playback()
 494         {
 495                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 496                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule fill playback."));
 497
 498                 if (key_context_ && FAILED(key_context_->output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 499                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule key playback."));
 500         }
 501
 502         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 503         virtual ULONG STDMETHODCALLTYPE AddRef()                                        {return 1;}
 504         virtual ULONG STDMETHODCALLTYPE Release()                               {return 1;}
 505
 506         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 507         {
 508                 is_running_ = false;
 509                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
 510                 return S_OK;
 511         }
 512
 513         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result)
 514         {
 515                 if(!is_running_)
 516                         return E_FAIL;
 517
 518                 try
 519                 {
 520                         auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 521                         current_presentation_delay_ = dframe->get_age_millis();
 522                         ++scheduled_frames_completed_;
 523
 524                         if (key_context_)
 525                                 graph_->set_value(
 526                                                 "key-offset",
 527                                                 static_cast<double>(
 528                                                                 scheduled_frames_completed_
 529                                                                 - key_context_->scheduled_frames_completed_)
 530                                                 * 0.1 + 0.5);
 531
 532                         if(result == bmdOutputFrameDisplayedLate)
 533                         {
 534                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "late-frame");
 535                                 video_scheduled_ += format_desc_.duration;
 536                                 audio_scheduled_ += dframe->audio_data().size() / out_channel_layout_.num_channels;
 537                                 //++video_scheduled_;
 538                                 //audio_scheduled_ += format_desc_.audio_cadence[0];
 539                                 //++audio_scheduled_;
 540                         }
 541                         else if(result == bmdOutputFrameDropped)
 542                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "dropped-frame");
 543                         else if(result == bmdOutputFrameFlushed)
 544                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "flushed-frame");
 545
 546                         UINT32 buffered;
 547                         output_->GetBufferedVideoFrameCount(&buffered);
 548                         graph_->set_value("buffered-video", static_cast<double>(buffered) / (config_.buffer_depth()));
 549
 550                         auto frame = core::const_frame::empty();
 551                         video_frame_buffer_.pop(frame);
 552                         send_completion_.try_completion();
 553                         schedule_next_video(frame);
 554                 }
 555                 catch(...)
 556                 {
 557                         lock(exception_mutex_, [&]
 558                         {
 559                                 exception_ = std::current_exception();
 560                         });
 561                         return E_FAIL;
 562                 }
 563
 564                 return S_OK;
 565         }
 566
 567         virtual HRESULT STDMETHODCALLTYPE RenderAudioSamples(BOOL preroll)
 568         {
 569                 if(!is_running_)
 570                         return E_FAIL;
 571
 572                 try
 573                 {
 574                         if(preroll)
 575                         {
 576                                 if(++preroll_count_ >= buffer_size_)
 577                                 {
 578                                         output_->EndAudioPreroll();
 579                                         start_playback();
 580                                 }
 581                                 else
 582                                 {
 583                                         schedule_next_audio(core::mutable_audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()] * out_channel_layout_.num_channels, 0));
 584                                 }
 585                         }
 586                         else
 587                         {
 588                                 auto frame = core::const_frame::empty();
 589
 590                                 while(audio_frame_buffer_.try_pop(frame))
 591                                 {
 592                                         UINT32 buffered;
 593                                         output_->GetBufferedAudioSampleFrameCount(&buffered);
 594                                         graph_->set_value("buffered-audio", static_cast<double>(buffered) / (format_desc_.audio_cadence[0] * config_.buffer_depth()));
 595
 596                                         send_completion_.try_completion();
 597                                         schedule_next_audio(channel_remapper_.mix_and_rearrange(frame.audio_data()));
 598                                 }
 599                         }
 600                 }
 601                 catch(...)
 602                 {
 603                         tbb::spin_mutex::scoped_lock lock(exception_mutex_);
 604                         exception_ = std::current_exception();
 605                         return E_FAIL;
 606                 }
 607
 608                 return S_OK;
 609         }
 610
 611         template<typename T>
 612         void schedule_next_audio(const T& audio_data)
 613         {
 614                 auto sample_frame_count = static_cast<int>(audio_data.size()/out_channel_layout_.num_channels);
 615
 616                 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
 617
 618                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
 619                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
 620
 621                 audio_scheduled_ += sample_frame_count;
 622         }
 623
 624         void schedule_next_video(core::const_frame frame)
 625         {
 626                 if (key_context_)
 627                 {
 628                         auto key_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, true, will_attempt_dma_));
 629                         if (FAILED(key_context_->output_->ScheduleVideoFrame(get_raw(key_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 630                                 CASPAR_LOG(error) << print() << L" Failed to schedule key video.";
 631                 }
 632
 633                 auto fill_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, config_.key_only, will_attempt_dma_));
 634                 if (FAILED(output_->ScheduleVideoFrame(get_raw(fill_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 635                         CASPAR_LOG(error) << print() << L" Failed to schedule fill video.";
 636
 637                 video_scheduled_ += format_desc_.duration;
 638
 639                 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
 640                 tick_timer_.restart();
 641
 642                 reference_signal_detector_.detect_change([this]() { return print(); });
 643         }
 644
 645         std::future<bool> send(core::const_frame frame)
 646         {
 647                 auto exception = lock(exception_mutex_, [&]
 648                 {
 649                         return exception_;
 650                 });
 651
 652                 if(exception != nullptr)
 653                         std::rethrow_exception(exception);
 654
 655                 if(!is_running_)
 656                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Is not running."));
 657
 658                 bool audio_ready = !config_.embedded_audio;
 659                 bool video_ready = false;
 660
 661                 auto enqueue_task = [audio_ready, video_ready, frame, this]() mutable -> boost::optional<bool>
 662                 {
 663                         if (!audio_ready)
 664                                 audio_ready = audio_frame_buffer_.try_push(frame);
 665
 666                         if (!video_ready)
 667                                 video_ready = video_frame_buffer_.try_push(frame);
 668
 669                         if (audio_ready && video_ready)
 670                                 return true;
 671                         else
 672                                 return boost::optional<bool>();
 673                 };
 674
 675                 if (enqueue_task())
 676                         return make_ready_future(true);
 677
 678                 send_completion_.set_task(enqueue_task);
 679
 680                 return send_completion_.get_future();
 681         }
 682
 683         std::wstring print() const
 684         {
 685                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 686                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 687                                 boost::lexical_cast<std::wstring>(config_.device_index) +
 688                                 L"&&" +
 689                                 boost::lexical_cast<std::wstring>(config_.key_device_index()) +
 690                                 L"|" +
 691                                 format_desc_.name + L"]";
 692                 else
 693                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 694                                 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
 695         }
 696 };
 697
 698 struct decklink_consumer_proxy : public core::frame_consumer
 699 {
 700         core::monitor::subject                          monitor_subject_;
 701         const configuration                                     config_;
 702         std::unique_ptr<decklink_consumer>      consumer_;
 703         core::video_format_desc                         format_desc_;
 704         executor                                                        executor_;
 705 public:
 706
 707         decklink_consumer_proxy(const configuration& config)
 708                 : config_(config)
 709                 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
 710         {
 711                 auto ctx = core::diagnostics::call_context::for_thread();
 712                 executor_.begin_invoke([=]
 713                 {
 714                         core::diagnostics::call_context::for_thread() = ctx;
 715                         com_initialize();
 716                 });
 717         }
 718
 719         ~decklink_consumer_proxy()
 720         {
 721                 executor_.invoke([=]
 722                 {
 723                         consumer_.reset();
 724                         com_uninitialize();
 725                 });
 726         }
 727
 728         // frame_consumer
 729
 730         void initialize(const core::video_format_desc& format_desc, const core::audio_channel_layout& channel_layout, int channel_index) override
 731         {
 732                 format_desc_ = format_desc;
 733                 executor_.invoke([=]
 734                 {
 735                         consumer_.reset();
 736                         consumer_.reset(new decklink_consumer(config_, format_desc, channel_layout, channel_index));
 737                 });
 738         }
 739
 740         std::future<bool> send(core::const_frame frame) override
 741         {
 742                 return consumer_->send(frame);
 743         }
 744
 745         std::wstring print() const override
 746         {
 747                 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
 748         }
 749
 750         std::wstring name() const override
 751         {
 752                 return L"decklink";
 753         }
 754
 755         boost::property_tree::wptree info() const override
 756         {
 757                 boost::property_tree::wptree info;
 758                 info.add(L"type", L"decklink");
 759                 info.add(L"key-only", config_.key_only);
 760                 info.add(L"device", config_.device_index);
 761
 762                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 763                 {
 764                         info.add(L"key-device", config_.key_device_index());
 765                 }
 766
 767                 info.add(L"low-latency", config_.latency == configuration::latency_t::low_latency);
 768                 info.add(L"embedded-audio", config_.embedded_audio);
 769                 info.add(L"presentation-frame-age", presentation_frame_age_millis());
 770                 //info.add(L"internal-key", config_.internal_key);
 771                 return info;
 772         }
 773
 774         int buffer_depth() const override
 775         {
 776                 return config_.buffer_depth() + 2;
 777         }
 778
 779         int index() const override
 780         {
 781                 return 300 + config_.device_index;
 782         }
 783
 784         int64_t presentation_frame_age_millis() const override
 785         {
 786                 return consumer_ ? static_cast<int64_t>(consumer_->current_presentation_delay_) : 0;
 787         }
 788
 789         core::monitor::subject& monitor_output()
 790         {
 791                 return monitor_subject_;
 792         }
 793 };
 794
 795 void describe_consumer(core::help_sink& sink, const core::help_repository& repo)
 796 {
 797         sink.short_description(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 798         sink.syntax(L"DECKLINK "
 799                                 L"{[device_index:int]|1} "
 800                                 L"{[keyer:INTERNAL_KEY,EXTERNAL_KEY,EXTERNAL_SEPARATE_DEVICE_KEY]} "
 801                                 L"{[low_latency:LOW_LATENCY]} "
 802                                 L"{[embedded_audio:EMBEDDED_AUDIO]} "
 803                                 L"{[key_only:KEY_ONLY]} "
 804                                 L"{CHANNEL_LAYOUT [channel_layout:string]}");
 805         sink.para()->text(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 806         sink.definitions()
 807                 ->item(L"device_index", L"The Blackmagic video card to use (See Blackmagic control panel for card order). Default is 1.")
 808                 ->item(L"keyer",
 809                                 L"If given tries to enable either internal or external keying. Not all Blackmagic cards supports this. "
 810                                 L"There is also a third experimental option (EXTERNAL_SEPARATE_DEVICE_KEY) which allocates device_index + 1 for synhronized key output.")
 811                 ->item(L"low_latency", L"Tries to enable low latency if given.")
 812                 ->item(L"embedded_audio", L"Embeds the audio into the SDI signal if given.")
 813                 ->item(L"key_only",
 814                                 L" will extract only the alpha channel from the "
 815                                 L"channel. This is useful when you have two SDI video cards, and neither has native support "
 816                                 L"for separate fill/key output")
 817                 ->item(L"channel_layout", L"If specified, overrides the audio channel layout used by the channel.");
 818         sink.para()->text(L"Examples:");
 819         sink.example(L">> ADD 1 DECKLINK", L"for using the default device_index of 1.");
 820         sink.example(L">> ADD 1 DECKLINK 2", L"uses device_index 2.");
 821         sink.example(L">> ADD 1 DECKLINK 1 EXTERNAL_KEY EMBEDDED_AUDIO");
 822         sink.example(
 823                         L">> ADD 1 DECKLINK 1 EMBEDDED_AUDIO\n"
 824                         L">> ADD 1 DECKLINK 2 KEY_ONLY", L"uses device with index 1 as fill output with audio and device with index 2 as key output.");
 825         sink.example(
 826                         L">> ADD 1 DECKLINK 1 EXTERNAL_SEPARATE_DEVICE_KEY EMBEDDED_AUDIO",
 827                         L"Uses device 2 for key output. May give better sync between key and fill than the previous method.");
 828 }
 829
 830 spl::shared_ptr<core::frame_consumer> create_consumer(
 831                 const std::vector<std::wstring>& params, core::interaction_sink*)
 832 {
 833         if (params.size() < 1 || !boost::iequals(params.at(0), L"DECKLINK"))
 834                 return core::frame_consumer::empty();
 835
 836         configuration config;
 837
 838         if (params.size() > 1)
 839                 config.device_index = boost::lexical_cast<int>(params.at(1));
 840
 841         if (contains_param(L"INTERNAL_KEY", params))
 842                 config.keyer = configuration::keyer_t::internal_keyer;
 843         else if (contains_param(L"EXTERNAL_KEY", params))
 844                 config.keyer = configuration::keyer_t::external_keyer;
 845         else if (contains_param(L"EXTERNAL_SEPARATE_DEVICE_KEY", params))
 846                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 847         else
 848                 config.keyer = configuration::keyer_t::default_keyer;
 849
 850         if (contains_param(L"LOW_LATENCY", params))
 851                 config.latency = configuration::latency_t::low_latency;
 852
 853         config.embedded_audio   = contains_param(L"EMBEDDED_AUDIO", params);
 854         config.key_only                 = contains_param(L"KEY_ONLY", params);
 855
 856         auto channel_layout = get_param(L"CHANNEL_LAYOUT", params);
 857
 858         if (!channel_layout.empty())
 859         {
 860                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(channel_layout);
 861
 862                 if (!found_layout)
 863                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + channel_layout + L" not found."));
 864
 865                 config.out_channel_layout = *found_layout;
 866         }
 867
 868         return spl::make_shared<decklink_consumer_proxy>(config);
 869 }
 870
 871 spl::shared_ptr<core::frame_consumer> create_preconfigured_consumer(
 872                 const boost::property_tree::wptree& ptree, core::interaction_sink*)
 873 {
 874         configuration config;
 875
 876         auto keyer = ptree.get(L"keyer", L"default");
 877         if(keyer == L"external")
 878                 config.keyer = configuration::keyer_t::external_keyer;
 879         else if(keyer == L"internal")
 880                 config.keyer = configuration::keyer_t::internal_keyer;
 881         else if (keyer == L"external_separate_device")
 882                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 883
 884         auto latency = ptree.get(L"latency", L"default");
 885         if(latency == L"low")
 886                 config.latency = configuration::latency_t::low_latency;
 887         else if(latency == L"normal")
 888                 config.latency = configuration::latency_t::normal_latency;
 889
 890         auto channel_layout = ptree.get_optional<std::wstring>(L"channel-layout");
 891
 892         if (channel_layout)
 893         {
 894                 CASPAR_SCOPED_CONTEXT_MSG("/channel-layout")
 895
 896                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(*channel_layout);
 897
 898                 if (!found_layout)
 899                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + *channel_layout + L" not found."));
 900
 901                 config.out_channel_layout = *found_layout;
 902         }
 903
 904         config.key_only                         = ptree.get(L"key-only",                config.key_only);
 905         config.device_index                     = ptree.get(L"device",                  config.device_index);
 906         config.key_device_idx           = ptree.get(L"key-device",              config.key_device_idx);
 907         config.embedded_audio           = ptree.get(L"embedded-audio",  config.embedded_audio);
 908         config.base_buffer_depth        = ptree.get(L"buffer-depth",    config.base_buffer_depth);
 909
 910         return spl::make_shared<decklink_consumer_proxy>(config);
 911 }
 912
 913 }}
 914
 915 /*
 916 ##############################################################################
 917 Pre-rolling
 918
 919 Mail: 2011-05-09
 920
 921 Yoshan
 922 BMD Developer Support
 923 developer@blackmagic-design.com
 924
 925 -----------------------------------------------------------------------------
 926
 927 Thanks for your inquiry. The minimum number of frames that you can preroll
 928 for scheduled playback is three frames for video and four frames for audio.
 929 As you mentioned if you preroll less frames then playback will not start or
 930 playback will be very sporadic. From our experience with Media Express, we
 931 recommended that at least seven frames are prerolled for smooth playback.
 932
 933 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
 934 There can be around 3 frames worth of latency on scheduled output.
 935 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
 936 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync()
 937 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
 938 guarantee that the provided frame will be output as soon the previous
 939 frame output has been completed.
 940 ################################################################################
 941 */
 942
 943 /*
 944 ##############################################################################
 945 Async DMA Transfer without redundant copying
 946
 947 Mail: 2011-05-10
 948
 949 Yoshan
 950 BMD Developer Support
 951 developer@blackmagic-design.com
 952
 953 -----------------------------------------------------------------------------
 954
 955 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
 956 and providing a pointer to your video buffer when GetBytes() is called.
 957 This may help to keep copying to a minimum. Please ensure that the pixel
 958 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
 959 have to colourspace convert which may result in additional copying.
 960 ################################################################################
 961 */