git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Robert Nagy, ronag89@gmail.com
  20 */
  21
  22 #include "../StdAfx.h"
  23
  24 #include "decklink_consumer.h"
  25
  26 #include "../util/util.h"
  27
  28 #include "../decklink_api.h"
  29
  30 #include <core/frame/frame.h>
  31 #include <core/frame/audio_channel_layout.h>
  32 #include <core/mixer/audio/audio_mixer.h>
  33 #include <core/consumer/frame_consumer.h>
  34 #include <core/diagnostics/call_context.h>
  35 #include <core/help/help_sink.h>
  36 #include <core/help/help_repository.h>
  37
  38 #include <common/executor.h>
  39 #include <common/lock.h>
  40 #include <common/diagnostics/graph.h>
  41 #include <common/except.h>
  42 #include <common/memshfl.h>
  43 #include <common/memcpy.h>
  44 #include <common/no_init_proxy.h>
  45 #include <common/array.h>
  46 #include <common/future.h>
  47 #include <common/cache_aligned_vector.h>
  48 #include <common/timer.h>
  49 #include <common/param.h>
  50
  51 #include <tbb/concurrent_queue.h>
  52
  53 #include <common/assert.h>
  54 #include <boost/lexical_cast.hpp>
  55 #include <boost/circular_buffer.hpp>
  56 #include <boost/property_tree/ptree.hpp>
  57
  58 namespace caspar { namespace decklink {
  59
  60 struct configuration
  61 {
  62         enum class keyer_t
  63         {
  64                 internal_keyer,
  65                 external_keyer,
  66                 external_separate_device_keyer,
  67                 default_keyer                                   = external_keyer
  68         };
  69
  70         enum class latency_t
  71         {
  72                 low_latency,
  73                 normal_latency,
  74                 default_latency = normal_latency
  75         };
  76
  77         int                                                     device_index            = 1;
  78         int                                                     key_device_idx          = 0;
  79         bool                                            embedded_audio          = true;
  80         keyer_t                                         keyer                           = keyer_t::default_keyer;
  81         latency_t                                       latency                         = latency_t::default_latency;
  82         bool                                            key_only                        = false;
  83         int                                                     base_buffer_depth       = 3;
  84         core::audio_channel_layout      out_channel_layout      = core::audio_channel_layout::invalid();
  85
  86         int buffer_depth() const
  87         {
  88                 return base_buffer_depth + (latency == latency_t::low_latency ? 0 : 1) + (embedded_audio ? 1 : 0);
  89         }
  90
  91         int key_device_index() const
  92         {
  93                 return key_device_idx == 0 ? device_index + 1 : key_device_idx;
  94         }
  95
  96         core::audio_channel_layout get_adjusted_layout(const core::audio_channel_layout& in_layout) const
  97         {
  98                 auto adjusted = out_channel_layout == core::audio_channel_layout::invalid() ? in_layout : out_channel_layout;
  99
 100                 if (adjusted.num_channels == 1) // Duplicate mono-signal into both left and right.
 101                 {
 102                         adjusted.num_channels = 2;
 103                         adjusted.channel_order.push_back(adjusted.channel_order.at(0)); // Usually FC -> FC FC
 104                 }
 105                 else if (adjusted.num_channels == 2)
 106                 {
 107                         adjusted.num_channels = 2;
 108                 }
 109                 else if (adjusted.num_channels <= 8)
 110                 {
 111                         adjusted.num_channels = 8;
 112                 }
 113                 else // Over 8 always pad to 16 or drop >16
 114                 {
 115                         adjusted.num_channels = 16;
 116                 }
 117
 118                 return adjusted;
 119         }
 120 };
 121
 122 void set_latency(
 123                 const com_iface_ptr<IDeckLinkConfiguration>& config,
 124                 configuration::latency_t latency,
 125                 const std::wstring& print)
 126 {
 127         if (latency == configuration::latency_t::low_latency)
 128         {
 129                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
 130                 CASPAR_LOG(info) << print << L" Enabled low-latency mode.";
 131         }
 132         else if (latency == configuration::latency_t::normal_latency)
 133         {
 134                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
 135                 CASPAR_LOG(info) << print << L" Disabled low-latency mode.";
 136         }
 137 }
 138
 139 void set_keyer(
 140                 const com_iface_ptr<IDeckLinkAttributes>& attributes,
 141                 const com_iface_ptr<IDeckLinkKeyer>& decklink_keyer,
 142                 configuration::keyer_t keyer,
 143                 const std::wstring& print)
 144 {
 145         if (keyer == configuration::keyer_t::internal_keyer)
 146         {
 147                 BOOL value = true;
 148                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
 149                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 150                 else if (FAILED(decklink_keyer->Enable(FALSE)))
 151                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 152                 else if (FAILED(decklink_keyer->SetLevel(255)))
 153                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 154                 else
 155                         CASPAR_LOG(info) << print << L" Enabled internal keyer.";
 156         }
 157         else if (keyer == configuration::keyer_t::external_keyer)
 158         {
 159                 BOOL value = true;
 160                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
 161                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 162                 else if (FAILED(decklink_keyer->Enable(TRUE)))
 163                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 164                 else if (FAILED(decklink_keyer->SetLevel(255)))
 165                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 166                 else
 167                         CASPAR_LOG(info) << print << L" Enabled external keyer.";
 168         }
 169 }
 170
 171 class decklink_frame : public IDeckLinkVideoFrame
 172 {
 173         tbb::atomic<int>                                                                ref_count_;
 174         core::const_frame                                                               frame_;
 175         const core::video_format_desc                                   format_desc_;
 176
 177         const bool                                                                              key_only_;
 178         cache_aligned_vector<no_init_proxy<uint8_t>>    data_;
 179 public:
 180         decklink_frame(core::const_frame frame, const core::video_format_desc& format_desc, bool key_only)
 181                 : frame_(frame)
 182                 , format_desc_(format_desc)
 183                 , key_only_(key_only)
 184         {
 185                 ref_count_ = 0;
 186         }
 187
 188         // IUnknown
 189
 190         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)
 191         {
 192                 return E_NOINTERFACE;
 193         }
 194
 195         virtual ULONG STDMETHODCALLTYPE AddRef()
 196         {
 197                 return ++ref_count_;
 198         }
 199
 200         virtual ULONG STDMETHODCALLTYPE Release()
 201         {
 202                 if(--ref_count_ == 0)
 203                         delete this;
 204                 return ref_count_;
 205         }
 206
 207         // IDecklinkVideoFrame
 208
 209         virtual long STDMETHODCALLTYPE GetWidth()                   {return static_cast<long>(format_desc_.width);}
 210         virtual long STDMETHODCALLTYPE GetHeight()                  {return static_cast<long>(format_desc_.height);}
 211         virtual long STDMETHODCALLTYPE GetRowBytes()                {return static_cast<long>(format_desc_.width*4);}
 212         virtual BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat()   {return bmdFormat8BitBGRA;}
 213         virtual BMDFrameFlags STDMETHODCALLTYPE GetFlags()                      {return bmdFrameFlagDefault;}
 214
 215         virtual HRESULT STDMETHODCALLTYPE GetBytes(void** buffer)
 216         {
 217                 try
 218                 {
 219                         if(static_cast<int>(frame_.image_data().size()) != format_desc_.size)
 220                         {
 221                                 data_.resize(format_desc_.size);
 222                                 *buffer = data_.data();
 223                         }
 224                         else if(key_only_)
 225                         {
 226                                 if(data_.empty())
 227                                 {
 228                                         data_.resize(frame_.image_data().size());
 229                                         aligned_memshfl(data_.data(), frame_.image_data().begin(), frame_.image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
 230                                 }
 231                                 *buffer = data_.data();
 232                         }
 233                         else
 234                         {
 235                                 *buffer = const_cast<uint8_t*>(frame_.image_data().begin());
 236
 237 #if !defined(_MSC_VER)
 238                                 // On Linux Decklink cannot DMA transfer from memory returned by glMapBuffer
 239                                 data_.resize(frame_.image_data().size());
 240                                 fast_memcpy(data_.data(), *buffer, frame_.image_data().size());
 241                                 *buffer = data_.data();
 242 #endif
 243                         }
 244                 }
 245                 catch(...)
 246                 {
 247                         CASPAR_LOG_CURRENT_EXCEPTION();
 248                         return E_FAIL;
 249                 }
 250
 251                 return S_OK;
 252         }
 253
 254         virtual HRESULT STDMETHODCALLTYPE GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode) {return S_FALSE;}
 255         virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)              {return S_FALSE;}
 256
 257         // decklink_frame
 258
 259         const core::audio_buffer& audio_data()
 260         {
 261                 return frame_.audio_data();
 262         }
 263
 264         int64_t get_age_millis() const
 265         {
 266                 return frame_.get_age_millis();
 267         }
 268 };
 269
 270 struct key_video_context : public IDeckLinkVideoOutputCallback, boost::noncopyable
 271 {
 272         const configuration                                                                             config_;
 273         com_ptr<IDeckLink>                                                                              decklink_                                       = get_device(config_.key_device_index());
 274         com_iface_ptr<IDeckLinkOutput>                                                  output_                                         = iface_cast<IDeckLinkOutput>(decklink_);
 275         com_iface_ptr<IDeckLinkKeyer>                                                   keyer_                                          = iface_cast<IDeckLinkKeyer>(decklink_);
 276         com_iface_ptr<IDeckLinkAttributes>                                              attributes_                                     = iface_cast<IDeckLinkAttributes>(decklink_);
 277         com_iface_ptr<IDeckLinkConfiguration>                                   configuration_                          = iface_cast<IDeckLinkConfiguration>(decklink_);
 278         tbb::atomic<int64_t>                                                                    current_presentation_delay_;
 279         tbb::atomic<int64_t>                                                                    scheduled_frames_completed_;
 280
 281         key_video_context(const configuration& config, const std::wstring& print)
 282                 : config_(config)
 283         {
 284                 current_presentation_delay_ = 0;
 285                 scheduled_frames_completed_ = 0;
 286
 287                 set_latency(configuration_, config.latency, print);
 288                 set_keyer(attributes_, keyer_, config.keyer, print);
 289
 290                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 291                         CASPAR_THROW_EXCEPTION(caspar_exception()
 292                                         << msg_info(print + L" Failed to set key playback completion callback.")
 293                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 294         }
 295
 296         template<typename Print>
 297         void enable_video(BMDDisplayMode display_mode, const Print& print)
 298         {
 299                 if (FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 300                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable key video output."));
 301
 302                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 303                         CASPAR_THROW_EXCEPTION(caspar_exception()
 304                                         << msg_info(print() + L" Failed to set key playback completion callback.")
 305                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 306         }
 307
 308         virtual ~key_video_context()
 309         {
 310                 if (output_)
 311                 {
 312                         output_->StopScheduledPlayback(0, nullptr, 0);
 313                         output_->DisableVideoOutput();
 314                 }
 315         }
 316
 317         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 318         virtual ULONG STDMETHODCALLTYPE AddRef()                                                        {return 1;}
 319         virtual ULONG STDMETHODCALLTYPE Release()                                                       {return 1;}
 320
 321         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 322         {
 323                 return S_OK;
 324         }
 325
 326         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(
 327                         IDeckLinkVideoFrame* completed_frame,
 328                         BMDOutputFrameCompletionResult result)
 329         {
 330                 auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 331                 current_presentation_delay_ = dframe->get_age_millis();
 332                 ++scheduled_frames_completed_;
 333
 334                 // Let the fill callback keep the pace, so no scheduling here.
 335
 336                 return S_OK;
 337         }
 338 };
 339
 340 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
 341 {
 342         const int                                                                                       channel_index_;
 343         const configuration                                                                     config_;
 344
 345         com_ptr<IDeckLink>                                                                      decklink_                               = get_device(config_.device_index);
 346         com_iface_ptr<IDeckLinkOutput>                                          output_                                 = iface_cast<IDeckLinkOutput>(decklink_);
 347         com_iface_ptr<IDeckLinkConfiguration>                           configuration_                  = iface_cast<IDeckLinkConfiguration>(decklink_);
 348         com_iface_ptr<IDeckLinkKeyer>                                           keyer_                                  = iface_cast<IDeckLinkKeyer>(decklink_);
 349         com_iface_ptr<IDeckLinkAttributes>                                      attributes_                             = iface_cast<IDeckLinkAttributes>(decklink_);
 350
 351         tbb::spin_mutex                                                                         exception_mutex_;
 352         std::exception_ptr                                                                      exception_;
 353
 354         tbb::atomic<bool>                                                                       is_running_;
 355
 356         const std::wstring                                                                      model_name_                             = get_model_name(decklink_);
 357         const core::video_format_desc                                           format_desc_;
 358         const core::audio_channel_layout                                        in_channel_layout_;
 359         const core::audio_channel_layout                                        out_channel_layout_             = config_.get_adjusted_layout(in_channel_layout_);
 360         core::audio_channel_remapper                                            channel_remapper_               { in_channel_layout_, out_channel_layout_ };
 361         const int                                                                                       buffer_size_                    = config_.buffer_depth(); // Minimum buffer-size 3.
 362
 363         long long                                                                                       video_scheduled_                = 0;
 364         long long                                                                                       audio_scheduled_                = 0;
 365
 366         int                                                                                                     preroll_count_                  = 0;
 367
 368         boost::circular_buffer<std::vector<int32_t>>            audio_container_                { buffer_size_ + 1 };
 369
 370         tbb::concurrent_bounded_queue<core::const_frame>        video_frame_buffer_;
 371         tbb::concurrent_bounded_queue<core::const_frame>        audio_frame_buffer_;
 372
 373         spl::shared_ptr<diagnostics::graph>                                     graph_;
 374         caspar::timer                                                                           tick_timer_;
 375         retry_task<bool>                                                                        send_completion_;
 376         reference_signal_detector                                                       reference_signal_detector_      { output_ };
 377         tbb::atomic<int64_t>                                                            current_presentation_delay_;
 378         tbb::atomic<int64_t>                                                            scheduled_frames_completed_;
 379         std::unique_ptr<key_video_context>                                      key_context_;
 380
 381 public:
 382         decklink_consumer(
 383                         const configuration& config,
 384                         const core::video_format_desc& format_desc,
 385                         const core::audio_channel_layout& in_channel_layout,
 386                         int channel_index)
 387                 : channel_index_(channel_index)
 388                 , config_(config)
 389                 , format_desc_(format_desc)
 390                 , in_channel_layout_(in_channel_layout)
 391         {
 392                 is_running_ = true;
 393                 current_presentation_delay_ = 0;
 394                 scheduled_frames_completed_ = 0;
 395
 396                 video_frame_buffer_.set_capacity(1);
 397
 398                 // Blackmagic calls RenderAudioSamples() 50 times per second
 399                 // regardless of video mode so we sometimes need to give them
 400                 // samples from 2 frames in order to keep up
 401                 audio_frame_buffer_.set_capacity((format_desc.fps > 50.0) ? 2 : 1);
 402
 403                 if (config.keyer == configuration::keyer_t::external_separate_device_keyer)
 404                         key_context_.reset(new key_video_context(config, print()));
 405
 406                 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
 407                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
 408                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
 409                 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
 410                 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
 411                 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
 412
 413                 if (key_context_)
 414                 {
 415                         graph_->set_color("key-offset", diagnostics::color(1.0f, 0.0f, 0.0f));
 416                 }
 417
 418                 graph_->set_text(print());
 419                 diagnostics::register_graph(graph_);
 420
 421                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));
 422
 423                 if(config.embedded_audio)
 424                         enable_audio();
 425
 426                 set_latency(configuration_, config.latency, print());
 427                 set_keyer(attributes_, keyer_, config.keyer, print());
 428
 429                 if(config.embedded_audio)
 430                         output_->BeginAudioPreroll();
 431
 432                 for(int n = 0; n < buffer_size_; ++n)
 433                         schedule_next_video(core::const_frame::empty());
 434
 435                 if(!config.embedded_audio)
 436                         start_playback();
 437         }
 438
 439         ~decklink_consumer()
 440         {
 441                 is_running_ = false;
 442                 video_frame_buffer_.try_push(core::const_frame::empty());
 443                 audio_frame_buffer_.try_push(core::const_frame::empty());
 444
 445                 if(output_ != nullptr)
 446                 {
 447                         output_->StopScheduledPlayback(0, nullptr, 0);
 448                         if(config_.embedded_audio)
 449                                 output_->DisableAudioOutput();
 450                         output_->DisableVideoOutput();
 451                 }
 452         }
 453
 454         void enable_audio()
 455         {
 456                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, out_channel_layout_.num_channels, bmdAudioOutputStreamTimestamped)))
 457                                 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable audio output."));
 458
 459                 if(FAILED(output_->SetAudioCallback(this)))
 460                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not set audio callback."));
 461
 462                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
 463         }
 464
 465         void enable_video(BMDDisplayMode display_mode)
 466         {
 467                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 468                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable fill video output."));
 469
 470                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 471                         CASPAR_THROW_EXCEPTION(caspar_exception()
 472                                                                         << msg_info(print() + L" Failed to set fill playback completion callback.")
 473                                                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 474
 475                 if (key_context_)
 476                         key_context_->enable_video(display_mode, [this]() { return print(); });
 477         }
 478
 479         void start_playback()
 480         {
 481                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 482                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule fill playback."));
 483
 484                 if (key_context_ && FAILED(key_context_->output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 485                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule key playback."));
 486         }
 487
 488         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 489         virtual ULONG STDMETHODCALLTYPE AddRef()                                        {return 1;}
 490         virtual ULONG STDMETHODCALLTYPE Release()                               {return 1;}
 491
 492         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 493         {
 494                 is_running_ = false;
 495                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
 496                 return S_OK;
 497         }
 498
 499         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result)
 500         {
 501                 if(!is_running_)
 502                         return E_FAIL;
 503
 504                 try
 505                 {
 506                         auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 507                         current_presentation_delay_ = dframe->get_age_millis();
 508                         ++scheduled_frames_completed_;
 509
 510                         if (key_context_)
 511                                 graph_->set_value(
 512                                                 "key-offset",
 513                                                 static_cast<double>(
 514                                                                 scheduled_frames_completed_
 515                                                                 - key_context_->scheduled_frames_completed_)
 516                                                 * 0.1 + 0.5);
 517
 518                         if(result == bmdOutputFrameDisplayedLate)
 519                         {
 520                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "late-frame");
 521                                 video_scheduled_ += format_desc_.duration;
 522                                 audio_scheduled_ += dframe->audio_data().size() / out_channel_layout_.num_channels;
 523                                 //++video_scheduled_;
 524                                 //audio_scheduled_ += format_desc_.audio_cadence[0];
 525                                 //++audio_scheduled_;
 526                         }
 527                         else if(result == bmdOutputFrameDropped)
 528                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "dropped-frame");
 529                         else if(result == bmdOutputFrameFlushed)
 530                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "flushed-frame");
 531
 532                         UINT32 buffered;
 533                         output_->GetBufferedVideoFrameCount(&buffered);
 534                         graph_->set_value("buffered-video", static_cast<double>(buffered) / (config_.buffer_depth()));
 535
 536                         auto frame = core::const_frame::empty();
 537                         video_frame_buffer_.pop(frame);
 538                         send_completion_.try_completion();
 539                         schedule_next_video(frame);
 540                 }
 541                 catch(...)
 542                 {
 543                         lock(exception_mutex_, [&]
 544                         {
 545                                 exception_ = std::current_exception();
 546                         });
 547                         return E_FAIL;
 548                 }
 549
 550                 return S_OK;
 551         }
 552
 553         virtual HRESULT STDMETHODCALLTYPE RenderAudioSamples(BOOL preroll)
 554         {
 555                 if(!is_running_)
 556                         return E_FAIL;
 557
 558                 try
 559                 {
 560                         if(preroll)
 561                         {
 562                                 if(++preroll_count_ >= buffer_size_)
 563                                 {
 564                                         output_->EndAudioPreroll();
 565                                         start_playback();
 566                                 }
 567                                 else
 568                                 {
 569                                         schedule_next_audio(core::mutable_audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()] * out_channel_layout_.num_channels, 0));
 570                                 }
 571                         }
 572                         else
 573                         {
 574                                 auto frame = core::const_frame::empty();
 575
 576                                 while(audio_frame_buffer_.try_pop(frame))
 577                                 {
 578                                         UINT32 buffered;
 579                                         output_->GetBufferedAudioSampleFrameCount(&buffered);
 580                                         graph_->set_value("buffered-audio", static_cast<double>(buffered) / (format_desc_.audio_cadence[0] * config_.buffer_depth()));
 581
 582                                         send_completion_.try_completion();
 583                                         schedule_next_audio(channel_remapper_.mix_and_rearrange(frame.audio_data()));
 584                                 }
 585                         }
 586                 }
 587                 catch(...)
 588                 {
 589                         tbb::spin_mutex::scoped_lock lock(exception_mutex_);
 590                         exception_ = std::current_exception();
 591                         return E_FAIL;
 592                 }
 593
 594                 return S_OK;
 595         }
 596
 597         template<typename T>
 598         void schedule_next_audio(const T& audio_data)
 599         {
 600                 auto sample_frame_count = static_cast<int>(audio_data.size()/out_channel_layout_.num_channels);
 601
 602                 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
 603
 604                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
 605                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
 606
 607                 audio_scheduled_ += sample_frame_count;
 608         }
 609
 610         void schedule_next_video(core::const_frame frame)
 611         {
 612                 if (key_context_)
 613                 {
 614                         auto key_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, true));
 615                         if (FAILED(key_context_->output_->ScheduleVideoFrame(get_raw(key_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 616                                 CASPAR_LOG(error) << print() << L" Failed to schedule key video.";
 617                 }
 618
 619                 auto fill_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, config_.key_only));
 620                 if (FAILED(output_->ScheduleVideoFrame(get_raw(fill_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 621                         CASPAR_LOG(error) << print() << L" Failed to schedule fill video.";
 622
 623                 video_scheduled_ += format_desc_.duration;
 624
 625                 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
 626                 tick_timer_.restart();
 627
 628                 reference_signal_detector_.detect_change([this]() { return print(); });
 629         }
 630
 631         std::future<bool> send(core::const_frame frame)
 632         {
 633                 auto exception = lock(exception_mutex_, [&]
 634                 {
 635                         return exception_;
 636                 });
 637
 638                 if(exception != nullptr)
 639                         std::rethrow_exception(exception);
 640
 641                 if(!is_running_)
 642                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Is not running."));
 643
 644                 bool audio_ready = !config_.embedded_audio;
 645                 bool video_ready = false;
 646
 647                 auto enqueue_task = [audio_ready, video_ready, frame, this]() mutable -> boost::optional<bool>
 648                 {
 649                         if (!audio_ready)
 650                                 audio_ready = audio_frame_buffer_.try_push(frame);
 651
 652                         if (!video_ready)
 653                                 video_ready = video_frame_buffer_.try_push(frame);
 654
 655                         if (audio_ready && video_ready)
 656                                 return true;
 657                         else
 658                                 return boost::optional<bool>();
 659                 };
 660
 661                 if (enqueue_task())
 662                         return make_ready_future(true);
 663
 664                 send_completion_.set_task(enqueue_task);
 665
 666                 return send_completion_.get_future();
 667         }
 668
 669         std::wstring print() const
 670         {
 671                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 672                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 673                                 boost::lexical_cast<std::wstring>(config_.device_index) +
 674                                 L"&&" +
 675                                 boost::lexical_cast<std::wstring>(config_.key_device_index()) +
 676                                 L"|" +
 677                                 format_desc_.name + L"]";
 678                 else
 679                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 680                                 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
 681         }
 682 };
 683
 684 struct decklink_consumer_proxy : public core::frame_consumer
 685 {
 686         core::monitor::subject                          monitor_subject_;
 687         const configuration                                     config_;
 688         std::unique_ptr<decklink_consumer>      consumer_;
 689         core::video_format_desc                         format_desc_;
 690         executor                                                        executor_;
 691 public:
 692
 693         decklink_consumer_proxy(const configuration& config)
 694                 : config_(config)
 695                 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
 696         {
 697                 auto ctx = core::diagnostics::call_context::for_thread();
 698                 executor_.begin_invoke([=]
 699                 {
 700                         core::diagnostics::call_context::for_thread() = ctx;
 701                         com_initialize();
 702                 });
 703         }
 704
 705         ~decklink_consumer_proxy()
 706         {
 707                 executor_.invoke([=]
 708                 {
 709                         consumer_.reset();
 710                         com_uninitialize();
 711                 });
 712         }
 713
 714         // frame_consumer
 715
 716         void initialize(const core::video_format_desc& format_desc, const core::audio_channel_layout& channel_layout, int channel_index) override
 717         {
 718                 format_desc_ = format_desc;
 719                 executor_.invoke([=]
 720                 {
 721                         consumer_.reset();
 722                         consumer_.reset(new decklink_consumer(config_, format_desc, channel_layout, channel_index));
 723                 });
 724         }
 725
 726         std::future<bool> send(core::const_frame frame) override
 727         {
 728                 return consumer_->send(frame);
 729         }
 730
 731         std::wstring print() const override
 732         {
 733                 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
 734         }
 735
 736         std::wstring name() const override
 737         {
 738                 return L"decklink";
 739         }
 740
 741         boost::property_tree::wptree info() const override
 742         {
 743                 boost::property_tree::wptree info;
 744                 info.add(L"type", L"decklink");
 745                 info.add(L"key-only", config_.key_only);
 746                 info.add(L"device", config_.device_index);
 747
 748                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 749                 {
 750                         info.add(L"key-device", config_.key_device_index());
 751                 }
 752
 753                 info.add(L"low-latency", config_.latency == configuration::latency_t::low_latency);
 754                 info.add(L"embedded-audio", config_.embedded_audio);
 755                 info.add(L"presentation-frame-age", presentation_frame_age_millis());
 756                 //info.add(L"internal-key", config_.internal_key);
 757                 return info;
 758         }
 759
 760         int buffer_depth() const override
 761         {
 762                 return config_.buffer_depth() + 2;
 763         }
 764
 765         int index() const override
 766         {
 767                 return 300 + config_.device_index;
 768         }
 769
 770         int64_t presentation_frame_age_millis() const override
 771         {
 772                 return consumer_ ? static_cast<int64_t>(consumer_->current_presentation_delay_) : 0;
 773         }
 774
 775         core::monitor::subject& monitor_output()
 776         {
 777                 return monitor_subject_;
 778         }
 779 };
 780
 781 void describe_consumer(core::help_sink& sink, const core::help_repository& repo)
 782 {
 783         sink.short_description(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 784         sink.syntax(L"DECKLINK "
 785                                 L"{[device_index:int]|1} "
 786                                 L"{[keyer:INTERNAL_KEY,EXTERNAL_KEY,EXTERNAL_SEPARATE_DEVICE_KEY]} "
 787                                 L"{[low_latency:LOW_LATENCY]} "
 788                                 L"{[embedded_audio:EMBEDDED_AUDIO]} "
 789                                 L"{[key_only:KEY_ONLY]} "
 790                                 L"{CHANNEL_LAYOUT [channel_layout:string]}");
 791         sink.para()->text(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 792         sink.definitions()
 793                 ->item(L"device_index", L"The Blackmagic video card to use (See Blackmagic control panel for card order). Default is 1.")
 794                 ->item(L"keyer",
 795                                 L"If given tries to enable either internal or external keying. Not all Blackmagic cards supports this. "
 796                                 L"There is also a third experimental option (EXTERNAL_SEPARATE_DEVICE_KEY) which allocates device_index + 1 for synhronized key output.")
 797                 ->item(L"low_latency", L"Tries to enable low latency if given.")
 798                 ->item(L"embedded_audio", L"Embeds the audio into the SDI signal if given.")
 799                 ->item(L"key_only",
 800                                 L" will extract only the alpha channel from the "
 801                                 L"channel. This is useful when you have two SDI video cards, and neither has native support "
 802                                 L"for separate fill/key output")
 803                 ->item(L"channel_layout", L"If specified, overrides the audio channel layout used by the channel.");
 804         sink.para()->text(L"Examples:");
 805         sink.example(L">> ADD 1 DECKLINK", L"for using the default device_index of 1.");
 806         sink.example(L">> ADD 1 DECKLINK 2", L"uses device_index 2.");
 807         sink.example(L">> ADD 1 DECKLINK 1 EXTERNAL_KEY EMBEDDED_AUDIO");
 808         sink.example(
 809                         L">> ADD 1 DECKLINK 1 EMBEDDED_AUDIO\n"
 810                         L">> ADD 1 DECKLINK 2 KEY_ONLY", L"uses device with index 1 as fill output with audio and device with index 2 as key output.");
 811         sink.example(
 812                         L">> ADD 1 DECKLINK 1 EXTERNAL_SEPARATE_DEVICE_KEY EMBEDDED_AUDIO",
 813                         L"Uses device 2 for key output. May give better sync between key and fill than the previous method.");
 814 }
 815
 816 spl::shared_ptr<core::frame_consumer> create_consumer(
 817                 const std::vector<std::wstring>& params, core::interaction_sink*)
 818 {
 819         if (params.size() < 1 || !boost::iequals(params.at(0), L"DECKLINK"))
 820                 return core::frame_consumer::empty();
 821
 822         configuration config;
 823
 824         if (params.size() > 1)
 825                 config.device_index = boost::lexical_cast<int>(params.at(1));
 826
 827         if (contains_param(L"INTERNAL_KEY", params))
 828                 config.keyer = configuration::keyer_t::internal_keyer;
 829         else if (contains_param(L"EXTERNAL_KEY", params))
 830                 config.keyer = configuration::keyer_t::external_keyer;
 831         else if (contains_param(L"EXTERNAL_SEPARATE_DEVICE_KEY", params))
 832                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 833         else
 834                 config.keyer = configuration::keyer_t::default_keyer;
 835
 836         if (contains_param(L"LOW_LATENCY", params))
 837                 config.latency = configuration::latency_t::low_latency;
 838
 839         config.embedded_audio   = contains_param(L"EMBEDDED_AUDIO", params);
 840         config.key_only                 = contains_param(L"KEY_ONLY", params);
 841
 842         auto channel_layout = get_param(L"CHANNEL_LAYOUT", params);
 843
 844         if (!channel_layout.empty())
 845         {
 846                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(channel_layout);
 847
 848                 if (!found_layout)
 849                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + channel_layout + L" not found."));
 850
 851                 config.out_channel_layout = *found_layout;
 852         }
 853
 854         return spl::make_shared<decklink_consumer_proxy>(config);
 855 }
 856
 857 spl::shared_ptr<core::frame_consumer> create_preconfigured_consumer(
 858                 const boost::property_tree::wptree& ptree, core::interaction_sink*)
 859 {
 860         configuration config;
 861
 862         auto keyer = ptree.get(L"keyer", L"default");
 863         if(keyer == L"external")
 864                 config.keyer = configuration::keyer_t::external_keyer;
 865         else if(keyer == L"internal")
 866                 config.keyer = configuration::keyer_t::internal_keyer;
 867         else if (keyer == L"external_separate_device")
 868                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 869
 870         auto latency = ptree.get(L"latency", L"default");
 871         if(latency == L"low")
 872                 config.latency = configuration::latency_t::low_latency;
 873         else if(latency == L"normal")
 874                 config.latency = configuration::latency_t::normal_latency;
 875
 876         auto channel_layout = ptree.get_optional<std::wstring>(L"channel-layout");
 877
 878         if (channel_layout)
 879         {
 880                 CASPAR_SCOPED_CONTEXT_MSG("/channel-layout")
 881
 882                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(*channel_layout);
 883
 884                 if (!found_layout)
 885                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + *channel_layout + L" not found."));
 886
 887                 config.out_channel_layout = *found_layout;
 888         }
 889
 890         config.key_only                         = ptree.get(L"key-only",                config.key_only);
 891         config.device_index                     = ptree.get(L"device",                  config.device_index);
 892         config.key_device_idx           = ptree.get(L"key-device",              config.key_device_idx);
 893         config.embedded_audio           = ptree.get(L"embedded-audio",  config.embedded_audio);
 894         config.base_buffer_depth        = ptree.get(L"buffer-depth",    config.base_buffer_depth);
 895
 896         return spl::make_shared<decklink_consumer_proxy>(config);
 897 }
 898
 899 }}
 900
 901 /*
 902 ##############################################################################
 903 Pre-rolling
 904
 905 Mail: 2011-05-09
 906
 907 Yoshan
 908 BMD Developer Support
 909 developer@blackmagic-design.com
 910
 911 -----------------------------------------------------------------------------
 912
 913 Thanks for your inquiry. The minimum number of frames that you can preroll
 914 for scheduled playback is three frames for video and four frames for audio.
 915 As you mentioned if you preroll less frames then playback will not start or
 916 playback will be very sporadic. From our experience with Media Express, we
 917 recommended that at least seven frames are prerolled for smooth playback.
 918
 919 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
 920 There can be around 3 frames worth of latency on scheduled output.
 921 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
 922 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync()
 923 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
 924 guarantee that the provided frame will be output as soon the previous
 925 frame output has been completed.
 926 ################################################################################
 927 */
 928
 929 /*
 930 ##############################################################################
 931 Async DMA Transfer without redundant copying
 932
 933 Mail: 2011-05-10
 934
 935 Yoshan
 936 BMD Developer Support
 937 developer@blackmagic-design.com
 938
 939 -----------------------------------------------------------------------------
 940
 941 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
 942 and providing a pointer to your video buffer when GetBytes() is called.
 943 This may help to keep copying to a minimum. Please ensure that the pixel
 944 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
 945 have to colourspace convert which may result in additional copying.
 946 ################################################################################
 947 */