git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Robert Nagy, ronag89@gmail.com
  20 */
  21
  22 #include "../StdAfx.h"
  23
  24 #include "decklink_consumer.h"
  25
  26 #include "../util/util.h"
  27
  28 #include "../decklink_api.h"
  29
  30 #include <core/frame/frame.h>
  31 #include <core/mixer/audio/audio_mixer.h>
  32 #include <core/consumer/frame_consumer.h>
  33 #include <core/diagnostics/call_context.h>
  34 #include <core/help/help_sink.h>
  35 #include <core/help/help_repository.h>
  36
  37 #include <common/executor.h>
  38 #include <common/lock.h>
  39 #include <common/diagnostics/graph.h>
  40 #include <common/except.h>
  41 #include <common/memshfl.h>
  42 #include <common/array.h>
  43 #include <common/future.h>
  44 #include <common/cache_aligned_vector.h>
  45 #include <common/timer.h>
  46 #include <common/param.h>
  47
  48 #include <tbb/concurrent_queue.h>
  49
  50 #include <common/assert.h>
  51 #include <boost/lexical_cast.hpp>
  52 #include <boost/circular_buffer.hpp>
  53 #include <boost/property_tree/ptree.hpp>
  54
  55 namespace caspar { namespace decklink {
  56
  57 struct configuration
  58 {
  59         enum class keyer_t
  60         {
  61                 internal_keyer,
  62                 external_keyer,
  63                 external_separate_device_keyer,
  64                 default_keyer
  65         };
  66
  67         enum class latency_t
  68         {
  69                 low_latency,
  70                 normal_latency,
  71                 default_latency
  72         };
  73
  74         int                     device_index            = 1;
  75         int                     key_device_idx          = 0;
  76         bool            embedded_audio          = true;
  77         keyer_t         keyer                           = keyer_t::default_keyer;
  78         latency_t       latency                         = latency_t::default_latency;
  79         bool            key_only                        = false;
  80         int                     base_buffer_depth       = 3;
  81
  82         int buffer_depth() const
  83         {
  84                 return base_buffer_depth + (latency == latency_t::low_latency ? 0 : 1) + (embedded_audio ? 1 : 0);
  85         }
  86
  87         int key_device_index() const
  88         {
  89                 return key_device_idx == 0 ? device_index + 1 : key_device_idx;
  90         }
  91 };
  92
  93 void set_latency(
  94                 const com_iface_ptr<IDeckLinkConfiguration>& config,
  95                 configuration::latency_t latency,
  96                 const std::wstring& print)
  97 {
  98         if (latency == configuration::latency_t::low_latency)
  99         {
 100                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
 101                 CASPAR_LOG(info) << print << L" Enabled low-latency mode.";
 102         }
 103         else if (latency == configuration::latency_t::normal_latency)
 104         {
 105                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
 106                 CASPAR_LOG(info) << print << L" Disabled low-latency mode.";
 107         }
 108 }
 109
 110 void set_keyer(
 111                 const com_iface_ptr<IDeckLinkAttributes>& attributes,
 112                 const com_iface_ptr<IDeckLinkKeyer>& decklink_keyer,
 113                 configuration::keyer_t keyer,
 114                 const std::wstring& print)
 115 {
 116         if (keyer == configuration::keyer_t::internal_keyer
 117                         || keyer == configuration::keyer_t::external_separate_device_keyer)
 118         {
 119                 BOOL value = true;
 120                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
 121                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 122                 else if (FAILED(decklink_keyer->Enable(FALSE)))
 123                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 124                 else if (FAILED(decklink_keyer->SetLevel(255)))
 125                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 126                 else
 127                         CASPAR_LOG(info) << print << L" Enabled internal keyer.";
 128         }
 129         else if (keyer == configuration::keyer_t::external_keyer)
 130         {
 131                 BOOL value = true;
 132                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
 133                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 134                 else if (FAILED(decklink_keyer->Enable(TRUE)))
 135                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 136                 else if (FAILED(decklink_keyer->SetLevel(255)))
 137                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 138                 else
 139                         CASPAR_LOG(info) << print << L" Enabled external keyer.";
 140         }
 141 }
 142
 143 class decklink_frame : public IDeckLinkVideoFrame
 144 {
 145         tbb::atomic<int>                                ref_count_;
 146         core::const_frame                               frame_;
 147         const core::video_format_desc   format_desc_;
 148
 149         const bool                                              key_only_;
 150         cache_aligned_vector<uint8_t>   data_;
 151 public:
 152         decklink_frame(core::const_frame frame, const core::video_format_desc& format_desc, bool key_only)
 153                 : frame_(frame)
 154                 , format_desc_(format_desc)
 155                 , key_only_(key_only)
 156         {
 157                 ref_count_ = 0;
 158         }
 159
 160         // IUnknown
 161
 162         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)
 163         {
 164                 return E_NOINTERFACE;
 165         }
 166
 167         virtual ULONG STDMETHODCALLTYPE AddRef()
 168         {
 169                 return ++ref_count_;
 170         }
 171
 172         virtual ULONG STDMETHODCALLTYPE Release()
 173         {
 174                 if(--ref_count_ == 0)
 175                         delete this;
 176                 return ref_count_;
 177         }
 178
 179         // IDecklinkVideoFrame
 180
 181         virtual long STDMETHODCALLTYPE GetWidth()                   {return static_cast<long>(format_desc_.width);}
 182         virtual long STDMETHODCALLTYPE GetHeight()                  {return static_cast<long>(format_desc_.height);}
 183         virtual long STDMETHODCALLTYPE GetRowBytes()                {return static_cast<long>(format_desc_.width*4);}
 184         virtual BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat()   {return bmdFormat8BitBGRA;}
 185         virtual BMDFrameFlags STDMETHODCALLTYPE GetFlags()                      {return bmdFrameFlagDefault;}
 186
 187         virtual HRESULT STDMETHODCALLTYPE GetBytes(void** buffer)
 188         {
 189                 try
 190                 {
 191                         if(static_cast<int>(frame_.image_data().size()) != format_desc_.size)
 192                         {
 193                                 data_.resize(format_desc_.size, 0);
 194                                 *buffer = data_.data();
 195                         }
 196                         else if(key_only_)
 197                         {
 198                                 if(data_.empty())
 199                                 {
 200                                         data_.resize(frame_.image_data().size());
 201                                         aligned_memshfl(data_.data(), frame_.image_data().begin(), frame_.image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
 202                                 }
 203                                 *buffer = data_.data();
 204                         }
 205                         else
 206                                 *buffer = const_cast<uint8_t*>(frame_.image_data().begin());
 207                 }
 208                 catch(...)
 209                 {
 210                         CASPAR_LOG_CURRENT_EXCEPTION();
 211                         return E_FAIL;
 212                 }
 213
 214                 return S_OK;
 215         }
 216
 217         virtual HRESULT STDMETHODCALLTYPE GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode) {return S_FALSE;}
 218         virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)              {return S_FALSE;}
 219
 220         // decklink_frame
 221
 222         const core::audio_buffer& audio_data()
 223         {
 224                 return frame_.audio_data();
 225         }
 226
 227         int64_t get_age_millis() const
 228         {
 229                 return frame_.get_age_millis();
 230         }
 231 };
 232
 233 struct key_video_context : public IDeckLinkVideoOutputCallback, boost::noncopyable
 234 {
 235         const configuration                                                                             config_;
 236         com_ptr<IDeckLink>                                                                              decklink_                                       = get_device(config_.key_device_index());
 237         com_iface_ptr<IDeckLinkOutput>                                                  output_                                         = iface_cast<IDeckLinkOutput>(decklink_);
 238         com_iface_ptr<IDeckLinkKeyer>                                                   keyer_                                          = iface_cast<IDeckLinkKeyer>(decklink_);
 239         com_iface_ptr<IDeckLinkAttributes>                                              attributes_                                     = iface_cast<IDeckLinkAttributes>(decklink_);
 240         com_iface_ptr<IDeckLinkConfiguration>                                   configuration_                          = iface_cast<IDeckLinkConfiguration>(decklink_);
 241         tbb::atomic<int64_t>                                                                    current_presentation_delay_;
 242         tbb::atomic<int64_t>                                                                    scheduled_frames_completed_;
 243
 244         key_video_context(const configuration& config, const std::wstring& print)
 245                 : config_(config)
 246                 , output_(decklink_)
 247                 , keyer_(decklink_)
 248                 , attributes_(decklink_)
 249                 , configuration_(decklink_)
 250         {
 251                 current_presentation_delay_ = 0;
 252                 scheduled_frames_completed_ = 0;
 253
 254                 set_latency(configuration_, config.latency, print);
 255                 set_keyer(attributes_, keyer_, config.keyer, print);
 256
 257                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 258                         CASPAR_THROW_EXCEPTION(caspar_exception()
 259                                         << msg_info(u8(print) + " Failed to set key playback completion callback.")
 260                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 261         }
 262
 263         template<typename Print>
 264         void enable_video(BMDDisplayMode display_mode, const Print& print)
 265         {
 266                 if (FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 267                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable key video output."));
 268
 269                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 270                         CASPAR_THROW_EXCEPTION(caspar_exception()
 271                                         << msg_info(u8(print()) + " Failed to set key playback completion callback.")
 272                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 273         }
 274
 275         virtual ~key_video_context()
 276         {
 277                 if (output_)
 278                 {
 279                         output_->StopScheduledPlayback(0, nullptr, 0);
 280                         output_->DisableVideoOutput();
 281                 }
 282         }
 283
 284         STDMETHOD(QueryInterface(REFIID, LPVOID*))      { return E_NOINTERFACE; }
 285         STDMETHOD_(ULONG, AddRef())                                     { return 1; }
 286         STDMETHOD_(ULONG, Release())                            { return 1; }
 287
 288         STDMETHOD(ScheduledPlaybackHasStopped())
 289         {
 290                 return S_OK;
 291         }
 292
 293         STDMETHOD(ScheduledFrameCompleted(
 294                         IDeckLinkVideoFrame* completed_frame,
 295                         BMDOutputFrameCompletionResult result))
 296         {
 297                 auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 298                 current_presentation_delay_ = dframe->get_age_millis();
 299                 ++scheduled_frames_completed_;
 300
 301                 // Let the fill callback keep the pace, so no scheduling here.
 302
 303                 return S_OK;
 304         }
 305 };
 306
 307 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
 308 {
 309         const int                                                                                       channel_index_;
 310         const configuration                                                                     config_;
 311
 312         com_ptr<IDeckLink>                                                                      decklink_                               = get_device(config_.device_index);
 313         com_iface_ptr<IDeckLinkOutput>                                          output_                                 = iface_cast<IDeckLinkOutput>(decklink_);
 314         com_iface_ptr<IDeckLinkConfiguration>                           configuration_                  = iface_cast<IDeckLinkConfiguration>(decklink_);
 315         com_iface_ptr<IDeckLinkKeyer>                                           keyer_                                  = iface_cast<IDeckLinkKeyer>(decklink_);
 316         com_iface_ptr<IDeckLinkAttributes>                                      attributes_                             = iface_cast<IDeckLinkAttributes>(decklink_);
 317
 318         tbb::spin_mutex                                                                         exception_mutex_;
 319         std::exception_ptr                                                                      exception_;
 320
 321         tbb::atomic<bool>                                                                       is_running_;
 322
 323         const std::wstring                                                                      model_name_                             = get_model_name(decklink_);
 324         const core::video_format_desc                                           format_desc_;
 325         const int                                                                                       buffer_size_                    = config_.buffer_depth(); // Minimum buffer-size 3.
 326
 327         long long                                                                                       video_scheduled_                = 0;
 328         long long                                                                                       audio_scheduled_                = 0;
 329
 330         int                                                                                                     preroll_count_                  = 0;
 331
 332         boost::circular_buffer<std::vector<int32_t>>            audio_container_                { buffer_size_ + 1 };
 333
 334         tbb::concurrent_bounded_queue<core::const_frame>        video_frame_buffer_;
 335         tbb::concurrent_bounded_queue<core::const_frame>        audio_frame_buffer_;
 336
 337         spl::shared_ptr<diagnostics::graph>                                     graph_;
 338         tbb::atomic<int64_t>                                                            current_presentation_delay_;
 339         caspar::timer                                                                           tick_timer_;
 340         retry_task<bool>                                                                        send_completion_;
 341         tbb::atomic<int64_t>                                                            scheduled_frames_completed_;
 342         std::unique_ptr<key_video_context>                                      key_context_;
 343
 344 public:
 345         decklink_consumer(const configuration& config, const core::video_format_desc& format_desc, int channel_index)
 346                 : channel_index_(channel_index)
 347                 , config_(config)
 348                 , format_desc_(format_desc)
 349         {
 350                 is_running_ = true;
 351                 current_presentation_delay_ = 0;
 352                 scheduled_frames_completed_ = 0;
 353
 354                 video_frame_buffer_.set_capacity(1);
 355
 356                 // Blackmagic calls RenderAudioSamples() 50 times per second
 357                 // regardless of video mode so we sometimes need to give them
 358                 // samples from 2 frames in order to keep up
 359                 audio_frame_buffer_.set_capacity((format_desc.fps > 50.0) ? 2 : 1);
 360
 361                 if (config.keyer == configuration::keyer_t::external_separate_device_keyer)
 362                         key_context_.reset(new key_video_context(config, print()));
 363
 364                 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
 365                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
 366                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
 367                 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
 368                 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
 369                 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
 370
 371                 if (key_context_)
 372                 {
 373                         graph_->set_color("key-offset", diagnostics::color(1.0f, 0.0f, 0.0f));
 374                 }
 375
 376                 graph_->set_text(print());
 377                 diagnostics::register_graph(graph_);
 378
 379                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));
 380
 381                 if(config.embedded_audio)
 382                         enable_audio();
 383
 384                 set_latency(configuration_, config.latency, print());
 385                 set_keyer(attributes_, keyer_, config.keyer, print());
 386
 387                 if(config.embedded_audio)
 388                         output_->BeginAudioPreroll();
 389
 390                 for(int n = 0; n < buffer_size_; ++n)
 391                         schedule_next_video(core::const_frame::empty());
 392
 393                 if(!config.embedded_audio)
 394                         start_playback();
 395         }
 396
 397         ~decklink_consumer()
 398         {
 399                 is_running_ = false;
 400                 video_frame_buffer_.try_push(core::const_frame::empty());
 401                 audio_frame_buffer_.try_push(core::const_frame::empty());
 402
 403                 if(output_ != nullptr)
 404                 {
 405                         output_->StopScheduledPlayback(0, nullptr, 0);
 406                         if(config_.embedded_audio)
 407                                 output_->DisableAudioOutput();
 408                         output_->DisableVideoOutput();
 409                 }
 410         }
 411
 412         void enable_audio()
 413         {
 414                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped)))
 415                                 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable audio output."));
 416
 417                 if(FAILED(output_->SetAudioCallback(this)))
 418                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not set audio callback."));
 419
 420                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
 421         }
 422
 423         void enable_video(BMDDisplayMode display_mode)
 424         {
 425                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 426                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable fill video output."));
 427
 428                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 429                         CASPAR_THROW_EXCEPTION(caspar_exception()
 430                                                                         << msg_info(u8(print()) + " Failed to set fill playback completion callback.")
 431                                                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 432
 433                 if (key_context_)
 434                         key_context_->enable_video(display_mode, [this]() { return print(); });
 435         }
 436
 437         void start_playback()
 438         {
 439                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 440                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Failed to schedule fill playback."));
 441
 442                 if (key_context_ && FAILED(key_context_->output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 443                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Failed to schedule key playback."));
 444         }
 445
 446         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 447         virtual ULONG STDMETHODCALLTYPE AddRef()                                        {return 1;}
 448         virtual ULONG STDMETHODCALLTYPE Release()                               {return 1;}
 449
 450         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 451         {
 452                 is_running_ = false;
 453                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
 454                 return S_OK;
 455         }
 456
 457         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result)
 458         {
 459                 if(!is_running_)
 460                         return E_FAIL;
 461
 462                 try
 463                 {
 464                         auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 465                         current_presentation_delay_ = dframe->get_age_millis();
 466                         ++scheduled_frames_completed_;
 467
 468                         if (key_context_)
 469                                 graph_->set_value(
 470                                                 "key-offset",
 471                                                 static_cast<double>(
 472                                                                 scheduled_frames_completed_
 473                                                                 - key_context_->scheduled_frames_completed_)
 474                                                 * 0.1 + 0.5);
 475
 476                         if(result == bmdOutputFrameDisplayedLate)
 477                         {
 478                                 graph_->set_tag("late-frame");
 479                                 video_scheduled_ += format_desc_.duration;
 480                                 audio_scheduled_ += dframe->audio_data().size() / format_desc_.audio_channels;
 481                                 //++video_scheduled_;
 482                                 //audio_scheduled_ += format_desc_.audio_cadence[0];
 483                                 //++audio_scheduled_;
 484                         }
 485                         else if(result == bmdOutputFrameDropped)
 486                                 graph_->set_tag("dropped-frame");
 487                         else if(result == bmdOutputFrameFlushed)
 488                                 graph_->set_tag("flushed-frame");
 489
 490                         auto frame = core::const_frame::empty();
 491                         video_frame_buffer_.pop(frame);
 492                         send_completion_.try_completion();
 493                         schedule_next_video(frame);
 494
 495                         UINT32 buffered;
 496                         output_->GetBufferedVideoFrameCount(&buffered);
 497                         graph_->set_value("buffered-video", static_cast<double>(buffered)/format_desc_.fps);
 498                 }
 499                 catch(...)
 500                 {
 501                         lock(exception_mutex_, [&]
 502                         {
 503                                 exception_ = std::current_exception();
 504                         });
 505                         return E_FAIL;
 506                 }
 507
 508                 return S_OK;
 509         }
 510
 511         virtual HRESULT STDMETHODCALLTYPE RenderAudioSamples(BOOL preroll)
 512         {
 513                 if(!is_running_)
 514                         return E_FAIL;
 515
 516                 try
 517                 {
 518                         if(preroll)
 519                         {
 520                                 if(++preroll_count_ >= buffer_size_)
 521                                 {
 522                                         output_->EndAudioPreroll();
 523                                         start_playback();
 524                                 }
 525                                 else
 526                                 {
 527                                         schedule_next_audio(core::audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()] * format_desc_.audio_channels, 0));
 528                                 }
 529                         }
 530                         else
 531                         {
 532                                 auto frame = core::const_frame::empty();
 533
 534                                 while(audio_frame_buffer_.try_pop(frame))
 535                                 {
 536                                         send_completion_.try_completion();
 537                                         schedule_next_audio(frame.audio_data());
 538                                 }
 539                         }
 540
 541                         UINT32 buffered;
 542                         output_->GetBufferedAudioSampleFrameCount(&buffered);
 543                         graph_->set_value("buffered-audio", static_cast<double>(buffered) / (format_desc_.audio_cadence[0] * format_desc_.audio_channels * 2));
 544                 }
 545                 catch(...)
 546                 {
 547                         tbb::spin_mutex::scoped_lock lock(exception_mutex_);
 548                         exception_ = std::current_exception();
 549                         return E_FAIL;
 550                 }
 551
 552                 return S_OK;
 553         }
 554
 555         template<typename T>
 556         void schedule_next_audio(const T& audio_data)
 557         {
 558                 auto sample_frame_count = static_cast<int>(audio_data.size()/format_desc_.audio_channels);
 559
 560                 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
 561
 562                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
 563                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
 564
 565                 audio_scheduled_ += sample_frame_count;
 566         }
 567
 568         void schedule_next_video(core::const_frame frame)
 569         {
 570                 if (key_context_)
 571                 {
 572                         auto key_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, true));
 573                         if (FAILED(key_context_->output_->ScheduleVideoFrame(key_frame, video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 574                                 CASPAR_LOG(error) << print() << L" Failed to schedule key video.";
 575                 }
 576
 577                 auto fill_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, config_.key_only));
 578                 if (FAILED(output_->ScheduleVideoFrame(get_raw(fill_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 579                         CASPAR_LOG(error) << print() << L" Failed to schedule fill video.";
 580
 581                 video_scheduled_ += format_desc_.duration;
 582
 583                 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
 584                 tick_timer_.restart();
 585         }
 586
 587         std::future<bool> send(core::const_frame frame)
 588         {
 589                 auto exception = lock(exception_mutex_, [&]
 590                 {
 591                         return exception_;
 592                 });
 593
 594                 if(exception != nullptr)
 595                         std::rethrow_exception(exception);
 596
 597                 if(!is_running_)
 598                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Is not running."));
 599
 600                 bool audio_ready = !config_.embedded_audio;
 601                 bool video_ready = false;
 602
 603                 auto enqueue_task = [audio_ready, video_ready, frame, this]() mutable -> boost::optional<bool>
 604                 {
 605                         if (!audio_ready)
 606                                 audio_ready = audio_frame_buffer_.try_push(frame);
 607
 608                         if (!video_ready)
 609                                 video_ready = video_frame_buffer_.try_push(frame);
 610
 611                         if (audio_ready && video_ready)
 612                                 return true;
 613                         else
 614                                 return boost::optional<bool>();
 615                 };
 616
 617                 if (enqueue_task())
 618                         return make_ready_future(true);
 619
 620                 send_completion_.set_task(enqueue_task);
 621
 622                 return send_completion_.get_future();
 623         }
 624
 625         std::wstring print() const
 626         {
 627                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 628                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 629                                 boost::lexical_cast<std::wstring>(config_.device_index) +
 630                                 L"&&" +
 631                                 boost::lexical_cast<std::wstring>(config_.key_device_index()) +
 632                                 L"|" +
 633                                 format_desc_.name + L"]";
 634                 else
 635                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 636                                 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
 637         }
 638 };
 639
 640 struct decklink_consumer_proxy : public core::frame_consumer
 641 {
 642         core::monitor::subject                          monitor_subject_;
 643         const configuration                                     config_;
 644         std::unique_ptr<decklink_consumer>      consumer_;
 645         core::video_format_desc                         format_desc_;
 646         executor                                                        executor_;
 647 public:
 648
 649         decklink_consumer_proxy(const configuration& config)
 650                 : config_(config)
 651                 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
 652         {
 653                 auto ctx = core::diagnostics::call_context::for_thread();
 654                 executor_.begin_invoke([=]
 655                 {
 656                         core::diagnostics::call_context::for_thread() = ctx;
 657                         com_initialize();
 658                 });
 659         }
 660
 661         ~decklink_consumer_proxy()
 662         {
 663                 executor_.invoke([=]
 664                 {
 665                         consumer_.reset();
 666                         com_uninitialize();
 667                 });
 668         }
 669
 670         // frame_consumer
 671
 672         void initialize(const core::video_format_desc& format_desc, int channel_index) override
 673         {
 674                 format_desc_ = format_desc;
 675                 executor_.invoke([=]
 676                 {
 677                         consumer_.reset();
 678                         consumer_.reset(new decklink_consumer(config_, format_desc, channel_index));
 679                 });
 680         }
 681
 682         std::future<bool> send(core::const_frame frame) override
 683         {
 684                 return consumer_->send(frame);
 685         }
 686
 687         std::wstring print() const override
 688         {
 689                 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
 690         }
 691
 692         std::wstring name() const override
 693         {
 694                 return L"decklink";
 695         }
 696
 697         boost::property_tree::wptree info() const override
 698         {
 699                 boost::property_tree::wptree info;
 700                 info.add(L"type", L"decklink");
 701                 info.add(L"key-only", config_.key_only);
 702                 info.add(L"device", config_.device_index);
 703
 704                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 705                 {
 706                         info.add(L"key-device", config_.key_device_index());
 707                 }
 708
 709                 info.add(L"low-latency", config_.latency == configuration::latency_t::low_latency);
 710                 info.add(L"embedded-audio", config_.embedded_audio);
 711                 info.add(L"presentation-frame-age", presentation_frame_age_millis());
 712                 //info.add(L"internal-key", config_.internal_key);
 713                 return info;
 714         }
 715
 716         int buffer_depth() const override
 717         {
 718                 return config_.buffer_depth() + 2;
 719         }
 720
 721         int index() const override
 722         {
 723                 return 300 + config_.device_index;
 724         }
 725
 726         int64_t presentation_frame_age_millis() const override
 727         {
 728                 return consumer_ ? static_cast<int64_t>(consumer_->current_presentation_delay_) : 0;
 729         }
 730
 731         core::monitor::subject& monitor_output()
 732         {
 733                 return monitor_subject_;
 734         }
 735 };
 736
 737 void describe_consumer(core::help_sink& sink, const core::help_repository& repo)
 738 {
 739         sink.short_description(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 740         sink.syntax(L"DECKLINK "
 741                                 L"{[device_index:int]|1} "
 742                                 L"{[keyer:INTERNAL_KEY,EXTERNAL_KEY,EXTERNAL_SEPARATE_DEVICE_KEY]} "
 743                                 L"{[low_latency:LOW_LATENCY]} "
 744                                 L"{[embedded_audio:EMBEDDED_AUDIO]} "
 745                                 L"{[key_only:KEY_ONLY]}");
 746         sink.para()->text(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 747         sink.definitions()
 748                 ->item(L"device_index", L"The Blackmagic video card to use (See Blackmagic control panel for card order). Default is 1.")
 749                 ->item(L"keyer",
 750                                 L"If given tries to enable either internal or external keying. Not all Blackmagic cards supports this. "
 751                                 L"There is also a third experimental option (EXTERNAL_SEPARATE_DEVICE_KEY) which allocates device_index + 1 for synhronized key output.")
 752                 ->item(L"low_latency", L"Tries to enable low latency if given.")
 753                 ->item(L"embedded_audio", L"Embeds the audio into the SDI signal if given.")
 754                 ->item(L"key_only",
 755                                 L" will extract only the alpha channel from the "
 756                                 L"channel. This is useful when you have two SDI video cards, and neither has native support "
 757                                 L"for separate fill/key output");
 758         sink.para()->text(L"Examples:");
 759         sink.example(L">> ADD 1 DECKLINK", L"for using the default device_index of 1.");
 760         sink.example(L">> ADD 1 DECKLINK 2", L"uses device_index 2.");
 761         sink.example(L">> ADD 1 DECKLINK 1 EXTERNAL_KEY EMBEDDED_AUDIO");
 762         sink.example(
 763                         L">> ADD 1 DECKLINK 1 EMBEDDED_AUDIO\n"
 764                         L">> ADD 1 DECKLINK 2 KEY_ONLY", L"uses device with index 1 as fill output with audio and device with index 2 as key output.");
 765         sink.example(
 766                         L">> ADD 1 DECKLINK 1 EXTERNAL_SEPARATE_DEVICE_KEY EMBEDDED_AUDIO",
 767                         L"Uses device 2 for key output. May give better sync between key and fill than the previous method.");
 768 }
 769
 770 spl::shared_ptr<core::frame_consumer> create_consumer(
 771                 const std::vector<std::wstring>& params, core::interaction_sink*)
 772 {
 773         if (params.size() < 1 || !boost::iequals(params.at(0), L"DECKLINK"))
 774                 return core::frame_consumer::empty();
 775
 776         configuration config;
 777
 778         if (params.size() > 1)
 779                 config.device_index = boost::lexical_cast<int>(params.at(1));
 780
 781         if (contains_param(L"INTERNAL_KEY", params))
 782                 config.keyer = configuration::keyer_t::internal_keyer;
 783         else if (contains_param(L"EXTERNAL_KEY", params))
 784                 config.keyer = configuration::keyer_t::external_keyer;
 785         else if (contains_param(L"EXTERNAL_SEPARATE_DEVICE_KEY", params))
 786                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 787         else
 788                 config.keyer = configuration::keyer_t::default_keyer;
 789
 790         if (contains_param(L"LOW_LATENCY", params))
 791                 config.latency = configuration::latency_t::low_latency;
 792
 793         config.embedded_audio   = contains_param(L"EMBEDDED_AUDIO", params);
 794         config.key_only                 = contains_param(L"KEY_ONLY", params);
 795
 796         return spl::make_shared<decklink_consumer_proxy>(config);
 797 }
 798
 799 spl::shared_ptr<core::frame_consumer> create_preconfigured_consumer(
 800                 const boost::property_tree::wptree& ptree, core::interaction_sink*)
 801 {
 802         configuration config;
 803
 804         auto keyer = ptree.get(L"keyer", L"default");
 805         if(keyer == L"external")
 806                 config.keyer = configuration::keyer_t::external_keyer;
 807         else if(keyer == L"internal")
 808                 config.keyer = configuration::keyer_t::internal_keyer;
 809         else if (keyer == L"external_separate_device")
 810                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 811
 812         auto latency = ptree.get(L"latency", L"normal");
 813         if(latency == L"low")
 814                 config.latency = configuration::latency_t::low_latency;
 815         else if(latency == L"normal")
 816                 config.latency = configuration::latency_t::normal_latency;
 817
 818         config.key_only                         = ptree.get(L"key-only",                config.key_only);
 819         config.device_index                     = ptree.get(L"device",                  config.device_index);
 820         config.key_device_idx           = ptree.get(L"key-device",              config.key_device_idx);
 821         config.embedded_audio           = ptree.get(L"embedded-audio",  config.embedded_audio);
 822         config.base_buffer_depth        = ptree.get(L"buffer-depth",    config.base_buffer_depth);
 823
 824         return spl::make_shared<decklink_consumer_proxy>(config);
 825 }
 826
 827 }}
 828
 829 /*
 830 ##############################################################################
 831 Pre-rolling
 832
 833 Mail: 2011-05-09
 834
 835 Yoshan
 836 BMD Developer Support
 837 developer@blackmagic-design.com
 838
 839 -----------------------------------------------------------------------------
 840
 841 Thanks for your inquiry. The minimum number of frames that you can preroll
 842 for scheduled playback is three frames for video and four frames for audio.
 843 As you mentioned if you preroll less frames then playback will not start or
 844 playback will be very sporadic. From our experience with Media Express, we
 845 recommended that at least seven frames are prerolled for smooth playback.
 846
 847 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
 848 There can be around 3 frames worth of latency on scheduled output.
 849 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
 850 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync()
 851 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
 852 guarantee that the provided frame will be output as soon the previous
 853 frame output has been completed.
 854 ################################################################################
 855 */
 856
 857 /*
 858 ##############################################################################
 859 Async DMA Transfer without redundant copying
 860
 861 Mail: 2011-05-10
 862
 863 Yoshan
 864 BMD Developer Support
 865 developer@blackmagic-design.com
 866
 867 -----------------------------------------------------------------------------
 868
 869 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
 870 and providing a pointer to your video buffer when GetBytes() is called.
 871 This may help to keep copying to a minimum. Please ensure that the pixel
 872 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
 873 have to colourspace convert which may result in additional copying.
 874 ################################################################################
 875 */