git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Robert Nagy, ronag89@gmail.com
  20 */
  21
  22 #include "../StdAfx.h"
  23
  24 #include "decklink_consumer.h"
  25
  26 #include "../util/util.h"
  27
  28 #include "../decklink_api.h"
  29
  30 #include <core/frame/frame.h>
  31 #include <core/frame/audio_channel_layout.h>
  32 #include <core/mixer/audio/audio_mixer.h>
  33 #include <core/consumer/frame_consumer.h>
  34 #include <core/diagnostics/call_context.h>
  35 #include <core/help/help_sink.h>
  36 #include <core/help/help_repository.h>
  37
  38 #include <common/executor.h>
  39 #include <common/lock.h>
  40 #include <common/diagnostics/graph.h>
  41 #include <common/except.h>
  42 #include <common/memshfl.h>
  43 #include <common/array.h>
  44 #include <common/future.h>
  45 #include <common/cache_aligned_vector.h>
  46 #include <common/timer.h>
  47 #include <common/param.h>
  48
  49 #include <tbb/concurrent_queue.h>
  50
  51 #include <common/assert.h>
  52 #include <boost/lexical_cast.hpp>
  53 #include <boost/circular_buffer.hpp>
  54 #include <boost/property_tree/ptree.hpp>
  55
  56 namespace caspar { namespace decklink {
  57
  58 struct configuration
  59 {
  60         enum class keyer_t
  61         {
  62                 internal_keyer,
  63                 external_keyer,
  64                 external_separate_device_keyer,
  65                 default_keyer
  66         };
  67
  68         enum class latency_t
  69         {
  70                 low_latency,
  71                 normal_latency,
  72                 default_latency
  73         };
  74
  75         int                                                     device_index            = 1;
  76         int                                                     key_device_idx          = 0;
  77         bool                                            embedded_audio          = true;
  78         keyer_t                                         keyer                           = keyer_t::default_keyer;
  79         latency_t                                       latency                         = latency_t::default_latency;
  80         bool                                            key_only                        = false;
  81         int                                                     base_buffer_depth       = 3;
  82         core::audio_channel_layout      out_channel_layout      = core::audio_channel_layout::invalid();
  83
  84         int buffer_depth() const
  85         {
  86                 return base_buffer_depth + (latency == latency_t::low_latency ? 0 : 1) + (embedded_audio ? 1 : 0);
  87         }
  88
  89         int key_device_index() const
  90         {
  91                 return key_device_idx == 0 ? device_index + 1 : key_device_idx;
  92         }
  93
  94         core::audio_channel_layout get_adjusted_layout(const core::audio_channel_layout& in_layout) const
  95         {
  96                 auto adjusted = out_channel_layout == core::audio_channel_layout::invalid() ? in_layout : out_channel_layout;
  97
  98                 if (adjusted.num_channels == 1) // Duplicate mono-signal into both left and right.
  99                 {
 100                         adjusted.num_channels = 2;
 101                         adjusted.channel_order.push_back(adjusted.channel_order.at(0)); // Usually FC -> FC FC
 102                 }
 103                 else if (adjusted.num_channels == 2)
 104                 {
 105                         adjusted.num_channels = 2;
 106                 }
 107                 else if (adjusted.num_channels <= 8)
 108                 {
 109                         adjusted.num_channels = 8;
 110                 }
 111                 else // Over 8 always pad to 16 or drop >16
 112                 {
 113                         adjusted.num_channels = 16;
 114                 }
 115
 116                 return adjusted;
 117         }
 118 };
 119
 120 void set_latency(
 121                 const com_iface_ptr<IDeckLinkConfiguration>& config,
 122                 configuration::latency_t latency,
 123                 const std::wstring& print)
 124 {
 125         if (latency == configuration::latency_t::low_latency)
 126         {
 127                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
 128                 CASPAR_LOG(info) << print << L" Enabled low-latency mode.";
 129         }
 130         else if (latency == configuration::latency_t::normal_latency)
 131         {
 132                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
 133                 CASPAR_LOG(info) << print << L" Disabled low-latency mode.";
 134         }
 135 }
 136
 137 void set_keyer(
 138                 const com_iface_ptr<IDeckLinkAttributes>& attributes,
 139                 const com_iface_ptr<IDeckLinkKeyer>& decklink_keyer,
 140                 configuration::keyer_t keyer,
 141                 const std::wstring& print)
 142 {
 143         if (keyer == configuration::keyer_t::internal_keyer)
 144         {
 145                 BOOL value = true;
 146                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
 147                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 148                 else if (FAILED(decklink_keyer->Enable(FALSE)))
 149                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 150                 else if (FAILED(decklink_keyer->SetLevel(255)))
 151                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 152                 else
 153                         CASPAR_LOG(info) << print << L" Enabled internal keyer.";
 154         }
 155         else if (keyer == configuration::keyer_t::external_keyer)
 156         {
 157                 BOOL value = true;
 158                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
 159                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 160                 else if (FAILED(decklink_keyer->Enable(TRUE)))
 161                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 162                 else if (FAILED(decklink_keyer->SetLevel(255)))
 163                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 164                 else
 165                         CASPAR_LOG(info) << print << L" Enabled external keyer.";
 166         }
 167 }
 168
 169 class decklink_frame : public IDeckLinkVideoFrame
 170 {
 171         tbb::atomic<int>                                ref_count_;
 172         core::const_frame                               frame_;
 173         const core::video_format_desc   format_desc_;
 174
 175         const bool                                              key_only_;
 176         cache_aligned_vector<uint8_t>   data_;
 177 public:
 178         decklink_frame(core::const_frame frame, const core::video_format_desc& format_desc, bool key_only)
 179                 : frame_(frame)
 180                 , format_desc_(format_desc)
 181                 , key_only_(key_only)
 182         {
 183                 ref_count_ = 0;
 184         }
 185
 186         // IUnknown
 187
 188         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)
 189         {
 190                 return E_NOINTERFACE;
 191         }
 192
 193         virtual ULONG STDMETHODCALLTYPE AddRef()
 194         {
 195                 return ++ref_count_;
 196         }
 197
 198         virtual ULONG STDMETHODCALLTYPE Release()
 199         {
 200                 if(--ref_count_ == 0)
 201                         delete this;
 202                 return ref_count_;
 203         }
 204
 205         // IDecklinkVideoFrame
 206
 207         virtual long STDMETHODCALLTYPE GetWidth()                   {return static_cast<long>(format_desc_.width);}
 208         virtual long STDMETHODCALLTYPE GetHeight()                  {return static_cast<long>(format_desc_.height);}
 209         virtual long STDMETHODCALLTYPE GetRowBytes()                {return static_cast<long>(format_desc_.width*4);}
 210         virtual BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat()   {return bmdFormat8BitBGRA;}
 211         virtual BMDFrameFlags STDMETHODCALLTYPE GetFlags()                      {return bmdFrameFlagDefault;}
 212
 213         virtual HRESULT STDMETHODCALLTYPE GetBytes(void** buffer)
 214         {
 215                 try
 216                 {
 217                         if(static_cast<int>(frame_.image_data().size()) != format_desc_.size)
 218                         {
 219                                 data_.resize(format_desc_.size, 0);
 220                                 *buffer = data_.data();
 221                         }
 222                         else if(key_only_)
 223                         {
 224                                 if(data_.empty())
 225                                 {
 226                                         data_.resize(frame_.image_data().size());
 227                                         aligned_memshfl(data_.data(), frame_.image_data().begin(), frame_.image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
 228                                 }
 229                                 *buffer = data_.data();
 230                         }
 231                         else
 232                                 *buffer = const_cast<uint8_t*>(frame_.image_data().begin());
 233                 }
 234                 catch(...)
 235                 {
 236                         CASPAR_LOG_CURRENT_EXCEPTION();
 237                         return E_FAIL;
 238                 }
 239
 240                 return S_OK;
 241         }
 242
 243         virtual HRESULT STDMETHODCALLTYPE GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode) {return S_FALSE;}
 244         virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)              {return S_FALSE;}
 245
 246         // decklink_frame
 247
 248         const core::audio_buffer& audio_data()
 249         {
 250                 return frame_.audio_data();
 251         }
 252
 253         int64_t get_age_millis() const
 254         {
 255                 return frame_.get_age_millis();
 256         }
 257 };
 258
 259 struct key_video_context : public IDeckLinkVideoOutputCallback, boost::noncopyable
 260 {
 261         const configuration                                                                             config_;
 262         com_ptr<IDeckLink>                                                                              decklink_                                       = get_device(config_.key_device_index());
 263         com_iface_ptr<IDeckLinkOutput>                                                  output_                                         = iface_cast<IDeckLinkOutput>(decklink_);
 264         com_iface_ptr<IDeckLinkKeyer>                                                   keyer_                                          = iface_cast<IDeckLinkKeyer>(decklink_);
 265         com_iface_ptr<IDeckLinkAttributes>                                              attributes_                                     = iface_cast<IDeckLinkAttributes>(decklink_);
 266         com_iface_ptr<IDeckLinkConfiguration>                                   configuration_                          = iface_cast<IDeckLinkConfiguration>(decklink_);
 267         tbb::atomic<int64_t>                                                                    current_presentation_delay_;
 268         tbb::atomic<int64_t>                                                                    scheduled_frames_completed_;
 269
 270         key_video_context(const configuration& config, const std::wstring& print)
 271                 : config_(config)
 272         {
 273                 current_presentation_delay_ = 0;
 274                 scheduled_frames_completed_ = 0;
 275
 276                 set_latency(configuration_, config.latency, print);
 277                 set_keyer(attributes_, keyer_, config.keyer, print);
 278
 279                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 280                         CASPAR_THROW_EXCEPTION(caspar_exception()
 281                                         << msg_info(u8(print) + " Failed to set key playback completion callback.")
 282                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 283         }
 284
 285         template<typename Print>
 286         void enable_video(BMDDisplayMode display_mode, const Print& print)
 287         {
 288                 if (FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 289                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable key video output."));
 290
 291                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 292                         CASPAR_THROW_EXCEPTION(caspar_exception()
 293                                         << msg_info(u8(print()) + " Failed to set key playback completion callback.")
 294                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 295         }
 296
 297         virtual ~key_video_context()
 298         {
 299                 if (output_)
 300                 {
 301                         output_->StopScheduledPlayback(0, nullptr, 0);
 302                         output_->DisableVideoOutput();
 303                 }
 304         }
 305
 306         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 307         virtual ULONG STDMETHODCALLTYPE AddRef()                                                        {return 1;}
 308         virtual ULONG STDMETHODCALLTYPE Release()                                                       {return 1;}
 309
 310         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 311         {
 312                 return S_OK;
 313         }
 314
 315         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(
 316                         IDeckLinkVideoFrame* completed_frame,
 317                         BMDOutputFrameCompletionResult result)
 318         {
 319                 auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 320                 current_presentation_delay_ = dframe->get_age_millis();
 321                 ++scheduled_frames_completed_;
 322
 323                 // Let the fill callback keep the pace, so no scheduling here.
 324
 325                 return S_OK;
 326         }
 327 };
 328
 329 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
 330 {
 331         const int                                                                                       channel_index_;
 332         const configuration                                                                     config_;
 333
 334         com_ptr<IDeckLink>                                                                      decklink_                               = get_device(config_.device_index);
 335         com_iface_ptr<IDeckLinkOutput>                                          output_                                 = iface_cast<IDeckLinkOutput>(decklink_);
 336         com_iface_ptr<IDeckLinkConfiguration>                           configuration_                  = iface_cast<IDeckLinkConfiguration>(decklink_);
 337         com_iface_ptr<IDeckLinkKeyer>                                           keyer_                                  = iface_cast<IDeckLinkKeyer>(decklink_);
 338         com_iface_ptr<IDeckLinkAttributes>                                      attributes_                             = iface_cast<IDeckLinkAttributes>(decklink_);
 339
 340         tbb::spin_mutex                                                                         exception_mutex_;
 341         std::exception_ptr                                                                      exception_;
 342
 343         tbb::atomic<bool>                                                                       is_running_;
 344
 345         const std::wstring                                                                      model_name_                             = get_model_name(decklink_);
 346         const core::video_format_desc                                           format_desc_;
 347         const core::audio_channel_layout                                        in_channel_layout_;
 348         const core::audio_channel_layout                                        out_channel_layout_             = config_.get_adjusted_layout(in_channel_layout_);
 349         core::audio_channel_remapper                                            channel_remapper_               { in_channel_layout_, out_channel_layout_ };
 350         const int                                                                                       buffer_size_                    = config_.buffer_depth(); // Minimum buffer-size 3.
 351
 352         long long                                                                                       video_scheduled_                = 0;
 353         long long                                                                                       audio_scheduled_                = 0;
 354
 355         int                                                                                                     preroll_count_                  = 0;
 356
 357         boost::circular_buffer<std::vector<int32_t>>            audio_container_                { buffer_size_ + 1 };
 358
 359         tbb::concurrent_bounded_queue<core::const_frame>        video_frame_buffer_;
 360         tbb::concurrent_bounded_queue<core::const_frame>        audio_frame_buffer_;
 361
 362         spl::shared_ptr<diagnostics::graph>                                     graph_;
 363         tbb::atomic<int64_t>                                                            current_presentation_delay_;
 364         caspar::timer                                                                           tick_timer_;
 365         retry_task<bool>                                                                        send_completion_;
 366         tbb::atomic<int64_t>                                                            scheduled_frames_completed_;
 367         std::unique_ptr<key_video_context>                                      key_context_;
 368
 369 public:
 370         decklink_consumer(
 371                         const configuration& config,
 372                         const core::video_format_desc& format_desc,
 373                         const core::audio_channel_layout& in_channel_layout,
 374                         int channel_index)
 375                 : channel_index_(channel_index)
 376                 , config_(config)
 377                 , format_desc_(format_desc)
 378                 , in_channel_layout_(in_channel_layout)
 379         {
 380                 is_running_ = true;
 381                 current_presentation_delay_ = 0;
 382                 scheduled_frames_completed_ = 0;
 383
 384                 video_frame_buffer_.set_capacity(1);
 385
 386                 // Blackmagic calls RenderAudioSamples() 50 times per second
 387                 // regardless of video mode so we sometimes need to give them
 388                 // samples from 2 frames in order to keep up
 389                 audio_frame_buffer_.set_capacity((format_desc.fps > 50.0) ? 2 : 1);
 390
 391                 if (config.keyer == configuration::keyer_t::external_separate_device_keyer)
 392                         key_context_.reset(new key_video_context(config, print()));
 393
 394                 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
 395                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
 396                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
 397                 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
 398                 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
 399                 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
 400
 401                 if (key_context_)
 402                 {
 403                         graph_->set_color("key-offset", diagnostics::color(1.0f, 0.0f, 0.0f));
 404                 }
 405
 406                 graph_->set_text(print());
 407                 diagnostics::register_graph(graph_);
 408
 409                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));
 410
 411                 if(config.embedded_audio)
 412                         enable_audio();
 413
 414                 set_latency(configuration_, config.latency, print());
 415                 set_keyer(attributes_, keyer_, config.keyer, print());
 416
 417                 if(config.embedded_audio)
 418                         output_->BeginAudioPreroll();
 419
 420                 for(int n = 0; n < buffer_size_; ++n)
 421                         schedule_next_video(core::const_frame::empty());
 422
 423                 if(!config.embedded_audio)
 424                         start_playback();
 425         }
 426
 427         ~decklink_consumer()
 428         {
 429                 is_running_ = false;
 430                 video_frame_buffer_.try_push(core::const_frame::empty());
 431                 audio_frame_buffer_.try_push(core::const_frame::empty());
 432
 433                 if(output_ != nullptr)
 434                 {
 435                         output_->StopScheduledPlayback(0, nullptr, 0);
 436                         if(config_.embedded_audio)
 437                                 output_->DisableAudioOutput();
 438                         output_->DisableVideoOutput();
 439                 }
 440         }
 441
 442         void enable_audio()
 443         {
 444                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, out_channel_layout_.num_channels, bmdAudioOutputStreamTimestamped)))
 445                                 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable audio output."));
 446
 447                 if(FAILED(output_->SetAudioCallback(this)))
 448                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not set audio callback."));
 449
 450                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
 451         }
 452
 453         void enable_video(BMDDisplayMode display_mode)
 454         {
 455                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 456                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Could not enable fill video output."));
 457
 458                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 459                         CASPAR_THROW_EXCEPTION(caspar_exception()
 460                                                                         << msg_info(u8(print()) + " Failed to set fill playback completion callback.")
 461                                                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 462
 463                 if (key_context_)
 464                         key_context_->enable_video(display_mode, [this]() { return print(); });
 465         }
 466
 467         void start_playback()
 468         {
 469                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 470                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Failed to schedule fill playback."));
 471
 472                 if (key_context_ && FAILED(key_context_->output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 473                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Failed to schedule key playback."));
 474         }
 475
 476         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 477         virtual ULONG STDMETHODCALLTYPE AddRef()                                        {return 1;}
 478         virtual ULONG STDMETHODCALLTYPE Release()                               {return 1;}
 479
 480         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 481         {
 482                 is_running_ = false;
 483                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
 484                 return S_OK;
 485         }
 486
 487         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result)
 488         {
 489                 if(!is_running_)
 490                         return E_FAIL;
 491
 492                 try
 493                 {
 494                         auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 495                         current_presentation_delay_ = dframe->get_age_millis();
 496                         ++scheduled_frames_completed_;
 497
 498                         if (key_context_)
 499                                 graph_->set_value(
 500                                                 "key-offset",
 501                                                 static_cast<double>(
 502                                                                 scheduled_frames_completed_
 503                                                                 - key_context_->scheduled_frames_completed_)
 504                                                 * 0.1 + 0.5);
 505
 506                         if(result == bmdOutputFrameDisplayedLate)
 507                         {
 508                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "late-frame");
 509                                 video_scheduled_ += format_desc_.duration;
 510                                 audio_scheduled_ += dframe->audio_data().size() / out_channel_layout_.num_channels;
 511                                 //++video_scheduled_;
 512                                 //audio_scheduled_ += format_desc_.audio_cadence[0];
 513                                 //++audio_scheduled_;
 514                         }
 515                         else if(result == bmdOutputFrameDropped)
 516                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "dropped-frame");
 517                         else if(result == bmdOutputFrameFlushed)
 518                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "flushed-frame");
 519
 520                         auto frame = core::const_frame::empty();
 521                         video_frame_buffer_.pop(frame);
 522                         send_completion_.try_completion();
 523                         schedule_next_video(frame);
 524
 525                         UINT32 buffered;
 526                         output_->GetBufferedVideoFrameCount(&buffered);
 527                         graph_->set_value("buffered-video", static_cast<double>(buffered)/format_desc_.fps);
 528                 }
 529                 catch(...)
 530                 {
 531                         lock(exception_mutex_, [&]
 532                         {
 533                                 exception_ = std::current_exception();
 534                         });
 535                         return E_FAIL;
 536                 }
 537
 538                 return S_OK;
 539         }
 540
 541         virtual HRESULT STDMETHODCALLTYPE RenderAudioSamples(BOOL preroll)
 542         {
 543                 if(!is_running_)
 544                         return E_FAIL;
 545
 546                 try
 547                 {
 548                         if(preroll)
 549                         {
 550                                 if(++preroll_count_ >= buffer_size_)
 551                                 {
 552                                         output_->EndAudioPreroll();
 553                                         start_playback();
 554                                 }
 555                                 else
 556                                 {
 557                                         schedule_next_audio(core::mutable_audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()] * out_channel_layout_.num_channels, 0));
 558                                 }
 559                         }
 560                         else
 561                         {
 562                                 auto frame = core::const_frame::empty();
 563
 564                                 while(audio_frame_buffer_.try_pop(frame))
 565                                 {
 566                                         send_completion_.try_completion();
 567                                         schedule_next_audio(channel_remapper_.mix_and_rearrange(frame.audio_data()));
 568                                 }
 569                         }
 570
 571                         UINT32 buffered;
 572                         output_->GetBufferedAudioSampleFrameCount(&buffered);
 573                         graph_->set_value("buffered-audio", static_cast<double>(buffered) / (format_desc_.audio_cadence[0] * config_.buffer_depth()));
 574                 }
 575                 catch(...)
 576                 {
 577                         tbb::spin_mutex::scoped_lock lock(exception_mutex_);
 578                         exception_ = std::current_exception();
 579                         return E_FAIL;
 580                 }
 581
 582                 return S_OK;
 583         }
 584
 585         template<typename T>
 586         void schedule_next_audio(const T& audio_data)
 587         {
 588                 auto sample_frame_count = static_cast<int>(audio_data.size()/out_channel_layout_.num_channels);
 589
 590                 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
 591
 592                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
 593                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
 594
 595                 audio_scheduled_ += sample_frame_count;
 596         }
 597
 598         void schedule_next_video(core::const_frame frame)
 599         {
 600                 if (key_context_)
 601                 {
 602                         auto key_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, true));
 603                         if (FAILED(key_context_->output_->ScheduleVideoFrame(get_raw(key_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 604                                 CASPAR_LOG(error) << print() << L" Failed to schedule key video.";
 605                 }
 606
 607                 auto fill_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, config_.key_only));
 608                 if (FAILED(output_->ScheduleVideoFrame(get_raw(fill_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 609                         CASPAR_LOG(error) << print() << L" Failed to schedule fill video.";
 610
 611                 video_scheduled_ += format_desc_.duration;
 612
 613                 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
 614                 tick_timer_.restart();
 615         }
 616
 617         std::future<bool> send(core::const_frame frame)
 618         {
 619                 auto exception = lock(exception_mutex_, [&]
 620                 {
 621                         return exception_;
 622                 });
 623
 624                 if(exception != nullptr)
 625                         std::rethrow_exception(exception);
 626
 627                 if(!is_running_)
 628                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(u8(print()) + " Is not running."));
 629
 630                 bool audio_ready = !config_.embedded_audio;
 631                 bool video_ready = false;
 632
 633                 auto enqueue_task = [audio_ready, video_ready, frame, this]() mutable -> boost::optional<bool>
 634                 {
 635                         if (!audio_ready)
 636                                 audio_ready = audio_frame_buffer_.try_push(frame);
 637
 638                         if (!video_ready)
 639                                 video_ready = video_frame_buffer_.try_push(frame);
 640
 641                         if (audio_ready && video_ready)
 642                                 return true;
 643                         else
 644                                 return boost::optional<bool>();
 645                 };
 646
 647                 if (enqueue_task())
 648                         return make_ready_future(true);
 649
 650                 send_completion_.set_task(enqueue_task);
 651
 652                 return send_completion_.get_future();
 653         }
 654
 655         std::wstring print() const
 656         {
 657                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 658                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 659                                 boost::lexical_cast<std::wstring>(config_.device_index) +
 660                                 L"&&" +
 661                                 boost::lexical_cast<std::wstring>(config_.key_device_index()) +
 662                                 L"|" +
 663                                 format_desc_.name + L"]";
 664                 else
 665                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 666                                 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
 667         }
 668 };
 669
 670 struct decklink_consumer_proxy : public core::frame_consumer
 671 {
 672         core::monitor::subject                          monitor_subject_;
 673         const configuration                                     config_;
 674         std::unique_ptr<decklink_consumer>      consumer_;
 675         core::video_format_desc                         format_desc_;
 676         executor                                                        executor_;
 677 public:
 678
 679         decklink_consumer_proxy(const configuration& config)
 680                 : config_(config)
 681                 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
 682         {
 683                 auto ctx = core::diagnostics::call_context::for_thread();
 684                 executor_.begin_invoke([=]
 685                 {
 686                         core::diagnostics::call_context::for_thread() = ctx;
 687                         com_initialize();
 688                 });
 689         }
 690
 691         ~decklink_consumer_proxy()
 692         {
 693                 executor_.invoke([=]
 694                 {
 695                         consumer_.reset();
 696                         com_uninitialize();
 697                 });
 698         }
 699
 700         // frame_consumer
 701
 702         void initialize(const core::video_format_desc& format_desc, const core::audio_channel_layout& channel_layout, int channel_index) override
 703         {
 704                 format_desc_ = format_desc;
 705                 executor_.invoke([=]
 706                 {
 707                         consumer_.reset();
 708                         consumer_.reset(new decklink_consumer(config_, format_desc, channel_layout, channel_index));
 709                 });
 710         }
 711
 712         std::future<bool> send(core::const_frame frame) override
 713         {
 714                 return consumer_->send(frame);
 715         }
 716
 717         std::wstring print() const override
 718         {
 719                 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
 720         }
 721
 722         std::wstring name() const override
 723         {
 724                 return L"decklink";
 725         }
 726
 727         boost::property_tree::wptree info() const override
 728         {
 729                 boost::property_tree::wptree info;
 730                 info.add(L"type", L"decklink");
 731                 info.add(L"key-only", config_.key_only);
 732                 info.add(L"device", config_.device_index);
 733
 734                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 735                 {
 736                         info.add(L"key-device", config_.key_device_index());
 737                 }
 738
 739                 info.add(L"low-latency", config_.latency == configuration::latency_t::low_latency);
 740                 info.add(L"embedded-audio", config_.embedded_audio);
 741                 info.add(L"presentation-frame-age", presentation_frame_age_millis());
 742                 //info.add(L"internal-key", config_.internal_key);
 743                 return info;
 744         }
 745
 746         int buffer_depth() const override
 747         {
 748                 return config_.buffer_depth() + 2;
 749         }
 750
 751         int index() const override
 752         {
 753                 return 300 + config_.device_index;
 754         }
 755
 756         int64_t presentation_frame_age_millis() const override
 757         {
 758                 return consumer_ ? static_cast<int64_t>(consumer_->current_presentation_delay_) : 0;
 759         }
 760
 761         core::monitor::subject& monitor_output()
 762         {
 763                 return monitor_subject_;
 764         }
 765 };
 766
 767 void describe_consumer(core::help_sink& sink, const core::help_repository& repo)
 768 {
 769         sink.short_description(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 770         sink.syntax(L"DECKLINK "
 771                                 L"{[device_index:int]|1} "
 772                                 L"{[keyer:INTERNAL_KEY,EXTERNAL_KEY,EXTERNAL_SEPARATE_DEVICE_KEY]} "
 773                                 L"{[low_latency:LOW_LATENCY]} "
 774                                 L"{[embedded_audio:EMBEDDED_AUDIO]} "
 775                                 L"{[key_only:KEY_ONLY]} "
 776                                 L"{CHANNEL_LAYOUT [channel_layout:string]}");
 777         sink.para()->text(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 778         sink.definitions()
 779                 ->item(L"device_index", L"The Blackmagic video card to use (See Blackmagic control panel for card order). Default is 1.")
 780                 ->item(L"keyer",
 781                                 L"If given tries to enable either internal or external keying. Not all Blackmagic cards supports this. "
 782                                 L"There is also a third experimental option (EXTERNAL_SEPARATE_DEVICE_KEY) which allocates device_index + 1 for synhronized key output.")
 783                 ->item(L"low_latency", L"Tries to enable low latency if given.")
 784                 ->item(L"embedded_audio", L"Embeds the audio into the SDI signal if given.")
 785                 ->item(L"key_only",
 786                                 L" will extract only the alpha channel from the "
 787                                 L"channel. This is useful when you have two SDI video cards, and neither has native support "
 788                                 L"for separate fill/key output")
 789                 ->item(L"channel_layout", L"If specified, overrides the audio channel layout used by the channel.");
 790         sink.para()->text(L"Examples:");
 791         sink.example(L">> ADD 1 DECKLINK", L"for using the default device_index of 1.");
 792         sink.example(L">> ADD 1 DECKLINK 2", L"uses device_index 2.");
 793         sink.example(L">> ADD 1 DECKLINK 1 EXTERNAL_KEY EMBEDDED_AUDIO");
 794         sink.example(
 795                         L">> ADD 1 DECKLINK 1 EMBEDDED_AUDIO\n"
 796                         L">> ADD 1 DECKLINK 2 KEY_ONLY", L"uses device with index 1 as fill output with audio and device with index 2 as key output.");
 797         sink.example(
 798                         L">> ADD 1 DECKLINK 1 EXTERNAL_SEPARATE_DEVICE_KEY EMBEDDED_AUDIO",
 799                         L"Uses device 2 for key output. May give better sync between key and fill than the previous method.");
 800 }
 801
 802 spl::shared_ptr<core::frame_consumer> create_consumer(
 803                 const std::vector<std::wstring>& params, core::interaction_sink*)
 804 {
 805         if (params.size() < 1 || !boost::iequals(params.at(0), L"DECKLINK"))
 806                 return core::frame_consumer::empty();
 807
 808         configuration config;
 809
 810         if (params.size() > 1)
 811                 config.device_index = boost::lexical_cast<int>(params.at(1));
 812
 813         if (contains_param(L"INTERNAL_KEY", params))
 814                 config.keyer = configuration::keyer_t::internal_keyer;
 815         else if (contains_param(L"EXTERNAL_KEY", params))
 816                 config.keyer = configuration::keyer_t::external_keyer;
 817         else if (contains_param(L"EXTERNAL_SEPARATE_DEVICE_KEY", params))
 818                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 819         else
 820                 config.keyer = configuration::keyer_t::default_keyer;
 821
 822         if (contains_param(L"LOW_LATENCY", params))
 823                 config.latency = configuration::latency_t::low_latency;
 824
 825         config.embedded_audio   = contains_param(L"EMBEDDED_AUDIO", params);
 826         config.key_only                 = contains_param(L"KEY_ONLY", params);
 827
 828         auto channel_layout = get_param(L"CHANNEL_LAYOUT", params);
 829
 830         if (!channel_layout.empty())
 831         {
 832                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(channel_layout);
 833
 834                 if (!found_layout)
 835                         CASPAR_THROW_EXCEPTION(file_not_found() << msg_info(L"Channel layout " + channel_layout + L" not found."));
 836
 837                 config.out_channel_layout = *found_layout;
 838         }
 839
 840         return spl::make_shared<decklink_consumer_proxy>(config);
 841 }
 842
 843 spl::shared_ptr<core::frame_consumer> create_preconfigured_consumer(
 844                 const boost::property_tree::wptree& ptree, core::interaction_sink*)
 845 {
 846         configuration config;
 847
 848         auto keyer = ptree.get(L"keyer", L"default");
 849         if(keyer == L"external")
 850                 config.keyer = configuration::keyer_t::external_keyer;
 851         else if(keyer == L"internal")
 852                 config.keyer = configuration::keyer_t::internal_keyer;
 853         else if (keyer == L"external_separate_device")
 854                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 855
 856         auto latency = ptree.get(L"latency", L"normal");
 857         if(latency == L"low")
 858                 config.latency = configuration::latency_t::low_latency;
 859         else if(latency == L"normal")
 860                 config.latency = configuration::latency_t::normal_latency;
 861
 862         auto channel_layout = ptree.get_optional<std::wstring>(L"channel-layout");
 863
 864         if (channel_layout)
 865         {
 866                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(*channel_layout);
 867
 868                 if (!found_layout)
 869                         CASPAR_THROW_EXCEPTION(file_not_found() << msg_info(L"Channel layout " + *channel_layout + L" not found."));
 870
 871                 config.out_channel_layout = *found_layout;
 872         }
 873
 874         config.key_only                         = ptree.get(L"key-only",                config.key_only);
 875         config.device_index                     = ptree.get(L"device",                  config.device_index);
 876         config.key_device_idx           = ptree.get(L"key-device",              config.key_device_idx);
 877         config.embedded_audio           = ptree.get(L"embedded-audio",  config.embedded_audio);
 878         config.base_buffer_depth        = ptree.get(L"buffer-depth",    config.base_buffer_depth);
 879
 880         return spl::make_shared<decklink_consumer_proxy>(config);
 881 }
 882
 883 }}
 884
 885 /*
 886 ##############################################################################
 887 Pre-rolling
 888
 889 Mail: 2011-05-09
 890
 891 Yoshan
 892 BMD Developer Support
 893 developer@blackmagic-design.com
 894
 895 -----------------------------------------------------------------------------
 896
 897 Thanks for your inquiry. The minimum number of frames that you can preroll
 898 for scheduled playback is three frames for video and four frames for audio.
 899 As you mentioned if you preroll less frames then playback will not start or
 900 playback will be very sporadic. From our experience with Media Express, we
 901 recommended that at least seven frames are prerolled for smooth playback.
 902
 903 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
 904 There can be around 3 frames worth of latency on scheduled output.
 905 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
 906 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync()
 907 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
 908 guarantee that the provided frame will be output as soon the previous
 909 frame output has been completed.
 910 ################################################################################
 911 */
 912
 913 /*
 914 ##############################################################################
 915 Async DMA Transfer without redundant copying
 916
 917 Mail: 2011-05-10
 918
 919 Yoshan
 920 BMD Developer Support
 921 developer@blackmagic-design.com
 922
 923 -----------------------------------------------------------------------------
 924
 925 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
 926 and providing a pointer to your video buffer when GetBytes() is called.
 927 This may help to keep copying to a minimum. Please ensure that the pixel
 928 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
 929 have to colourspace convert which may result in additional copying.
 930 ################################################################################
 931 */