git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Robert Nagy, ronag89@gmail.com
  20 */
  21
  22 #include "../StdAfx.h"
  23
  24 #include "decklink_consumer.h"
  25
  26 #include "../util/util.h"
  27
  28 #include "../decklink_api.h"
  29
  30 #include <core/frame/frame.h>
  31 #include <core/frame/audio_channel_layout.h>
  32 #include <core/mixer/audio/audio_mixer.h>
  33 #include <core/consumer/frame_consumer.h>
  34 #include <core/diagnostics/call_context.h>
  35 #include <core/help/help_sink.h>
  36 #include <core/help/help_repository.h>
  37
  38 #include <common/executor.h>
  39 #include <common/lock.h>
  40 #include <common/diagnostics/graph.h>
  41 #include <common/except.h>
  42 #include <common/memshfl.h>
  43 #include <common/memcpy.h>
  44 #include <common/no_init_proxy.h>
  45 #include <common/array.h>
  46 #include <common/future.h>
  47 #include <common/cache_aligned_vector.h>
  48 #include <common/timer.h>
  49 #include <common/param.h>
  50
  51 #include <tbb/concurrent_queue.h>
  52
  53 #include <common/assert.h>
  54 #include <boost/lexical_cast.hpp>
  55 #include <boost/circular_buffer.hpp>
  56 #include <boost/property_tree/ptree.hpp>
  57
  58 namespace caspar { namespace decklink {
  59
  60 struct configuration
  61 {
  62         enum class keyer_t
  63         {
  64                 internal_keyer,
  65                 external_keyer,
  66                 external_separate_device_keyer,
  67                 default_keyer                                   = external_keyer
  68         };
  69
  70         enum class latency_t
  71         {
  72                 low_latency,
  73                 normal_latency,
  74                 default_latency = normal_latency
  75         };
  76
  77         int                                                     device_index            = 1;
  78         int                                                     key_device_idx          = 0;
  79         bool                                            embedded_audio          = true;
  80         keyer_t                                         keyer                           = keyer_t::default_keyer;
  81         latency_t                                       latency                         = latency_t::default_latency;
  82         bool                                            key_only                        = false;
  83         int                                                     base_buffer_depth       = 3;
  84         core::audio_channel_layout      out_channel_layout      = core::audio_channel_layout::invalid();
  85
  86         int buffer_depth() const
  87         {
  88                 return base_buffer_depth + (latency == latency_t::low_latency ? 0 : 1) + (embedded_audio ? 1 : 0);
  89         }
  90
  91         int key_device_index() const
  92         {
  93                 return key_device_idx == 0 ? device_index + 1 : key_device_idx;
  94         }
  95
  96         core::audio_channel_layout get_adjusted_layout(const core::audio_channel_layout& in_layout) const
  97         {
  98                 auto adjusted = out_channel_layout == core::audio_channel_layout::invalid() ? in_layout : out_channel_layout;
  99
 100                 if (adjusted.num_channels == 1) // Duplicate mono-signal into both left and right.
 101                 {
 102                         adjusted.num_channels = 2;
 103                         adjusted.channel_order.push_back(adjusted.channel_order.at(0)); // Usually FC -> FC FC
 104                 }
 105                 else if (adjusted.num_channels == 2)
 106                 {
 107                         adjusted.num_channels = 2;
 108                 }
 109                 else if (adjusted.num_channels <= 8)
 110                 {
 111                         adjusted.num_channels = 8;
 112                 }
 113                 else // Over 8 always pad to 16 or drop >16
 114                 {
 115                         adjusted.num_channels = 16;
 116                 }
 117
 118                 return adjusted;
 119         }
 120 };
 121
 122 void set_latency(
 123                 const com_iface_ptr<IDeckLinkConfiguration>& config,
 124                 configuration::latency_t latency,
 125                 const std::wstring& print)
 126 {
 127         if (latency == configuration::latency_t::low_latency)
 128         {
 129                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);
 130                 CASPAR_LOG(info) << print << L" Enabled low-latency mode.";
 131         }
 132         else if (latency == configuration::latency_t::normal_latency)
 133         {
 134                 config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);
 135                 CASPAR_LOG(info) << print << L" Disabled low-latency mode.";
 136         }
 137 }
 138
 139 void set_keyer(
 140                 const com_iface_ptr<IDeckLinkAttributes>& attributes,
 141                 const com_iface_ptr<IDeckLinkKeyer>& decklink_keyer,
 142                 configuration::keyer_t keyer,
 143                 const std::wstring& print)
 144 {
 145         if (keyer == configuration::keyer_t::internal_keyer)
 146         {
 147                 BOOL value = true;
 148                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsInternalKeying, &value)) && !value)
 149                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 150                 else if (FAILED(decklink_keyer->Enable(FALSE)))
 151                         CASPAR_LOG(error) << print << L" Failed to enable internal keyer.";
 152                 else if (FAILED(decklink_keyer->SetLevel(255)))
 153                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 154                 else
 155                         CASPAR_LOG(info) << print << L" Enabled internal keyer.";
 156         }
 157         else if (keyer == configuration::keyer_t::external_keyer)
 158         {
 159                 BOOL value = true;
 160                 if (SUCCEEDED(attributes->GetFlag(BMDDeckLinkSupportsExternalKeying, &value)) && !value)
 161                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 162                 else if (FAILED(decklink_keyer->Enable(TRUE)))
 163                         CASPAR_LOG(error) << print << L" Failed to enable external keyer.";
 164                 else if (FAILED(decklink_keyer->SetLevel(255)))
 165                         CASPAR_LOG(error) << print << L" Failed to set key-level to max.";
 166                 else
 167                         CASPAR_LOG(info) << print << L" Enabled external keyer.";
 168         }
 169 }
 170
 171 class decklink_frame : public IDeckLinkVideoFrame
 172 {
 173         tbb::atomic<int>                                                                ref_count_;
 174         core::const_frame                                                               frame_;
 175         const core::video_format_desc                                   format_desc_;
 176
 177         const bool                                                                              key_only_;
 178         bool                                                                                    needs_to_copy_;
 179         cache_aligned_vector<no_init_proxy<uint8_t>>    data_;
 180 public:
 181         decklink_frame(core::const_frame frame, const core::video_format_desc& format_desc, bool key_only, bool will_attempt_dma)
 182                 : frame_(frame)
 183                 , format_desc_(format_desc)
 184                 , key_only_(key_only)
 185         {
 186                 ref_count_ = 0;
 187
 188 #if !defined(_MSC_VER)
 189                 // On Linux Decklink cannot DMA transfer from memory returned by glMapBuffer
 190                 needs_to_copy_ = will_attempt_dma;
 191 #else
 192                 // On Windows it does
 193                 needs_to_copy_ = false;
 194 #endif
 195         }
 196
 197         // IUnknown
 198
 199         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)
 200         {
 201                 return E_NOINTERFACE;
 202         }
 203
 204         virtual ULONG STDMETHODCALLTYPE AddRef()
 205         {
 206                 return ++ref_count_;
 207         }
 208
 209         virtual ULONG STDMETHODCALLTYPE Release()
 210         {
 211                 if(--ref_count_ == 0)
 212                         delete this;
 213                 return ref_count_;
 214         }
 215
 216         // IDecklinkVideoFrame
 217
 218         virtual long STDMETHODCALLTYPE GetWidth()                   {return static_cast<long>(format_desc_.width);}
 219         virtual long STDMETHODCALLTYPE GetHeight()                  {return static_cast<long>(format_desc_.height);}
 220         virtual long STDMETHODCALLTYPE GetRowBytes()                {return static_cast<long>(format_desc_.width*4);}
 221         virtual BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat()   {return bmdFormat8BitBGRA;}
 222         virtual BMDFrameFlags STDMETHODCALLTYPE GetFlags()                      {return bmdFrameFlagDefault;}
 223
 224         virtual HRESULT STDMETHODCALLTYPE GetBytes(void** buffer)
 225         {
 226                 try
 227                 {
 228                         if(static_cast<int>(frame_.image_data().size()) != format_desc_.size)
 229                         {
 230                                 data_.resize(format_desc_.size);
 231                                 *buffer = data_.data();
 232                         }
 233                         else if(key_only_)
 234                         {
 235                                 if(data_.empty())
 236                                 {
 237                                         data_.resize(frame_.image_data().size());
 238                                         aligned_memshfl(data_.data(), frame_.image_data().begin(), frame_.image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
 239                                 }
 240                                 *buffer = data_.data();
 241                         }
 242                         else
 243                         {
 244                                 *buffer = const_cast<uint8_t*>(frame_.image_data().begin());
 245
 246                                 if (needs_to_copy_)
 247                                 {
 248                                         data_.resize(frame_.image_data().size());
 249                                         fast_memcpy(data_.data(), *buffer, frame_.image_data().size());
 250                                         *buffer = data_.data();
 251                                 }
 252                         }
 253                 }
 254                 catch(...)
 255                 {
 256                         CASPAR_LOG_CURRENT_EXCEPTION();
 257                         return E_FAIL;
 258                 }
 259
 260                 return S_OK;
 261         }
 262
 263         virtual HRESULT STDMETHODCALLTYPE GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode) {return S_FALSE;}
 264         virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary)              {return S_FALSE;}
 265
 266         // decklink_frame
 267
 268         const core::audio_buffer& audio_data()
 269         {
 270                 return frame_.audio_data();
 271         }
 272
 273         int64_t get_age_millis() const
 274         {
 275                 return frame_.get_age_millis();
 276         }
 277 };
 278
 279 struct key_video_context : public IDeckLinkVideoOutputCallback, boost::noncopyable
 280 {
 281         const configuration                                                                             config_;
 282         com_ptr<IDeckLink>                                                                              decklink_                                       = get_device(config_.key_device_index());
 283         com_iface_ptr<IDeckLinkOutput>                                                  output_                                         = iface_cast<IDeckLinkOutput>(decklink_);
 284         com_iface_ptr<IDeckLinkKeyer>                                                   keyer_                                          = iface_cast<IDeckLinkKeyer>(decklink_);
 285         com_iface_ptr<IDeckLinkAttributes>                                              attributes_                                     = iface_cast<IDeckLinkAttributes>(decklink_);
 286         com_iface_ptr<IDeckLinkConfiguration>                                   configuration_                          = iface_cast<IDeckLinkConfiguration>(decklink_);
 287         tbb::atomic<int64_t>                                                                    current_presentation_delay_;
 288         tbb::atomic<int64_t>                                                                    scheduled_frames_completed_;
 289
 290         key_video_context(const configuration& config, const std::wstring& print)
 291                 : config_(config)
 292         {
 293                 current_presentation_delay_ = 0;
 294                 scheduled_frames_completed_ = 0;
 295
 296                 set_latency(configuration_, config.latency, print);
 297                 set_keyer(attributes_, keyer_, config.keyer, print);
 298
 299                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 300                         CASPAR_THROW_EXCEPTION(caspar_exception()
 301                                         << msg_info(print + L" Failed to set key playback completion callback.")
 302                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 303         }
 304
 305         template<typename Print>
 306         void enable_video(BMDDisplayMode display_mode, const Print& print)
 307         {
 308                 if (FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 309                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable key video output."));
 310
 311                 if (FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 312                         CASPAR_THROW_EXCEPTION(caspar_exception()
 313                                         << msg_info(print() + L" Failed to set key playback completion callback.")
 314                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 315         }
 316
 317         virtual ~key_video_context()
 318         {
 319                 if (output_)
 320                 {
 321                         output_->StopScheduledPlayback(0, nullptr, 0);
 322                         output_->DisableVideoOutput();
 323                 }
 324         }
 325
 326         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 327         virtual ULONG STDMETHODCALLTYPE AddRef()                                                        {return 1;}
 328         virtual ULONG STDMETHODCALLTYPE Release()                                                       {return 1;}
 329
 330         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 331         {
 332                 return S_OK;
 333         }
 334
 335         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(
 336                         IDeckLinkVideoFrame* completed_frame,
 337                         BMDOutputFrameCompletionResult result)
 338         {
 339                 auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 340                 current_presentation_delay_ = dframe->get_age_millis();
 341                 ++scheduled_frames_completed_;
 342
 343                 // Let the fill callback keep the pace, so no scheduling here.
 344
 345                 return S_OK;
 346         }
 347 };
 348
 349 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable
 350 {
 351         const int                                                                                       channel_index_;
 352         const configuration                                                                     config_;
 353
 354         com_ptr<IDeckLink>                                                                      decklink_                               = get_device(config_.device_index);
 355         com_iface_ptr<IDeckLinkOutput>                                          output_                                 = iface_cast<IDeckLinkOutput>(decklink_);
 356         com_iface_ptr<IDeckLinkConfiguration>                           configuration_                  = iface_cast<IDeckLinkConfiguration>(decklink_);
 357         com_iface_ptr<IDeckLinkKeyer>                                           keyer_                                  = iface_cast<IDeckLinkKeyer>(decklink_);
 358         com_iface_ptr<IDeckLinkAttributes>                                      attributes_                             = iface_cast<IDeckLinkAttributes>(decklink_);
 359
 360         tbb::spin_mutex                                                                         exception_mutex_;
 361         std::exception_ptr                                                                      exception_;
 362
 363         tbb::atomic<bool>                                                                       is_running_;
 364
 365         const std::wstring                                                                      model_name_                             = get_model_name(decklink_);
 366         bool                                                                                            will_attempt_dma_;
 367         const core::video_format_desc                                           format_desc_;
 368         const core::audio_channel_layout                                        in_channel_layout_;
 369         const core::audio_channel_layout                                        out_channel_layout_             = config_.get_adjusted_layout(in_channel_layout_);
 370         core::audio_channel_remapper                                            channel_remapper_               { in_channel_layout_, out_channel_layout_ };
 371         const int                                                                                       buffer_size_                    = config_.buffer_depth(); // Minimum buffer-size 3.
 372
 373         long long                                                                                       video_scheduled_                = 0;
 374         long long                                                                                       audio_scheduled_                = 0;
 375
 376         int                                                                                                     preroll_count_                  = 0;
 377
 378         boost::circular_buffer<std::vector<int32_t>>            audio_container_                { buffer_size_ + 1 };
 379
 380         tbb::concurrent_bounded_queue<core::const_frame>        video_frame_buffer_;
 381         tbb::concurrent_bounded_queue<core::const_frame>        audio_frame_buffer_;
 382
 383         spl::shared_ptr<diagnostics::graph>                                     graph_;
 384         caspar::timer                                                                           tick_timer_;
 385         retry_task<bool>                                                                        send_completion_;
 386         reference_signal_detector                                                       reference_signal_detector_      { output_ };
 387         tbb::atomic<int64_t>                                                            current_presentation_delay_;
 388         tbb::atomic<int64_t>                                                            scheduled_frames_completed_;
 389         std::unique_ptr<key_video_context>                                      key_context_;
 390
 391 public:
 392         decklink_consumer(
 393                         const configuration& config,
 394                         const core::video_format_desc& format_desc,
 395                         const core::audio_channel_layout& in_channel_layout,
 396                         int channel_index)
 397                 : channel_index_(channel_index)
 398                 , config_(config)
 399                 , format_desc_(format_desc)
 400                 , in_channel_layout_(in_channel_layout)
 401         {
 402                 is_running_ = true;
 403                 current_presentation_delay_ = 0;
 404                 scheduled_frames_completed_ = 0;
 405
 406                 video_frame_buffer_.set_capacity(1);
 407
 408                 // Blackmagic calls RenderAudioSamples() 50 times per second
 409                 // regardless of video mode so we sometimes need to give them
 410                 // samples from 2 frames in order to keep up
 411                 audio_frame_buffer_.set_capacity((format_desc.fps > 50.0) ? 2 : 1);
 412
 413                 if (config.keyer == configuration::keyer_t::external_separate_device_keyer)
 414                         key_context_.reset(new key_video_context(config, print()));
 415
 416                 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
 417                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
 418                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
 419                 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));
 420                 graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
 421                 graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));
 422
 423                 if (key_context_)
 424                 {
 425                         graph_->set_color("key-offset", diagnostics::color(1.0f, 0.0f, 0.0f));
 426                 }
 427
 428                 graph_->set_text(print());
 429                 diagnostics::register_graph(graph_);
 430
 431                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault, will_attempt_dma_));
 432
 433                 if(config.embedded_audio)
 434                         enable_audio();
 435
 436                 set_latency(configuration_, config.latency, print());
 437                 set_keyer(attributes_, keyer_, config.keyer, print());
 438
 439                 if(config.embedded_audio)
 440                         output_->BeginAudioPreroll();
 441
 442                 for(int n = 0; n < buffer_size_; ++n)
 443                         schedule_next_video(core::const_frame::empty());
 444
 445                 if(!config.embedded_audio)
 446                         start_playback();
 447         }
 448
 449         ~decklink_consumer()
 450         {
 451                 is_running_ = false;
 452                 video_frame_buffer_.try_push(core::const_frame::empty());
 453                 audio_frame_buffer_.try_push(core::const_frame::empty());
 454
 455                 if(output_ != nullptr)
 456                 {
 457                         output_->StopScheduledPlayback(0, nullptr, 0);
 458                         if(config_.embedded_audio)
 459                                 output_->DisableAudioOutput();
 460                         output_->DisableVideoOutput();
 461                 }
 462         }
 463
 464         void enable_audio()
 465         {
 466                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, out_channel_layout_.num_channels, bmdAudioOutputStreamTimestamped)))
 467                                 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable audio output."));
 468
 469                 if(FAILED(output_->SetAudioCallback(this)))
 470                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not set audio callback."));
 471
 472                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
 473         }
 474
 475         void enable_video(BMDDisplayMode display_mode)
 476         {
 477                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 478                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Could not enable fill video output."));
 479
 480                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))
 481                         CASPAR_THROW_EXCEPTION(caspar_exception()
 482                                                                         << msg_info(print() + L" Failed to set fill playback completion callback.")
 483                                                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));
 484
 485                 if (key_context_)
 486                         key_context_->enable_video(display_mode, [this]() { return print(); });
 487         }
 488
 489         void start_playback()
 490         {
 491                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 492                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule fill playback."));
 493
 494                 if (key_context_ && FAILED(key_context_->output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))
 495                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to schedule key playback."));
 496         }
 497
 498         virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID, LPVOID*)       {return E_NOINTERFACE;}
 499         virtual ULONG STDMETHODCALLTYPE AddRef()                                        {return 1;}
 500         virtual ULONG STDMETHODCALLTYPE Release()                               {return 1;}
 501
 502         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
 503         {
 504                 is_running_ = false;
 505                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";
 506                 return S_OK;
 507         }
 508
 509         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result)
 510         {
 511                 if(!is_running_)
 512                         return E_FAIL;
 513
 514                 try
 515                 {
 516                         auto dframe = reinterpret_cast<decklink_frame*>(completed_frame);
 517                         current_presentation_delay_ = dframe->get_age_millis();
 518                         ++scheduled_frames_completed_;
 519
 520                         if (key_context_)
 521                                 graph_->set_value(
 522                                                 "key-offset",
 523                                                 static_cast<double>(
 524                                                                 scheduled_frames_completed_
 525                                                                 - key_context_->scheduled_frames_completed_)
 526                                                 * 0.1 + 0.5);
 527
 528                         if(result == bmdOutputFrameDisplayedLate)
 529                         {
 530                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "late-frame");
 531                                 video_scheduled_ += format_desc_.duration;
 532                                 audio_scheduled_ += dframe->audio_data().size() / out_channel_layout_.num_channels;
 533                                 //++video_scheduled_;
 534                                 //audio_scheduled_ += format_desc_.audio_cadence[0];
 535                                 //++audio_scheduled_;
 536                         }
 537                         else if(result == bmdOutputFrameDropped)
 538                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "dropped-frame");
 539                         else if(result == bmdOutputFrameFlushed)
 540                                 graph_->set_tag(diagnostics::tag_severity::WARNING, "flushed-frame");
 541
 542                         UINT32 buffered;
 543                         output_->GetBufferedVideoFrameCount(&buffered);
 544                         graph_->set_value("buffered-video", static_cast<double>(buffered) / (config_.buffer_depth()));
 545
 546                         auto frame = core::const_frame::empty();
 547                         video_frame_buffer_.pop(frame);
 548                         send_completion_.try_completion();
 549                         schedule_next_video(frame);
 550                 }
 551                 catch(...)
 552                 {
 553                         lock(exception_mutex_, [&]
 554                         {
 555                                 exception_ = std::current_exception();
 556                         });
 557                         return E_FAIL;
 558                 }
 559
 560                 return S_OK;
 561         }
 562
 563         virtual HRESULT STDMETHODCALLTYPE RenderAudioSamples(BOOL preroll)
 564         {
 565                 if(!is_running_)
 566                         return E_FAIL;
 567
 568                 try
 569                 {
 570                         if(preroll)
 571                         {
 572                                 if(++preroll_count_ >= buffer_size_)
 573                                 {
 574                                         output_->EndAudioPreroll();
 575                                         start_playback();
 576                                 }
 577                                 else
 578                                 {
 579                                         schedule_next_audio(core::mutable_audio_buffer(format_desc_.audio_cadence[preroll % format_desc_.audio_cadence.size()] * out_channel_layout_.num_channels, 0));
 580                                 }
 581                         }
 582                         else
 583                         {
 584                                 auto frame = core::const_frame::empty();
 585
 586                                 while(audio_frame_buffer_.try_pop(frame))
 587                                 {
 588                                         UINT32 buffered;
 589                                         output_->GetBufferedAudioSampleFrameCount(&buffered);
 590                                         graph_->set_value("buffered-audio", static_cast<double>(buffered) / (format_desc_.audio_cadence[0] * config_.buffer_depth()));
 591
 592                                         send_completion_.try_completion();
 593                                         schedule_next_audio(channel_remapper_.mix_and_rearrange(frame.audio_data()));
 594                                 }
 595                         }
 596                 }
 597                 catch(...)
 598                 {
 599                         tbb::spin_mutex::scoped_lock lock(exception_mutex_);
 600                         exception_ = std::current_exception();
 601                         return E_FAIL;
 602                 }
 603
 604                 return S_OK;
 605         }
 606
 607         template<typename T>
 608         void schedule_next_audio(const T& audio_data)
 609         {
 610                 auto sample_frame_count = static_cast<int>(audio_data.size()/out_channel_layout_.num_channels);
 611
 612                 audio_container_.push_back(std::vector<int32_t>(audio_data.begin(), audio_data.end()));
 613
 614                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, audio_scheduled_, format_desc_.audio_sample_rate, nullptr)))
 615                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";
 616
 617                 audio_scheduled_ += sample_frame_count;
 618         }
 619
 620         void schedule_next_video(core::const_frame frame)
 621         {
 622                 if (key_context_)
 623                 {
 624                         auto key_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, true, will_attempt_dma_));
 625                         if (FAILED(key_context_->output_->ScheduleVideoFrame(get_raw(key_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 626                                 CASPAR_LOG(error) << print() << L" Failed to schedule key video.";
 627                 }
 628
 629                 auto fill_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(new decklink_frame(frame, format_desc_, config_.key_only, will_attempt_dma_));
 630                 if (FAILED(output_->ScheduleVideoFrame(get_raw(fill_frame), video_scheduled_, format_desc_.duration, format_desc_.time_scale)))
 631                         CASPAR_LOG(error) << print() << L" Failed to schedule fill video.";
 632
 633                 video_scheduled_ += format_desc_.duration;
 634
 635                 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
 636                 tick_timer_.restart();
 637
 638                 reference_signal_detector_.detect_change([this]() { return print(); });
 639         }
 640
 641         std::future<bool> send(core::const_frame frame)
 642         {
 643                 auto exception = lock(exception_mutex_, [&]
 644                 {
 645                         return exception_;
 646                 });
 647
 648                 if(exception != nullptr)
 649                         std::rethrow_exception(exception);
 650
 651                 if(!is_running_)
 652                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Is not running."));
 653
 654                 bool audio_ready = !config_.embedded_audio;
 655                 bool video_ready = false;
 656
 657                 auto enqueue_task = [audio_ready, video_ready, frame, this]() mutable -> boost::optional<bool>
 658                 {
 659                         if (!audio_ready)
 660                                 audio_ready = audio_frame_buffer_.try_push(frame);
 661
 662                         if (!video_ready)
 663                                 video_ready = video_frame_buffer_.try_push(frame);
 664
 665                         if (audio_ready && video_ready)
 666                                 return true;
 667                         else
 668                                 return boost::optional<bool>();
 669                 };
 670
 671                 if (enqueue_task())
 672                         return make_ready_future(true);
 673
 674                 send_completion_.set_task(enqueue_task);
 675
 676                 return send_completion_.get_future();
 677         }
 678
 679         std::wstring print() const
 680         {
 681                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 682                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 683                                 boost::lexical_cast<std::wstring>(config_.device_index) +
 684                                 L"&&" +
 685                                 boost::lexical_cast<std::wstring>(config_.key_device_index()) +
 686                                 L"|" +
 687                                 format_desc_.name + L"]";
 688                 else
 689                         return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_)+L"-" +
 690                                 boost::lexical_cast<std::wstring>(config_.device_index) + L"|" + format_desc_.name + L"]";
 691         }
 692 };
 693
 694 struct decklink_consumer_proxy : public core::frame_consumer
 695 {
 696         core::monitor::subject                          monitor_subject_;
 697         const configuration                                     config_;
 698         std::unique_ptr<decklink_consumer>      consumer_;
 699         core::video_format_desc                         format_desc_;
 700         executor                                                        executor_;
 701 public:
 702
 703         decklink_consumer_proxy(const configuration& config)
 704                 : config_(config)
 705                 , executor_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
 706         {
 707                 auto ctx = core::diagnostics::call_context::for_thread();
 708                 executor_.begin_invoke([=]
 709                 {
 710                         core::diagnostics::call_context::for_thread() = ctx;
 711                         com_initialize();
 712                 });
 713         }
 714
 715         ~decklink_consumer_proxy()
 716         {
 717                 executor_.invoke([=]
 718                 {
 719                         consumer_.reset();
 720                         com_uninitialize();
 721                 });
 722         }
 723
 724         // frame_consumer
 725
 726         void initialize(const core::video_format_desc& format_desc, const core::audio_channel_layout& channel_layout, int channel_index) override
 727         {
 728                 format_desc_ = format_desc;
 729                 executor_.invoke([=]
 730                 {
 731                         consumer_.reset();
 732                         consumer_.reset(new decklink_consumer(config_, format_desc, channel_layout, channel_index));
 733                 });
 734         }
 735
 736         std::future<bool> send(core::const_frame frame) override
 737         {
 738                 return consumer_->send(frame);
 739         }
 740
 741         std::wstring print() const override
 742         {
 743                 return consumer_ ? consumer_->print() : L"[decklink_consumer]";
 744         }
 745
 746         std::wstring name() const override
 747         {
 748                 return L"decklink";
 749         }
 750
 751         boost::property_tree::wptree info() const override
 752         {
 753                 boost::property_tree::wptree info;
 754                 info.add(L"type", L"decklink");
 755                 info.add(L"key-only", config_.key_only);
 756                 info.add(L"device", config_.device_index);
 757
 758                 if (config_.keyer == configuration::keyer_t::external_separate_device_keyer)
 759                 {
 760                         info.add(L"key-device", config_.key_device_index());
 761                 }
 762
 763                 info.add(L"low-latency", config_.latency == configuration::latency_t::low_latency);
 764                 info.add(L"embedded-audio", config_.embedded_audio);
 765                 info.add(L"presentation-frame-age", presentation_frame_age_millis());
 766                 //info.add(L"internal-key", config_.internal_key);
 767                 return info;
 768         }
 769
 770         int buffer_depth() const override
 771         {
 772                 return config_.buffer_depth() + 2;
 773         }
 774
 775         int index() const override
 776         {
 777                 return 300 + config_.device_index;
 778         }
 779
 780         int64_t presentation_frame_age_millis() const override
 781         {
 782                 return consumer_ ? static_cast<int64_t>(consumer_->current_presentation_delay_) : 0;
 783         }
 784
 785         core::monitor::subject& monitor_output()
 786         {
 787                 return monitor_subject_;
 788         }
 789 };
 790
 791 void describe_consumer(core::help_sink& sink, const core::help_repository& repo)
 792 {
 793         sink.short_description(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 794         sink.syntax(L"DECKLINK "
 795                                 L"{[device_index:int]|1} "
 796                                 L"{[keyer:INTERNAL_KEY,EXTERNAL_KEY,EXTERNAL_SEPARATE_DEVICE_KEY]} "
 797                                 L"{[low_latency:LOW_LATENCY]} "
 798                                 L"{[embedded_audio:EMBEDDED_AUDIO]} "
 799                                 L"{[key_only:KEY_ONLY]} "
 800                                 L"{CHANNEL_LAYOUT [channel_layout:string]}");
 801         sink.para()->text(L"Sends video on an SDI output using Blackmagic Decklink video cards.");
 802         sink.definitions()
 803                 ->item(L"device_index", L"The Blackmagic video card to use (See Blackmagic control panel for card order). Default is 1.")
 804                 ->item(L"keyer",
 805                                 L"If given tries to enable either internal or external keying. Not all Blackmagic cards supports this. "
 806                                 L"There is also a third experimental option (EXTERNAL_SEPARATE_DEVICE_KEY) which allocates device_index + 1 for synhronized key output.")
 807                 ->item(L"low_latency", L"Tries to enable low latency if given.")
 808                 ->item(L"embedded_audio", L"Embeds the audio into the SDI signal if given.")
 809                 ->item(L"key_only",
 810                                 L" will extract only the alpha channel from the "
 811                                 L"channel. This is useful when you have two SDI video cards, and neither has native support "
 812                                 L"for separate fill/key output")
 813                 ->item(L"channel_layout", L"If specified, overrides the audio channel layout used by the channel.");
 814         sink.para()->text(L"Examples:");
 815         sink.example(L">> ADD 1 DECKLINK", L"for using the default device_index of 1.");
 816         sink.example(L">> ADD 1 DECKLINK 2", L"uses device_index 2.");
 817         sink.example(L">> ADD 1 DECKLINK 1 EXTERNAL_KEY EMBEDDED_AUDIO");
 818         sink.example(
 819                         L">> ADD 1 DECKLINK 1 EMBEDDED_AUDIO\n"
 820                         L">> ADD 1 DECKLINK 2 KEY_ONLY", L"uses device with index 1 as fill output with audio and device with index 2 as key output.");
 821         sink.example(
 822                         L">> ADD 1 DECKLINK 1 EXTERNAL_SEPARATE_DEVICE_KEY EMBEDDED_AUDIO",
 823                         L"Uses device 2 for key output. May give better sync between key and fill than the previous method.");
 824 }
 825
 826 spl::shared_ptr<core::frame_consumer> create_consumer(
 827                 const std::vector<std::wstring>& params, core::interaction_sink*)
 828 {
 829         if (params.size() < 1 || !boost::iequals(params.at(0), L"DECKLINK"))
 830                 return core::frame_consumer::empty();
 831
 832         configuration config;
 833
 834         if (params.size() > 1)
 835                 config.device_index = boost::lexical_cast<int>(params.at(1));
 836
 837         if (contains_param(L"INTERNAL_KEY", params))
 838                 config.keyer = configuration::keyer_t::internal_keyer;
 839         else if (contains_param(L"EXTERNAL_KEY", params))
 840                 config.keyer = configuration::keyer_t::external_keyer;
 841         else if (contains_param(L"EXTERNAL_SEPARATE_DEVICE_KEY", params))
 842                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 843         else
 844                 config.keyer = configuration::keyer_t::default_keyer;
 845
 846         if (contains_param(L"LOW_LATENCY", params))
 847                 config.latency = configuration::latency_t::low_latency;
 848
 849         config.embedded_audio   = contains_param(L"EMBEDDED_AUDIO", params);
 850         config.key_only                 = contains_param(L"KEY_ONLY", params);
 851
 852         auto channel_layout = get_param(L"CHANNEL_LAYOUT", params);
 853
 854         if (!channel_layout.empty())
 855         {
 856                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(channel_layout);
 857
 858                 if (!found_layout)
 859                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + channel_layout + L" not found."));
 860
 861                 config.out_channel_layout = *found_layout;
 862         }
 863
 864         return spl::make_shared<decklink_consumer_proxy>(config);
 865 }
 866
 867 spl::shared_ptr<core::frame_consumer> create_preconfigured_consumer(
 868                 const boost::property_tree::wptree& ptree, core::interaction_sink*)
 869 {
 870         configuration config;
 871
 872         auto keyer = ptree.get(L"keyer", L"default");
 873         if(keyer == L"external")
 874                 config.keyer = configuration::keyer_t::external_keyer;
 875         else if(keyer == L"internal")
 876                 config.keyer = configuration::keyer_t::internal_keyer;
 877         else if (keyer == L"external_separate_device")
 878                 config.keyer = configuration::keyer_t::external_separate_device_keyer;
 879
 880         auto latency = ptree.get(L"latency", L"default");
 881         if(latency == L"low")
 882                 config.latency = configuration::latency_t::low_latency;
 883         else if(latency == L"normal")
 884                 config.latency = configuration::latency_t::normal_latency;
 885
 886         auto channel_layout = ptree.get_optional<std::wstring>(L"channel-layout");
 887
 888         if (channel_layout)
 889         {
 890                 CASPAR_SCOPED_CONTEXT_MSG("/channel-layout")
 891
 892                 auto found_layout = core::audio_channel_layout_repository::get_default()->get_layout(*channel_layout);
 893
 894                 if (!found_layout)
 895                         CASPAR_THROW_EXCEPTION(user_error() << msg_info(L"Channel layout " + *channel_layout + L" not found."));
 896
 897                 config.out_channel_layout = *found_layout;
 898         }
 899
 900         config.key_only                         = ptree.get(L"key-only",                config.key_only);
 901         config.device_index                     = ptree.get(L"device",                  config.device_index);
 902         config.key_device_idx           = ptree.get(L"key-device",              config.key_device_idx);
 903         config.embedded_audio           = ptree.get(L"embedded-audio",  config.embedded_audio);
 904         config.base_buffer_depth        = ptree.get(L"buffer-depth",    config.base_buffer_depth);
 905
 906         return spl::make_shared<decklink_consumer_proxy>(config);
 907 }
 908
 909 }}
 910
 911 /*
 912 ##############################################################################
 913 Pre-rolling
 914
 915 Mail: 2011-05-09
 916
 917 Yoshan
 918 BMD Developer Support
 919 developer@blackmagic-design.com
 920
 921 -----------------------------------------------------------------------------
 922
 923 Thanks for your inquiry. The minimum number of frames that you can preroll
 924 for scheduled playback is three frames for video and four frames for audio.
 925 As you mentioned if you preroll less frames then playback will not start or
 926 playback will be very sporadic. From our experience with Media Express, we
 927 recommended that at least seven frames are prerolled for smooth playback.
 928
 929 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:
 930 There can be around 3 frames worth of latency on scheduled output.
 931 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is
 932 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync()
 933 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will
 934 guarantee that the provided frame will be output as soon the previous
 935 frame output has been completed.
 936 ################################################################################
 937 */
 938
 939 /*
 940 ##############################################################################
 941 Async DMA Transfer without redundant copying
 942
 943 Mail: 2011-05-10
 944
 945 Yoshan
 946 BMD Developer Support
 947 developer@blackmagic-design.com
 948
 949 -----------------------------------------------------------------------------
 950
 951 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame
 952 and providing a pointer to your video buffer when GetBytes() is called.
 953 This may help to keep copying to a minimum. Please ensure that the pixel
 954 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will
 955 have to colourspace convert which may result in additional copying.
 956 ################################################################################
 957 */