git.sesse.net Git - casparcg/blob - modules/decklink/consumer/blocking_decklink_consumer.cpp

   1 /*
   2 * Copyright 2013 Sveriges Television AB http://casparcg.com/
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Robert Nagy, ronag89@gmail.com
  20 */
  21
  22 #include "../StdAfx.h"
  23
  24 #include "decklink_consumer.h"
  25
  26 #include "../util/util.h"
  27 #include "../util/decklink_allocator.h"
  28
  29 #include "../interop/DeckLinkAPI_h.h"
  30
  31 #include <map>
  32
  33 #include <common/concurrency/com_context.h>
  34 #include <common/concurrency/executor.h>
  35 #include <common/diagnostics/graph.h>
  36 #include <common/exception/exceptions.h>
  37 #include <common/memory/memcpy.h>
  38 #include <common/memory/memclr.h>
  39 #include <common/utility/timer.h>
  40
  41 #include <core/parameters/parameters.h>
  42 #include <core/consumer/frame_consumer.h>
  43 #include <core/mixer/read_frame.h>
  44 #include <core/mixer/audio/audio_util.h>
  45
  46 #include <tbb/cache_aligned_allocator.h>
  47
  48 #include <boost/timer.hpp>
  49 #include <boost/property_tree/ptree.hpp>
  50 #include <boost/algorithm/string.hpp>
  51 #include <boost/circular_buffer.hpp>
  52
  53 namespace caspar { namespace decklink {
  54
  55 // second is the index in the array to consume the next time
  56 typedef std::pair<std::vector<int32_t>, size_t> audio_buffer;
  57
  58 struct blocking_decklink_consumer : boost::noncopyable
  59 {
  60         const int                                                               channel_index_;
  61         const configuration                                             config_;
  62
  63         std::unique_ptr<thread_safe_decklink_allocator> allocator_;
  64         CComPtr<IDeckLink>                                              decklink_;
  65         CComQIPtr<IDeckLinkOutput>                              output_;
  66         CComQIPtr<IDeckLinkKeyer>                               keyer_;
  67         CComQIPtr<IDeckLinkAttributes>                  attributes_;
  68         CComQIPtr<IDeckLinkConfiguration>               configuration_;
  69
  70         const std::wstring                                              model_name_;
  71         const core::video_format_desc                   format_desc_;
  72         std::shared_ptr<core::read_frame>               previous_frame_;
  73         boost::circular_buffer<audio_buffer>    audio_samples_;
  74         size_t                                                                  buffered_audio_samples_;
  75         BMDTimeValue                                                    last_reference_clock_value_;
  76
  77         safe_ptr<diagnostics::graph>                    graph_;
  78         boost::timer                                                    frame_timer_;
  79         boost::timer                                                    tick_timer_;
  80         boost::timer                                                    sync_timer_;
  81         reference_signal_detector                               reference_signal_detector_;
  82
  83         tbb::atomic<int64_t>                                    current_presentation_delay_;
  84         executor                                                                executor_;
  85
  86 public:
  87         blocking_decklink_consumer(const configuration& config, const core::video_format_desc& format_desc, int channel_index)
  88                 : channel_index_(channel_index)
  89                 , config_(config)
  90                 , decklink_(get_device(config.device_index))
  91                 , output_(decklink_)
  92                 , keyer_(decklink_)
  93                 , attributes_(decklink_)
  94                 , configuration_(decklink_)
  95                 , model_name_(get_model_name(decklink_))
  96                 , format_desc_(format_desc)
  97                 , buffered_audio_samples_(0)
  98                 , last_reference_clock_value_(-1)
  99                 , reference_signal_detector_(output_)
 100                 , executor_(L"blocking_decklink_consumer")
 101         {
 102                 audio_samples_.set_capacity(2);
 103                 current_presentation_delay_ = 0;
 104                 executor_.set_capacity(1);
 105
 106                 graph_->set_color("sync-time", diagnostics::color(1.0f, 0.0f, 0.0f));
 107                 graph_->set_color("frame-time", diagnostics::color(0.5f, 1.0f, 0.2f));
 108                 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
 109                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));
 110
 111                 if (config_.embedded_audio)
 112                         graph_->set_color(
 113                                         "buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
 114
 115                 graph_->set_text(print());
 116                 diagnostics::register_graph(graph_);
 117
 118                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));
 119
 120                 if(config.embedded_audio)
 121                         enable_audio();
 122
 123                 set_latency(configuration_, configuration::low_latency, print());
 124                 set_keyer(attributes_, keyer_, config.keyer, print());
 125         }
 126
 127         ~blocking_decklink_consumer()
 128         {
 129                 if(output_ != nullptr)
 130                 {
 131                         if(config_.embedded_audio)
 132                                 output_->DisableAudioOutput();
 133                         output_->DisableVideoOutput();
 134                 }
 135         }
 136
 137         void enable_audio()
 138         {
 139                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, config_.num_out_channels(), bmdAudioOutputStreamContinuous)))
 140                                 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not enable audio output."));
 141
 142                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";
 143         }
 144
 145         void enable_video(BMDDisplayMode display_mode)
 146         {
 147                 if (config_.custom_allocator)
 148                 {
 149                         allocator_.reset(new thread_safe_decklink_allocator(print()));
 150
 151                         if (FAILED(output_->SetVideoOutputFrameMemoryAllocator(allocator_.get())))
 152                                 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not set custom memory allocator."));
 153
 154                         CASPAR_LOG(info) << print() << L" Using custom allocator.";
 155                 }
 156
 157                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault)))
 158                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not enable video output."));
 159         }
 160
 161         void queue_audio_samples(const safe_ptr<core::read_frame>& frame)
 162         {
 163                 auto view = frame->multichannel_view();
 164                 const int sample_frame_count = view.num_samples();
 165
 166                 if (audio_samples_.full())
 167                 {
 168                         CASPAR_LOG(warning) << print() << L" Too much audio buffered. Discarding samples.";
 169                         audio_samples_.clear();
 170                         buffered_audio_samples_ = 0;
 171                 }
 172
 173                 if (core::needs_rearranging(
 174                                 view, config_.audio_layout, config_.num_out_channels()))
 175                 {
 176                         std::vector<int32_t> resulting_audio_data;
 177                         resulting_audio_data.resize(
 178                                         sample_frame_count * config_.num_out_channels());
 179
 180                         auto dest_view = core::make_multichannel_view<int32_t>(
 181                                         resulting_audio_data.begin(),
 182                                         resulting_audio_data.end(),
 183                                         config_.audio_layout,
 184                                         config_.num_out_channels());
 185
 186                         core::rearrange_or_rearrange_and_mix(
 187                                         view, dest_view, core::default_mix_config_repository());
 188
 189                         if (config_.audio_layout.num_channels == 1) // mono
 190                                 boost::copy(                            // duplicate L to R
 191                                                 dest_view.channel(0),
 192                                                 dest_view.channel(1).begin());
 193
 194                         audio_samples_.push_back(
 195                                         std::make_pair(std::move(resulting_audio_data), 0));
 196                 }
 197                 else
 198                 {
 199                         audio_samples_.push_back(std::make_pair(
 200                                         std::vector<int32_t>(
 201                                                         frame->audio_data().begin(),
 202                                                         frame->audio_data().end()),
 203                                         0));
 204                 }
 205
 206                 buffered_audio_samples_ += sample_frame_count;
 207                 graph_->set_value("buffered-audio",
 208                                 static_cast<double>(buffered_audio_samples_)
 209                                                 / format_desc_.audio_cadence[0] * 0.5);
 210         }
 211
 212         bool try_consume_audio(std::pair<std::vector<int32_t>, size_t>& buffer)
 213         {
 214                 size_t to_offer = (buffer.first.size() - buffer.second)
 215                                 / config_.num_out_channels();
 216                 auto begin = buffer.first.data() + buffer.second;
 217                 unsigned long samples_written;
 218
 219                 if (FAILED(output_->WriteAudioSamplesSync(
 220                                 begin,
 221                                 to_offer,
 222                                 &samples_written)))
 223                         CASPAR_LOG(error) << print() << L" Failed to write audio samples.";
 224
 225                 buffered_audio_samples_ -= samples_written;
 226
 227                 if (samples_written == to_offer)
 228                         return true; // depleted buffer
 229
 230                 size_t consumed = samples_written * config_.num_out_channels();
 231                 buffer.second += consumed;
 232
 233
 234                 return false;
 235         }
 236
 237         void write_audio_samples()
 238         {
 239                 while (!audio_samples_.empty())
 240                 {
 241                         auto buffer = audio_samples_.front();
 242
 243                         if (try_consume_audio(buffer))
 244                                 audio_samples_.pop_front();
 245                         else
 246                                 break;
 247                 }
 248         }
 249
 250         void wait_for_frame_to_be_displayed()
 251         {
 252                 sync_timer_.restart();
 253                 BMDTimeScale time_scale = 1000;
 254                 BMDTimeValue hardware_time;
 255                 BMDTimeValue time_in_frame;
 256                 BMDTimeValue ticks_per_frame;
 257
 258                 if (FAILED(output_->GetHardwareReferenceClock(
 259                                 time_scale, &hardware_time, &time_in_frame, &ticks_per_frame)))
 260                         CASPAR_LOG(error) << print() << L" Failed to determine time in frame.";
 261
 262                 auto reference_clock_value = hardware_time - time_in_frame;
 263                 auto frame_duration = static_cast<int>(1000 / format_desc_.fps);
 264                 auto actual_duration = last_reference_clock_value_ == -1
 265                                 ? frame_duration
 266                                 : reference_clock_value - last_reference_clock_value_;
 267
 268                 if (std::abs(frame_duration - actual_duration) > 1)
 269                         graph_->set_tag("late-frame");
 270                 else
 271                 {
 272                         auto to_wait = ticks_per_frame - time_in_frame;
 273                         high_prec_timer timer;
 274                         timer.tick_millis(0);
 275                         timer.tick_millis(static_cast<DWORD>(to_wait));
 276                 }
 277
 278
 279                 last_reference_clock_value_ = reference_clock_value;
 280                 graph_->set_value("sync-time",
 281                                 sync_timer_.elapsed() * format_desc_.fps * 0.5);
 282         }
 283
 284         void write_video_frame(
 285                         const safe_ptr<core::read_frame>& frame,
 286                         std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>>&& key)
 287         {
 288                 CComPtr<IDeckLinkVideoFrame> frame2;
 289
 290                 if (config_.key_only)
 291                         frame2 = CComPtr<IDeckLinkVideoFrame>(
 292                                         new decklink_frame(frame, format_desc_, std::move(key)));
 293                 else
 294                         frame2 = CComPtr<IDeckLinkVideoFrame>(
 295                                         new decklink_frame(frame, format_desc_, config_.key_only));
 296
 297                 if (FAILED(output_->DisplayVideoFrameSync(frame2)))
 298                         CASPAR_LOG(error) << print() << L" Failed to display video frame.";
 299
 300                 reference_signal_detector_.detect_change([this]() { return print(); });
 301         }
 302
 303         boost::unique_future<bool> send(const safe_ptr<core::read_frame>& frame)
 304         {
 305                 return executor_.begin_invoke([=]() -> bool
 306                 {
 307                         frame_timer_.restart();
 308                         std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>> key;
 309
 310                         if (config_.key_only)
 311                                 key = std::move(extract_key(frame));
 312
 313                         if (config_.embedded_audio)
 314                                 queue_audio_samples(frame);
 315
 316                         double frame_time = frame_timer_.elapsed();
 317
 318                         wait_for_frame_to_be_displayed();
 319
 320                         if (previous_frame_)
 321                         {
 322                                 // According to SDK when DisplayVideoFrameSync is used in
 323                                 // combination with low latency, the next frame displayed should
 324                                 // be the submitted frame but it appears to be an additional 2
 325                                 // frame delay before it is sent on SDI.
 326                                 int adjustment = static_cast<int>(2000 / format_desc_.fps);
 327
 328                                 current_presentation_delay_ =
 329                                                 previous_frame_->get_age_millis() + adjustment;
 330                         }
 331
 332                         previous_frame_ = frame;
 333
 334                         graph_->set_value(
 335                                         "tick-time",
 336                                         tick_timer_.elapsed() * format_desc_.fps * 0.5);
 337                         tick_timer_.restart();
 338
 339                         frame_timer_.restart();
 340                         write_video_frame(frame, std::move(key));
 341
 342                         if (config_.embedded_audio)
 343                                 write_audio_samples();
 344
 345                         frame_time += frame_timer_.elapsed();
 346                         graph_->set_value(
 347                                         "frame-time", frame_time * format_desc_.fps * 0.5);
 348
 349                         return true;
 350                 });
 351         }
 352
 353         std::wstring print() const
 354         {
 355                 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(channel_index_) + L"-" +
 356                         boost::lexical_cast<std::wstring>(config_.device_index) + L"|" +  format_desc_.name + L"]";
 357         }
 358 };
 359
 360 struct blocking_decklink_consumer_proxy : public core::frame_consumer
 361 {
 362         const configuration                                             config_;
 363         com_context<blocking_decklink_consumer> context_;
 364         std::vector<size_t>                                             audio_cadence_;
 365         core::video_format_desc                                 format_desc_;
 366 public:
 367
 368         blocking_decklink_consumer_proxy(const configuration& config)
 369                 : config_(config)
 370                 , context_(L"blocking_decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")
 371         {
 372         }
 373
 374         ~blocking_decklink_consumer_proxy()
 375         {
 376                 if(context_)
 377                 {
 378                         auto str = print();
 379                         context_.reset();
 380                         CASPAR_LOG(info) << str << L" Successfully Uninitialized.";
 381                 }
 382         }
 383
 384         // frame_consumer
 385
 386         virtual void initialize(const core::video_format_desc& format_desc, int channel_index) override
 387         {
 388                 context_.reset([&]{return new blocking_decklink_consumer(config_, format_desc, channel_index);});
 389                 audio_cadence_ = format_desc.audio_cadence;
 390                 format_desc_ = format_desc;
 391
 392                 CASPAR_LOG(info) << print() << L" Successfully Initialized.";
 393         }
 394
 395         virtual boost::unique_future<bool> send(const safe_ptr<core::read_frame>& frame) override
 396         {
 397                 CASPAR_VERIFY(audio_cadence_.front() * frame->num_channels() == static_cast<size_t>(frame->audio_data().size()));
 398                 boost::range::rotate(audio_cadence_, std::begin(audio_cadence_)+1);
 399
 400                 return context_->send(frame);
 401         }
 402
 403         virtual std::wstring print() const override
 404         {
 405                 return context_ ? context_->print() : L"[blocking_decklink_consumer]";
 406         }
 407
 408         virtual boost::property_tree::wptree info() const override
 409         {
 410                 boost::property_tree::wptree info;
 411                 info.add(L"type", L"blocking-decklink-consumer");
 412                 info.add(L"key-only", config_.key_only);
 413                 info.add(L"device", config_.device_index);
 414                 info.add(L"embedded-audio", config_.embedded_audio);
 415                 info.add(L"presentation-frame-age", presentation_frame_age_millis());
 416                 //info.add(L"internal-key", config_.internal_key);
 417                 return info;
 418         }
 419
 420         virtual size_t buffer_depth() const override
 421         {
 422                 // Should be 1 according to SDK when DisplayVideoFrameSync is used in
 423                 // combination with low latency, but it does not seem so.
 424                 return 3;
 425         }
 426
 427         virtual bool has_synchronization_clock() const override
 428         {
 429                 return true;
 430         }
 431
 432         virtual int index() const override
 433         {
 434                 return 350 + config_.device_index;
 435         }
 436
 437         virtual int64_t presentation_frame_age_millis() const
 438         {
 439                 return context_ ? context_->current_presentation_delay_ : 0;
 440         }
 441 };
 442
 443 safe_ptr<core::frame_consumer> create_blocking_consumer(const core::parameters& params)
 444 {
 445         if(params.size() < 1 || params[0] != L"BLOCKING_DECKLINK")
 446                 return core::frame_consumer::empty();
 447
 448         configuration config;
 449
 450         if(params.size() > 1)
 451                 config.device_index = lexical_cast_or_default<int>(params[1], config.device_index);
 452
 453         if(std::find(params.begin(), params.end(), L"INTERNAL_KEY")                     != params.end())
 454                 config.keyer = configuration::internal_keyer;
 455         else if(std::find(params.begin(), params.end(), L"EXTERNAL_KEY")        != params.end())
 456                 config.keyer = configuration::external_keyer;
 457         else
 458                 config.keyer = configuration::default_keyer;
 459
 460         config.embedded_audio   = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();
 461         config.key_only                 = std::find(params.begin(), params.end(), L"KEY_ONLY")           != params.end();
 462         config.audio_layout             = core::default_channel_layout_repository().get_by_name(
 463                         params.get(L"CHANNEL_LAYOUT", L"STEREO"));
 464
 465         return make_safe<blocking_decklink_consumer_proxy>(config);
 466 }
 467
 468 safe_ptr<core::frame_consumer> create_blocking_consumer(const boost::property_tree::wptree& ptree)
 469 {
 470         configuration config;
 471
 472         auto keyer = ptree.get(L"keyer", L"external");
 473         if(keyer == L"external")
 474                 config.keyer = configuration::external_keyer;
 475         else if(keyer == L"internal")
 476                 config.keyer = configuration::internal_keyer;
 477
 478         config.key_only                         = ptree.get(L"key-only",                config.key_only);
 479         config.device_index                     = ptree.get(L"device",                  config.device_index);
 480         config.embedded_audio           = ptree.get(L"embedded-audio",  config.embedded_audio);
 481         config.audio_layout =
 482                 core::default_channel_layout_repository().get_by_name(
 483                                 boost::to_upper_copy(ptree.get(L"channel-layout", L"STEREO")));
 484
 485         return make_safe<blocking_decklink_consumer_proxy>(config);
 486 }
 487
 488 }}