git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp

   1 /*\r
   2 * copyright (c) 2010 Sveriges Television AB <info@casparcg.com>\r
   3 *\r
   4 *  This file is part of CasparCG.\r
   5 *\r
   6 *    CasparCG is free software: you can redistribute it and/or modify\r
   7 *    it under the terms of the GNU General Public License as published by\r
   8 *    the Free Software Foundation, either version 3 of the License, or\r
   9 *    (at your option) any later version.\r
  10 *\r
  11 *    CasparCG is distributed in the hope that it will be useful,\r
  12 *    but WITHOUT ANY WARRANTY; without even the implied warranty of\r
  13 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\r
  14 *    GNU General Public License for more details.\r
  15 \r
  16 *    You should have received a copy of the GNU General Public License\r
  17 *    along with CasparCG.  If not, see <http://www.gnu.org/licenses/>.\r
  18 *\r
  19 */\r
  20 \r
  21 #include "../StdAfx.h"\r
  22  \r
  23 #include "decklink_consumer.h"\r
  24 \r
  25 #include "../util/util.h"\r
  26 \r
  27 #include "../interop/DeckLinkAPI_h.h"\r
  28 \r
  29 #include <core/mixer/read_frame.h>\r
  30 \r
  31 #include <common/concurrency/com_context.h>\r
  32 #include <common/diagnostics/graph.h>\r
  33 #include <common/exception/exceptions.h>\r
  34 #include <common/memory/memcpy.h>\r
  35 #include <common/memory/memclr.h>\r
  36 #include <common/memory/memshfl.h>\r
  37 \r
  38 #include <core/consumer/frame_consumer.h>\r
  39 \r
  40 #include <tbb/concurrent_queue.h>\r
  41 \r
  42 #include <boost/circular_buffer.hpp>\r
  43 #include <boost/timer.hpp>\r
  44 \r
  45 namespace caspar { \r
  46         \r
  47 struct configuration\r
  48 {\r
  49         size_t  device_index;\r
  50         bool    embedded_audio;\r
  51         bool    external_key;\r
  52         bool    low_latency;\r
  53         bool    key_only;\r
  54         size_t  buffer_depth;\r
  55         \r
  56         configuration()\r
  57                 : device_index(1)\r
  58                 , embedded_audio(false)\r
  59                 , external_key(false)\r
  60                 , low_latency(false)\r
  61                 , key_only(false)\r
  62                 , buffer_depth(core::consumer_buffer_depth()){}\r
  63 };\r
  64 \r
  65 class decklink_frame : public IDeckLinkVideoFrame\r
  66 {\r
  67         const safe_ptr<core::read_frame>        frame_;\r
  68         const core::video_format_desc           format_desc_;\r
  69 public:\r
  70         decklink_frame(const safe_ptr<core::read_frame>& frame, const core::video_format_desc& format_desc)\r
  71                 : frame_(frame)\r
  72                 , format_desc_(format_desc){}\r
  73         \r
  74         STDMETHOD (QueryInterface(REFIID, LPVOID*))     {return E_NOINTERFACE;}\r
  75         STDMETHOD_(ULONG, AddRef())                                     {return 1;}\r
  76         STDMETHOD_(ULONG, Release())                            {return 1;}\r
  77 \r
  78         STDMETHOD_(long, GetWidth())                            {return format_desc_.width;}        \r
  79     STDMETHOD_(long, GetHeight())                               {return format_desc_.height;}        \r
  80     STDMETHOD_(long, GetRowBytes())                             {return format_desc_.width*4;}        \r
  81         STDMETHOD_(BMDPixelFormat, GetPixelFormat()){return bmdFormat8BitBGRA;}        \r
  82     STDMETHOD_(BMDFrameFlags, GetFlags())               {return bmdFrameFlagDefault;}\r
  83         \r
  84     STDMETHOD(GetBytes(void** buffer))\r
  85         {\r
  86                 static std::vector<uint8_t> zeros(1920*1080*4, 0);\r
  87                 *buffer = const_cast<uint8_t*>(frame_->image_data().begin());\r
  88                 if(static_cast<size_t>(frame_->image_data().size()) != format_desc_.size)\r
  89                         *buffer = zeros.data();\r
  90                 return S_OK;\r
  91         }\r
  92         \r
  93     STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}        \r
  94     STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary))                 {return S_FALSE;}\r
  95 };\r
  96 \r
  97 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable\r
  98 {               \r
  99         const configuration config_;\r
 100 \r
 101         CComPtr<IDeckLink>                                      decklink_;\r
 102         CComQIPtr<IDeckLinkOutput>                      output_;\r
 103         CComQIPtr<IDeckLinkConfiguration>       configuration_;\r
 104         CComQIPtr<IDeckLinkKeyer>                       keyer_;\r
 105 \r
 106         tbb::spin_mutex                                         exception_mutex_;\r
 107         std::exception_ptr                                      exception_;\r
 108 \r
 109         tbb::atomic<bool>                                       is_running_;\r
 110                 \r
 111         const std::wstring                                      model_name_;\r
 112         const core::video_format_desc           format_desc_;\r
 113         const size_t                                            buffer_size_;\r
 114 \r
 115         long long                                                       frames_scheduled_;\r
 116         long long                                                       audio_scheduled_;\r
 117 \r
 118         size_t                                                          preroll_count_;\r
 119                 \r
 120         std::list<std::shared_ptr<IDeckLinkVideoFrame>> frame_container_; // Must be std::list in order to guarantee that pointers are always valid.\r
 121         boost::circular_buffer<std::vector<int16_t>>    audio_container_;\r
 122 \r
 123         tbb::concurrent_bounded_queue<std::shared_ptr<core::read_frame>> video_frame_buffer_;\r
 124         tbb::concurrent_bounded_queue<std::shared_ptr<core::read_frame>> audio_frame_buffer_;\r
 125         \r
 126         std::shared_ptr<diagnostics::graph> graph_;\r
 127         boost::timer tick_timer_;\r
 128 \r
 129 public:\r
 130         decklink_consumer(const configuration& config, const core::video_format_desc& format_desc) \r
 131                 : config_(config)\r
 132                 , decklink_(get_device(config.device_index))\r
 133                 , output_(decklink_)\r
 134                 , configuration_(decklink_)\r
 135                 , keyer_(decklink_)\r
 136                 , model_name_(get_model_name(decklink_))\r
 137                 , format_desc_(format_desc)\r
 138                 , buffer_size_(config.embedded_audio ? config.buffer_depth + 1 : config.buffer_depth) // Minimum buffer-size 3.\r
 139                 , frames_scheduled_(0)\r
 140                 , audio_scheduled_(0)\r
 141                 , preroll_count_(0)\r
 142                 , audio_container_(buffer_size_+1)\r
 143         {\r
 144                 is_running_ = true;\r
 145                                 \r
 146                 video_frame_buffer_.set_capacity(1);\r
 147                 audio_frame_buffer_.set_capacity(1);\r
 148 \r
 149                 graph_ = diagnostics::create_graph(narrow(print()));\r
 150                 graph_->add_guide("tick-time", 0.5);\r
 151                 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));   \r
 152                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));\r
 153                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));\r
 154                 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));\r
 155                 \r
 156                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));\r
 157                                 \r
 158                 if(config.embedded_audio)\r
 159                         enable_audio();\r
 160 \r
 161                 set_latency(config.low_latency);                                \r
 162                 set_keyer(config.external_key);\r
 163                                 \r
 164                 if(config.embedded_audio)               \r
 165                         output_->BeginAudioPreroll();           \r
 166                 \r
 167                 for(size_t n = 0; n < buffer_size_; ++n)\r
 168                         schedule_next_video(make_safe<core::read_frame>());\r
 169 \r
 170                 if(!config.embedded_audio)\r
 171                         start_playback();\r
 172                                 \r
 173                 CASPAR_LOG(info) << print() << L" Successfully Initialized.";   \r
 174         }\r
 175 \r
 176         ~decklink_consumer()\r
 177         {               \r
 178                 is_running_ = false;\r
 179                 video_frame_buffer_.try_push(std::make_shared<core::read_frame>());\r
 180                 audio_frame_buffer_.try_push(std::make_shared<core::read_frame>());\r
 181 \r
 182                 if(output_ != nullptr) \r
 183                 {\r
 184                         output_->StopScheduledPlayback(0, nullptr, 0);\r
 185                         if(config_.embedded_audio)\r
 186                                 output_->DisableAudioOutput();\r
 187                         output_->DisableVideoOutput();\r
 188                 }\r
 189         }\r
 190                         \r
 191         const core::video_format_desc& get_video_format_desc() const\r
 192         {\r
 193                 return format_desc_;\r
 194         }\r
 195 \r
 196         void set_latency(bool low_latency)\r
 197         {               \r
 198                 if(!low_latency)\r
 199                 {\r
 200                         configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);\r
 201                         CASPAR_LOG(info) << print() << L" Enabled normal-latency mode";\r
 202                 }\r
 203                 else\r
 204                 {                       \r
 205                         configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);\r
 206                         CASPAR_LOG(info) << print() << L" Enabled low-latency mode";\r
 207                 }\r
 208         }\r
 209 \r
 210         void set_keyer(bool external_key)\r
 211         {\r
 212                 if(!external_key) \r
 213                 {\r
 214                         if(FAILED(keyer_->Enable(FALSE)))                       \r
 215                                 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";                   \r
 216                         else if(FAILED(keyer_->SetLevel(255)))                  \r
 217                                 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";\r
 218                         else\r
 219                                 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";             \r
 220                 }\r
 221                 else\r
 222                 {\r
 223                         if(FAILED(keyer_->Enable(TRUE)))                        \r
 224                                 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";   \r
 225                         else if(FAILED(keyer_->SetLevel(255)))                  \r
 226                                 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";\r
 227                         else\r
 228                                 CASPAR_LOG(info) << print() << L" Enabled external keyer.";                     \r
 229                 }\r
 230         }\r
 231         \r
 232         void enable_audio()\r
 233         {\r
 234                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType16bitInteger, 2, bmdAudioOutputStreamTimestamped)))\r
 235                                 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not enable audio output."));\r
 236                                 \r
 237                 if(FAILED(output_->SetAudioCallback(this)))\r
 238                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not set audio callback."));\r
 239 \r
 240                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";\r
 241         }\r
 242 \r
 243         void enable_video(BMDDisplayMode display_mode)\r
 244         {\r
 245                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault))) \r
 246                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not enable video output."));\r
 247                 \r
 248                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))\r
 249                         BOOST_THROW_EXCEPTION(caspar_exception() \r
 250                                                                         << msg_info(narrow(print()) + " Failed to set playback completion callback.")\r
 251                                                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));\r
 252         }\r
 253 \r
 254         void start_playback()\r
 255         {\r
 256                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0))) \r
 257                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Failed to schedule playback."));\r
 258         }\r
 259         \r
 260         STDMETHOD (QueryInterface(REFIID, LPVOID*))     {return E_NOINTERFACE;}\r
 261         STDMETHOD_(ULONG, AddRef())                                     {return 1;}\r
 262         STDMETHOD_(ULONG, Release())                            {return 1;}\r
 263         \r
 264         STDMETHOD(ScheduledPlaybackHasStopped())\r
 265         {\r
 266                 is_running_ = false;\r
 267                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";\r
 268                 return S_OK;\r
 269         }\r
 270 \r
 271         STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))\r
 272         {\r
 273                 if(!is_running_)\r
 274                         return E_FAIL;\r
 275                 \r
 276                 try\r
 277                 {\r
 278                         if(result == bmdOutputFrameDisplayedLate)\r
 279                         {\r
 280                                 graph_->add_tag("late-frame");\r
 281                                 ++frames_scheduled_;\r
 282                                 ++audio_scheduled_;\r
 283                         }\r
 284                         else if(result == bmdOutputFrameDropped)\r
 285                                 graph_->add_tag("dropped-frame");\r
 286                         else if(result == bmdOutputFrameFlushed)\r
 287                                 graph_->add_tag("flushed-frame");\r
 288 \r
 289                         frame_container_.erase(std::find_if(frame_container_.begin(), frame_container_.end(), [&](const std::shared_ptr<IDeckLinkVideoFrame>& frame)\r
 290                         {\r
 291                                 return frame.get() == completed_frame;\r
 292                         }));\r
 293 \r
 294                         std::shared_ptr<core::read_frame> frame;        \r
 295                         video_frame_buffer_.pop(frame);                                 \r
 296                         schedule_next_video(make_safe(frame));  \r
 297                 }\r
 298                 catch(...)\r
 299                 {\r
 300                         tbb::spin_mutex::scoped_lock lock(exception_mutex_);\r
 301                         exception_ = std::current_exception();\r
 302                         return E_FAIL;\r
 303                 }\r
 304 \r
 305                 return S_OK;\r
 306         }\r
 307                 \r
 308         STDMETHOD(RenderAudioSamples(BOOL preroll))\r
 309         {\r
 310                 if(!is_running_)\r
 311                         return E_FAIL;\r
 312                 \r
 313                 try\r
 314                 {       \r
 315                         if(preroll)\r
 316                         {\r
 317                                 if(++preroll_count_ >= buffer_size_)\r
 318                                 {\r
 319                                         output_->EndAudioPreroll();\r
 320                                         start_playback();                               \r
 321                                 }\r
 322                                 else\r
 323                                         schedule_next_audio(make_safe<core::read_frame>());     \r
 324                         }\r
 325                         else\r
 326                         {\r
 327                                 std::shared_ptr<core::read_frame> frame;\r
 328                                 audio_frame_buffer_.pop(frame);\r
 329                                 schedule_next_audio(make_safe(frame));  \r
 330                         }\r
 331                 }\r
 332                 catch(...)\r
 333                 {\r
 334                         tbb::spin_mutex::scoped_lock lock(exception_mutex_);\r
 335                         exception_ = std::current_exception();\r
 336                         return E_FAIL;\r
 337                 }\r
 338 \r
 339                 return S_OK;\r
 340         }\r
 341 \r
 342         void schedule_next_audio(const safe_ptr<core::read_frame>& frame)\r
 343         {\r
 344                 const int sample_frame_count = frame->audio_data().size()/format_desc_.audio_channels;\r
 345 \r
 346                 audio_container_.push_back(std::vector<int16_t>(frame->audio_data().begin(), frame->audio_data().end()));\r
 347 \r
 348                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, (audio_scheduled_++) * sample_frame_count, format_desc_.audio_sample_rate, nullptr)))\r
 349                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";\r
 350         }\r
 351                         \r
 352         void schedule_next_video(const safe_ptr<core::read_frame>& frame)\r
 353         {\r
 354                 frame_container_.push_back(std::make_shared<decklink_frame>(frame, format_desc_));\r
 355                 if(FAILED(output_->ScheduleVideoFrame(frame_container_.back().get(), (frames_scheduled_++) * format_desc_.duration, format_desc_.duration, format_desc_.time_scale)))\r
 356                         CASPAR_LOG(error) << print() << L" Failed to schedule video.";\r
 357 \r
 358                 graph_->update_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);\r
 359                 tick_timer_.restart();\r
 360         }\r
 361 \r
 362         void send(const safe_ptr<core::read_frame>& frame)\r
 363         {\r
 364                 {\r
 365                         tbb::spin_mutex::scoped_lock lock(exception_mutex_);\r
 366                         if(exception_ != nullptr)\r
 367                                 std::rethrow_exception(exception_);\r
 368                 }\r
 369 \r
 370                 if(!is_running_)\r
 371                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Is not running."));\r
 372                 \r
 373                 if(config_.embedded_audio)\r
 374                         audio_frame_buffer_.push(frame);        \r
 375                 video_frame_buffer_.push(frame);        \r
 376         }\r
 377         \r
 378         std::wstring print() const\r
 379         {\r
 380                 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(config_.device_index) + L"|" +  format_desc_.name + L"]";\r
 381         }\r
 382 };\r
 383 \r
 384 struct decklink_consumer_proxy : public core::frame_consumer\r
 385 {\r
 386         const configuration                             config_;\r
 387         com_context<decklink_consumer>  context_;\r
 388         core::video_format_desc                 format_desc_;\r
 389         size_t                                                  fail_count_;\r
 390 public:\r
 391 \r
 392         decklink_consumer_proxy(const configuration& config)\r
 393                 : config_(config)\r
 394                 , context_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")\r
 395                 , fail_count_(0)\r
 396         {\r
 397         }\r
 398         \r
 399         virtual void initialize(const core::video_format_desc& format_desc)\r
 400         {\r
 401                 format_desc_ = format_desc;\r
 402                 context_.reset([&]{return new decklink_consumer(config_, format_desc_);});\r
 403         }\r
 404         \r
 405         virtual bool send(const safe_ptr<core::read_frame>& frame)\r
 406         {\r
 407                 if(!context_)\r
 408                         context_.reset([&]{return new decklink_consumer(config_, format_desc_);});\r
 409 \r
 410                 try\r
 411                 {\r
 412                         context_->send(frame);\r
 413                         fail_count_ = 0;\r
 414                 }\r
 415                 catch(...)\r
 416                 {\r
 417                         context_.reset();\r
 418 \r
 419                         if(fail_count_++ > 3)\r
 420                                 return false;  // Outside didn't handle exception properly, just give up.\r
 421                         \r
 422                         throw;\r
 423                 }\r
 424 \r
 425                 return true;\r
 426         }\r
 427         \r
 428         virtual std::wstring print() const\r
 429         {\r
 430                 return context_->print();\r
 431         }\r
 432 \r
 433         virtual bool key_only() const\r
 434         {\r
 435                 return config_.key_only;\r
 436         }\r
 437                 \r
 438         virtual const core::video_format_desc& get_video_format_desc() const\r
 439         {\r
 440                 return format_desc_;\r
 441         }\r
 442 };      \r
 443 \r
 444 safe_ptr<core::frame_consumer> create_decklink_consumer(const std::vector<std::wstring>& params) \r
 445 {\r
 446         if(params.size() < 1 || params[0] != L"DECKLINK")\r
 447                 return core::frame_consumer::empty();\r
 448         \r
 449         configuration config;\r
 450                 \r
 451         if(params.size() > 1)\r
 452                 config.device_index = lexical_cast_or_default<int>(params[1], config.device_index);\r
 453         \r
 454         config.external_key             = std::find(params.begin(), params.end(), L"EXTERNAL_KEY")       != params.end();\r
 455         config.low_latency              = std::find(params.begin(), params.end(), L"LOW_LATENCY")        != params.end();\r
 456         config.embedded_audio   = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();\r
 457         config.key_only                 = std::find(params.begin(), params.end(), L"KEY_ONLY")           != params.end();\r
 458 \r
 459         return make_safe<decklink_consumer_proxy>(config);\r
 460 }\r
 461 \r
 462 safe_ptr<core::frame_consumer> create_decklink_consumer(const boost::property_tree::ptree& ptree) \r
 463 {\r
 464         configuration config;\r
 465 \r
 466         config.external_key             = ptree.get("external-key",       config.external_key);\r
 467         config.low_latency              = ptree.get("low-latency",        config.low_latency);\r
 468         config.key_only                 = ptree.get("key-only",           config.key_only);\r
 469         config.device_index             = ptree.get("device",             config.device_index);\r
 470         config.embedded_audio   = ptree.get("embedded-audio", config.embedded_audio);\r
 471 \r
 472         return make_safe<decklink_consumer_proxy>(config);\r
 473 }\r
 474 \r
 475 }\r
 476 \r
 477 /*\r
 478 ##############################################################################\r
 479 Pre-rolling\r
 480 \r
 481 Mail: 2011-05-09\r
 482 \r
 483 Yoshan\r
 484 BMD Developer Support\r
 485 developer@blackmagic-design.com\r
 486 \r
 487 -----------------------------------------------------------------------------\r
 488 \r
 489 Thanks for your inquiry. The minimum number of frames that you can preroll \r
 490 for scheduled playback is three frames for video and four frames for audio. \r
 491 As you mentioned if you preroll less frames then playback will not start or\r
 492 playback will be very sporadic. From our experience with Media Express, we \r
 493 recommended that at least seven frames are prerolled for smooth playback. \r
 494 \r
 495 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:\r
 496 There can be around 3 frames worth of latency on scheduled output.\r
 497 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is\r
 498 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync() \r
 499 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will \r
 500 guarantee that the provided frame will be output as soon the previous \r
 501 frame output has been completed.\r
 502 ################################################################################\r
 503 */\r
 504 \r
 505 /*\r
 506 ##############################################################################\r
 507 Async DMA Transfer without redundant copying\r
 508 \r
 509 Mail: 2011-05-10\r
 510 \r
 511 Yoshan\r
 512 BMD Developer Support\r
 513 developer@blackmagic-design.com\r
 514 \r
 515 -----------------------------------------------------------------------------\r
 516 \r
 517 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame \r
 518 and providing a pointer to your video buffer when GetBytes() is called. \r
 519 This may help to keep copying to a minimum. Please ensure that the pixel \r
 520 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will \r
 521 have to colourspace convert which may result in additional copying.\r
 522 ################################################################################\r
 523 */