git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp

   1 /*\r
   2 * copyright (c) 2010 Sveriges Television AB <info@casparcg.com>\r
   3 *\r
   4 *  This file is part of CasparCG.\r
   5 *\r
   6 *    CasparCG is free software: you can redistribute it and/or modify\r
   7 *    it under the terms of the GNU General Public License as published by\r
   8 *    the Free Software Foundation, either version 3 of the License, or\r
   9 *    (at your option) any later version.\r
  10 *\r
  11 *    CasparCG is distributed in the hope that it will be useful,\r
  12 *    but WITHOUT ANY WARRANTY; without even the implied warranty of\r
  13 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\r
  14 *    GNU General Public License for more details.\r
  15 \r
  16 *    You should have received a copy of the GNU General Public License\r
  17 *    along with CasparCG.  If not, see <http://www.gnu.org/licenses/>.\r
  18 *\r
  19 */\r
  20 \r
  21 #include "../StdAfx.h"\r
  22  \r
  23 #include "decklink_consumer.h"\r
  24 \r
  25 #include "../util/util.h"\r
  26 \r
  27 #include "../interop/DeckLinkAPI_h.h"\r
  28 \r
  29 #include <core/mixer/read_frame.h>\r
  30 \r
  31 #include <common/concurrency/com_context.h>\r
  32 #include <common/diagnostics/graph.h>\r
  33 #include <common/exception/exceptions.h>\r
  34 #include <common/memory/memcpy.h>\r
  35 #include <common/memory/memclr.h>\r
  36 #include <common/memory/memshfl.h>\r
  37 \r
  38 #include <tbb/concurrent_queue.h>\r
  39 \r
  40 #include <boost/circular_buffer.hpp>\r
  41 #include <boost/timer.hpp>\r
  42 \r
  43 namespace caspar { \r
  44         \r
  45 struct configuration\r
  46 {\r
  47         size_t  device_index;\r
  48         bool    embedded_audio;\r
  49         bool    external_key;\r
  50         bool    low_latency;\r
  51         bool    key_only;\r
  52         size_t  buffer_depth;\r
  53         \r
  54         configuration()\r
  55                 : device_index(1)\r
  56                 , embedded_audio(false)\r
  57                 , external_key(false)\r
  58                 , low_latency(false)\r
  59                 , key_only(false)\r
  60                 , buffer_depth(CONSUMER_BUFFER_DEPTH){}\r
  61 };\r
  62 \r
  63 class decklink_frame_adapter : public IDeckLinkVideoFrame\r
  64 {\r
  65         const safe_ptr<core::read_frame>        frame_;\r
  66         const core::video_format_desc                   format_desc_;\r
  67 public:\r
  68         decklink_frame_adapter(const safe_ptr<core::read_frame>& frame, const core::video_format_desc& format_desc)\r
  69                 : frame_(frame)\r
  70                 , format_desc_(format_desc){}\r
  71         \r
  72         STDMETHOD (QueryInterface(REFIID, LPVOID*))     {return E_NOINTERFACE;}\r
  73         STDMETHOD_(ULONG, AddRef())                                     {return 1;}\r
  74         STDMETHOD_(ULONG, Release())                            {return 1;}\r
  75 \r
  76         STDMETHOD_(long, GetWidth())                            {return format_desc_.width;}        \r
  77     STDMETHOD_(long, GetHeight())                               {return format_desc_.height;}        \r
  78     STDMETHOD_(long, GetRowBytes())                             {return format_desc_.width*4;}        \r
  79         STDMETHOD_(BMDPixelFormat, GetPixelFormat()){return bmdFormat8BitBGRA;}        \r
  80     STDMETHOD_(BMDFrameFlags, GetFlags())               {return bmdFrameFlagDefault;}\r
  81         \r
  82     STDMETHOD(GetBytes(void** buffer))\r
  83         {\r
  84                 static std::vector<uint8_t> zeros(1920*1080*4, 0);\r
  85                 *buffer = const_cast<uint8_t*>(frame_->image_data().begin());\r
  86                 if(static_cast<size_t>(frame_->image_data().size()) != format_desc_.size)\r
  87                         *buffer = zeros.data();\r
  88                 return S_OK;\r
  89         }\r
  90         \r
  91     STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}        \r
  92     STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary))                 {return S_FALSE;}\r
  93 };\r
  94 \r
  95 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable\r
  96 {               \r
  97         const configuration config_;\r
  98 \r
  99         CComPtr<IDeckLink>                                      decklink_;\r
 100         CComQIPtr<IDeckLinkOutput>                      output_;\r
 101         CComQIPtr<IDeckLinkConfiguration>       configuration_;\r
 102         CComQIPtr<IDeckLinkKeyer>                       keyer_;\r
 103 \r
 104         std::exception_ptr exception_;\r
 105 \r
 106         tbb::atomic<bool> is_running_;\r
 107                 \r
 108         const std::wstring model_name_;\r
 109         const core::video_format_desc format_desc_;\r
 110         const size_t buffer_size_;\r
 111 \r
 112         unsigned long frames_scheduled_;\r
 113         unsigned long audio_scheduled_;\r
 114 \r
 115         size_t preroll_count_;\r
 116                 \r
 117         std::list<std::shared_ptr<IDeckLinkVideoFrame>> frame_container_; // Must be std::list in order to guarantee that pointers are always valid.\r
 118         boost::circular_buffer<std::vector<short>> audio_container_;\r
 119 \r
 120         tbb::concurrent_bounded_queue<std::shared_ptr<core::read_frame>> video_frame_buffer_;\r
 121         tbb::concurrent_bounded_queue<std::shared_ptr<core::read_frame>> audio_frame_buffer_;\r
 122         \r
 123         std::shared_ptr<diagnostics::graph> graph_;\r
 124         boost::timer tick_timer_;\r
 125 \r
 126 public:\r
 127         decklink_consumer(const configuration& config, const core::video_format_desc& format_desc) \r
 128                 : config_(config)\r
 129                 , decklink_(get_device(config.device_index))\r
 130                 , output_(decklink_)\r
 131                 , configuration_(decklink_)\r
 132                 , keyer_(decklink_)\r
 133                 , model_name_(get_model_name(decklink_))\r
 134                 , format_desc_(format_desc)\r
 135                 , buffer_size_(config.embedded_audio ? config.buffer_depth + 1 : config.buffer_depth) // Minimum buffer-size 3.\r
 136                 , frames_scheduled_(0)\r
 137                 , audio_scheduled_(0)\r
 138                 , preroll_count_(0)\r
 139                 , audio_container_(buffer_size_+1)\r
 140         {\r
 141                 is_running_ = true;\r
 142                                 \r
 143                 video_frame_buffer_.set_capacity(1);\r
 144                 audio_frame_buffer_.set_capacity(1);\r
 145 \r
 146                 graph_ = diagnostics::create_graph(narrow(print()));\r
 147                 graph_->add_guide("tick-time", 0.5);\r
 148                 graph_->set_color("tick-time", diagnostics::color(0.1f, 0.7f, 0.8f));\r
 149                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));\r
 150                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));\r
 151                 graph_->set_color("flushed-frame", diagnostics::color(0.3f, 0.3f, 0.6f));\r
 152                 \r
 153                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));\r
 154                                 \r
 155                 if(config.embedded_audio)\r
 156                         enable_audio();\r
 157 \r
 158                 set_latency(config.low_latency);                                \r
 159                 set_keyer(config.external_key);\r
 160                                 \r
 161                 if(config.embedded_audio)               \r
 162                         output_->BeginAudioPreroll();           \r
 163                 \r
 164                 for(size_t n = 0; n < buffer_size_; ++n)\r
 165                         schedule_next_video(make_safe<core::read_frame>());\r
 166 \r
 167                 if(!config.embedded_audio)\r
 168                         start_playback();\r
 169                                 \r
 170                 CASPAR_LOG(info) << print() << L" Successfully Initialized.";   \r
 171         }\r
 172 \r
 173         ~decklink_consumer()\r
 174         {               \r
 175                 is_running_ = false;\r
 176                 video_frame_buffer_.try_push(std::make_shared<core::read_frame>());\r
 177                 audio_frame_buffer_.try_push(std::make_shared<core::read_frame>());\r
 178 \r
 179                 if(output_ != nullptr) \r
 180                 {\r
 181                         output_->StopScheduledPlayback(0, nullptr, 0);\r
 182                         if(config_.embedded_audio)\r
 183                                 output_->DisableAudioOutput();\r
 184                         output_->DisableVideoOutput();\r
 185                 }\r
 186         }\r
 187                         \r
 188         const core::video_format_desc& get_video_format_desc() const\r
 189         {\r
 190                 return format_desc_;\r
 191         }\r
 192 \r
 193         void set_latency(bool low_latency)\r
 194         {               \r
 195                 if(!low_latency)\r
 196                 {\r
 197                         configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);\r
 198                         CASPAR_LOG(info) << print() << L" Enabled normal-latency mode";\r
 199                 }\r
 200                 else\r
 201                 {                       \r
 202                         configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);\r
 203                         CASPAR_LOG(info) << print() << L" Enabled low-latency mode";\r
 204                 }\r
 205         }\r
 206 \r
 207         void set_keyer(bool external_key)\r
 208         {\r
 209                 if(!external_key) \r
 210                 {\r
 211                         if(FAILED(keyer_->Enable(FALSE)))                       \r
 212                                 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";                   \r
 213                         else if(FAILED(keyer_->SetLevel(255)))                  \r
 214                                 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";\r
 215                         else\r
 216                                 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";             \r
 217                 }\r
 218                 else\r
 219                 {\r
 220                         if(FAILED(keyer_->Enable(TRUE)))                        \r
 221                                 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";   \r
 222                         else if(FAILED(keyer_->SetLevel(255)))                  \r
 223                                 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";\r
 224                         else\r
 225                                 CASPAR_LOG(info) << print() << L" Enabled external keyer.";                     \r
 226                 }\r
 227         }\r
 228         \r
 229         void enable_audio()\r
 230         {\r
 231                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType16bitInteger, 2, bmdAudioOutputStreamTimestamped)))\r
 232                                 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not enable audio output."));\r
 233                                 \r
 234                 if(FAILED(output_->SetAudioCallback(this)))\r
 235                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not set audio callback."));\r
 236 \r
 237                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";\r
 238         }\r
 239 \r
 240         void enable_video(BMDDisplayMode display_mode)\r
 241         {\r
 242                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault))) \r
 243                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not enable video output."));\r
 244                 \r
 245                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))\r
 246                         BOOST_THROW_EXCEPTION(caspar_exception() \r
 247                                                                         << msg_info(narrow(print()) + " Failed to set playback completion callback.")\r
 248                                                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));\r
 249         }\r
 250 \r
 251         void start_playback()\r
 252         {\r
 253                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0))) \r
 254                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Failed to schedule playback."));\r
 255         }\r
 256         \r
 257         STDMETHOD (QueryInterface(REFIID, LPVOID*))     {return E_NOINTERFACE;}\r
 258         STDMETHOD_(ULONG, AddRef())                                     {return 1;}\r
 259         STDMETHOD_(ULONG, Release())                            {return 1;}\r
 260         \r
 261         STDMETHOD(ScheduledPlaybackHasStopped())\r
 262         {\r
 263                 is_running_ = false;\r
 264                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";\r
 265                 return S_OK;\r
 266         }\r
 267 \r
 268         STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))\r
 269         {\r
 270                 if(!is_running_)\r
 271                         return E_FAIL;\r
 272                 \r
 273                 try\r
 274                 {\r
 275                         if(result == bmdOutputFrameDisplayedLate)\r
 276                         {\r
 277                                 graph_->add_tag("late-frame");\r
 278                                 ++frames_scheduled_;\r
 279                                 ++audio_scheduled_;\r
 280                         }\r
 281                         else if(result == bmdOutputFrameDropped)\r
 282                                 graph_->add_tag("dropped-frame");\r
 283                         else if(result == bmdOutputFrameFlushed)\r
 284                                 graph_->add_tag("flushed-frame");\r
 285 \r
 286                         frame_container_.erase(std::find_if(frame_container_.begin(), frame_container_.end(), [&](const std::shared_ptr<IDeckLinkVideoFrame>& frame)\r
 287                         {\r
 288                                 return frame.get() == completed_frame;\r
 289                         }));\r
 290 \r
 291                         std::shared_ptr<core::read_frame> frame;        \r
 292                         video_frame_buffer_.pop(frame);                                 \r
 293                         schedule_next_video(make_safe(frame));                  \r
 294                 }\r
 295                 catch(...)\r
 296                 {\r
 297                         exception_ = std::current_exception();\r
 298                         return E_FAIL;\r
 299                 }\r
 300 \r
 301                 return S_OK;\r
 302         }\r
 303                 \r
 304         STDMETHOD(RenderAudioSamples(BOOL preroll))\r
 305         {\r
 306                 if(!is_running_)\r
 307                         return E_FAIL;\r
 308                 \r
 309                 try\r
 310                 {       \r
 311                         if(preroll)\r
 312                         {\r
 313                                 if(++preroll_count_ >= buffer_size_)\r
 314                                 {\r
 315                                         output_->EndAudioPreroll();\r
 316                                         start_playback();                               \r
 317                                 }\r
 318                                 else\r
 319                                         schedule_next_audio(make_safe<core::read_frame>());     \r
 320                         }\r
 321                         else\r
 322                         {\r
 323                                 std::shared_ptr<core::read_frame> frame;\r
 324                                 audio_frame_buffer_.pop(frame);\r
 325                                 schedule_next_audio(make_safe(frame));  \r
 326                         }\r
 327                 }\r
 328                 catch(...)\r
 329                 {\r
 330                         exception_ = std::current_exception();\r
 331                         return E_FAIL;\r
 332                 }\r
 333 \r
 334                 return S_OK;\r
 335         }\r
 336 \r
 337         void schedule_next_audio(const safe_ptr<core::read_frame>& frame)\r
 338         {\r
 339                 static std::vector<short> silence(48000, 0);\r
 340                 \r
 341                 const int sample_count = format_desc_.audio_samples_per_frame;\r
 342                 const int sample_frame_count = sample_count/2;\r
 343 \r
 344                 const int16_t* frame_audio_data = frame->audio_data().size() == sample_count ? frame->audio_data().begin() : silence.data();\r
 345                 audio_container_.push_back(std::vector<int16_t>(frame_audio_data, frame_audio_data+sample_count));\r
 346 \r
 347                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, (audio_scheduled_++) * sample_frame_count, format_desc_.audio_sample_rate, nullptr)))\r
 348                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";\r
 349         }\r
 350                         \r
 351         void schedule_next_video(const safe_ptr<core::read_frame>& frame)\r
 352         {\r
 353                 frame_container_.push_back(std::make_shared<decklink_frame_adapter>(frame, format_desc_));\r
 354                 if(FAILED(output_->ScheduleVideoFrame(frame_container_.back().get(), (frames_scheduled_++) * format_desc_.duration, format_desc_.duration, format_desc_.time_scale)))\r
 355                         CASPAR_LOG(error) << print() << L" Failed to schedule video.";\r
 356 \r
 357                 graph_->update_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);\r
 358                 tick_timer_.restart();\r
 359         }\r
 360 \r
 361         void send(const safe_ptr<core::read_frame>& frame)\r
 362         {\r
 363                 if(exception_ != nullptr)\r
 364                         std::rethrow_exception(exception_);\r
 365 \r
 366                 if(!is_running_)\r
 367                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Is not running."));\r
 368                 \r
 369                 if(config_.embedded_audio)\r
 370                         audio_frame_buffer_.push(frame);        \r
 371                 video_frame_buffer_.push(frame);        \r
 372         }\r
 373         \r
 374         std::wstring print() const\r
 375         {\r
 376                 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(config_.device_index) + L"|" +  format_desc_.name + L"]";\r
 377         }\r
 378 };\r
 379 \r
 380 struct decklink_consumer_proxy : public core::frame_consumer\r
 381 {\r
 382         const configuration config_;\r
 383 \r
 384         com_context<decklink_consumer> context_;\r
 385 public:\r
 386 \r
 387         decklink_consumer_proxy(const configuration& config)\r
 388                 : config_(config)\r
 389                 , context_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]"){}\r
 390         \r
 391         virtual void initialize(const core::video_format_desc& format_desc)\r
 392         {\r
 393                 context_.reset([&]{return new decklink_consumer(config_, format_desc);});\r
 394         }\r
 395         \r
 396         virtual void send(const safe_ptr<core::read_frame>& frame)\r
 397         {\r
 398                 context_->send(frame);\r
 399         }\r
 400         \r
 401         virtual std::wstring print() const\r
 402         {\r
 403                 return context_->print();\r
 404         }\r
 405 \r
 406         virtual bool key_only() const\r
 407         {\r
 408                 return config_.key_only;\r
 409         }\r
 410                 \r
 411         virtual const core::video_format_desc& get_video_format_desc() const\r
 412         {\r
 413                 return context_->get_video_format_desc();\r
 414         }\r
 415 \r
 416         virtual size_t buffer_depth() const\r
 417         {\r
 418                 return context_->buffer_size_;\r
 419         }\r
 420 };      \r
 421 \r
 422 safe_ptr<core::frame_consumer> create_decklink_consumer(const std::vector<std::wstring>& params) \r
 423 {\r
 424         if(params.size() < 1 || params[0] != L"DECKLINK")\r
 425                 return core::frame_consumer::empty();\r
 426         \r
 427         configuration config;\r
 428                 \r
 429         if(params.size() > 1)\r
 430                 config.device_index = lexical_cast_or_default<int>(params[1], config.device_index);\r
 431         \r
 432         config.external_key             = std::find(params.begin(), params.end(), L"EXTERNAL_KEY")       != params.end();\r
 433         config.low_latency              = std::find(params.begin(), params.end(), L"LOW_LATENCY")        != params.end();\r
 434         config.embedded_audio   = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();\r
 435         config.key_only                 = std::find(params.begin(), params.end(), L"KEY_ONLY")           != params.end();\r
 436 \r
 437         return make_safe<decklink_consumer_proxy>(config);\r
 438 }\r
 439 \r
 440 safe_ptr<core::frame_consumer> create_decklink_consumer(const boost::property_tree::ptree& ptree) \r
 441 {\r
 442         configuration config;\r
 443 \r
 444         config.external_key             = ptree.get("external-key",       config.external_key);\r
 445         config.low_latency              = ptree.get("low-latency",        config.low_latency);\r
 446         config.key_only                 = ptree.get("key-only",           config.key_only);\r
 447         config.device_index             = ptree.get("device",             config.device_index);\r
 448         config.embedded_audio   = ptree.get("embedded-audio", config.embedded_audio);\r
 449 \r
 450         return make_safe<decklink_consumer_proxy>(config);\r
 451 }\r
 452 \r
 453 }\r
 454 \r
 455 /*\r
 456 ##############################################################################\r
 457 Pre-rolling\r
 458 \r
 459 Mail: 2011-05-09\r
 460 \r
 461 Yoshan\r
 462 BMD Developer Support\r
 463 developer@blackmagic-design.com\r
 464 \r
 465 -----------------------------------------------------------------------------\r
 466 \r
 467 Thanks for your inquiry. The minimum number of frames that you can preroll \r
 468 for scheduled playback is three frames for video and four frames for audio. \r
 469 As you mentioned if you preroll less frames then playback will not start or\r
 470 playback will be very sporadic. From our experience with Media Express, we \r
 471 recommended that at least seven frames are prerolled for smooth playback. \r
 472 \r
 473 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:\r
 474 There can be around 3 frames worth of latency on scheduled output.\r
 475 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is\r
 476 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync() \r
 477 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will \r
 478 guarantee that the provided frame will be output as soon the previous \r
 479 frame output has been completed.\r
 480 ################################################################################\r
 481 */\r
 482 \r
 483 /*\r
 484 ##############################################################################\r
 485 Async DMA Transfer without redundant copying\r
 486 \r
 487 Mail: 2011-05-10\r
 488 \r
 489 Yoshan\r
 490 BMD Developer Support\r
 491 developer@blackmagic-design.com\r
 492 \r
 493 -----------------------------------------------------------------------------\r
 494 \r
 495 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame \r
 496 and providing a pointer to your video buffer when GetBytes() is called. \r
 497 This may help to keep copying to a minimum. Please ensure that the pixel \r
 498 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will \r
 499 have to colourspace convert which may result in additional copying.\r
 500 ################################################################################\r
 501 */