git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp

   1 /*\r
   2 * copyright (c) 2010 Sveriges Television AB <info@casparcg.com>\r
   3 *\r
   4 *  This file is part of CasparCG.\r
   5 *\r
   6 *    CasparCG is free software: you can redistribute it and/or modify\r
   7 *    it under the terms of the GNU General Public License as published by\r
   8 *    the Free Software Foundation, either version 3 of the License, or\r
   9 *    (at your option) any later version.\r
  10 *\r
  11 *    CasparCG is distributed in the hope that it will be useful,\r
  12 *    but WITHOUT ANY WARRANTY; without even the implied warranty of\r
  13 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\r
  14 *    GNU General Public License for more details.\r
  15 \r
  16 *    You should have received a copy of the GNU General Public License\r
  17 *    along with CasparCG.  If not, see <http://www.gnu.org/licenses/>.\r
  18 *\r
  19 */\r
  20 \r
  21 #include "../StdAfx.h"\r
  22  \r
  23 #include "decklink_consumer.h"\r
  24 \r
  25 #include "../util/util.h"\r
  26 \r
  27 #include "../interop/DeckLinkAPI_h.h"\r
  28 \r
  29 #include <core/mixer/read_frame.h>\r
  30 \r
  31 #include <common/concurrency/com_context.h>\r
  32 #include <common/diagnostics/graph.h>\r
  33 #include <common/exception/exceptions.h>\r
  34 #include <common/memory/memcpy.h>\r
  35 #include <common/memory/memclr.h>\r
  36 #include <common/memory/memshfl.h>\r
  37 \r
  38 #include <tbb/concurrent_queue.h>\r
  39 \r
  40 #include <boost/circular_buffer.hpp>\r
  41 #include <boost/timer.hpp>\r
  42 \r
  43 namespace caspar { \r
  44         \r
  45 struct configuration\r
  46 {\r
  47         size_t  device_index;\r
  48         bool    embedded_audio;\r
  49         bool    external_key;\r
  50         bool    low_latency;\r
  51         bool    key_only;\r
  52         size_t  buffer_depth;\r
  53         \r
  54         configuration()\r
  55                 : device_index(1)\r
  56                 , embedded_audio(false)\r
  57                 , external_key(false)\r
  58                 , low_latency(false)\r
  59                 , key_only(false)\r
  60                 , buffer_depth(CONSUMER_BUFFER_DEPTH){}\r
  61 };\r
  62 \r
  63 class decklink_frame_adapter : public IDeckLinkVideoFrame\r
  64 {\r
  65         const safe_ptr<const core::read_frame>  frame_;\r
  66         const core::video_format_desc                   format_desc_;\r
  67 public:\r
  68         decklink_frame_adapter(const safe_ptr<const core::read_frame>& frame, const core::video_format_desc& format_desc)\r
  69                 : frame_(frame)\r
  70                 , format_desc_(format_desc){}\r
  71         \r
  72         STDMETHOD (QueryInterface(REFIID, LPVOID*))     {return E_NOINTERFACE;}\r
  73         STDMETHOD_(ULONG, AddRef())                                     {return 1;}\r
  74         STDMETHOD_(ULONG, Release())                            {return 1;}\r
  75 \r
  76         STDMETHOD_(long, GetWidth())                            {return format_desc_.width;}        \r
  77     STDMETHOD_(long, GetHeight())                               {return format_desc_.height;}        \r
  78     STDMETHOD_(long, GetRowBytes())                             {return format_desc_.width*4;}        \r
  79         STDMETHOD_(BMDPixelFormat, GetPixelFormat()){return bmdFormat8BitBGRA;}        \r
  80     STDMETHOD_(BMDFrameFlags, GetFlags())               {return bmdFrameFlagDefault;}\r
  81         \r
  82     STDMETHOD(GetBytes(void** buffer))\r
  83         {\r
  84                 static std::vector<unsigned char> zeros(1920*1080*4, 0);\r
  85                 *buffer = const_cast<unsigned char*>(frame_->image_data().begin());\r
  86                 if(static_cast<size_t>(frame_->image_data().size()) != format_desc_.size)\r
  87                         *buffer = zeros.data();\r
  88                 return S_OK;\r
  89         }\r
  90         \r
  91     STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}        \r
  92     STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary))                 {return S_FALSE;}\r
  93 };\r
  94 \r
  95 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable\r
  96 {               \r
  97         const configuration config_;\r
  98 \r
  99         CComPtr<IDeckLink>                                      decklink_;\r
 100         CComQIPtr<IDeckLinkOutput>                      output_;\r
 101         CComQIPtr<IDeckLinkConfiguration>       configuration_;\r
 102         CComQIPtr<IDeckLinkKeyer>                       keyer_;\r
 103 \r
 104         std::exception_ptr exception_;\r
 105 \r
 106         tbb::atomic<bool> is_running_;\r
 107                 \r
 108         const std::wstring model_name_;\r
 109         const core::video_format_desc format_desc_;\r
 110         const size_t buffer_size_;\r
 111 \r
 112         unsigned long frames_scheduled_;\r
 113         unsigned long audio_scheduled_;\r
 114 \r
 115         size_t preroll_count_;\r
 116                 \r
 117         std::list<std::shared_ptr<IDeckLinkVideoFrame>> frame_container_; // Must be std::list in order to guarantee that pointers are always valid.\r
 118         boost::circular_buffer<std::vector<short>> audio_container_;\r
 119 \r
 120         tbb::concurrent_bounded_queue<std::shared_ptr<const core::read_frame>> video_frame_buffer_;\r
 121         tbb::concurrent_bounded_queue<std::shared_ptr<const core::read_frame>> audio_frame_buffer_;\r
 122         \r
 123         std::shared_ptr<diagnostics::graph> graph_;\r
 124         boost::timer tick_timer_;\r
 125 \r
 126 public:\r
 127         decklink_consumer(const configuration& config, const core::video_format_desc& format_desc) \r
 128                 : config_(config)\r
 129                 , decklink_(get_device(config.device_index))\r
 130                 , output_(decklink_)\r
 131                 , configuration_(decklink_)\r
 132                 , keyer_(decklink_)\r
 133                 , model_name_(get_model_name(decklink_))\r
 134                 , format_desc_(format_desc)\r
 135                 , buffer_size_(config.embedded_audio ? config.buffer_depth + 1 : config.buffer_depth) // Minimum buffer-size 3.\r
 136                 , frames_scheduled_(0)\r
 137                 , audio_scheduled_(0)\r
 138                 , preroll_count_(0)\r
 139                 , audio_container_(buffer_size_+1)\r
 140         {\r
 141                 is_running_ = true;\r
 142                                 \r
 143                 video_frame_buffer_.set_capacity(1);\r
 144                 audio_frame_buffer_.set_capacity(1);\r
 145 \r
 146                 graph_ = diagnostics::create_graph(narrow(print()));\r
 147                 graph_->add_guide("tick-time", 0.5);\r
 148                 graph_->set_color("tick-time", diagnostics::color(0.1f, 0.7f, 0.8f));\r
 149                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));\r
 150                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));\r
 151                 graph_->set_color("flushed-frame", diagnostics::color(0.3f, 0.3f, 0.6f));\r
 152                 \r
 153                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));\r
 154                                 \r
 155                 if(config.embedded_audio)\r
 156                         enable_audio();\r
 157 \r
 158                 set_latency(config.low_latency);                                \r
 159                 set_keyer(config.external_key);\r
 160                                 \r
 161                 if(config.embedded_audio)               \r
 162                         output_->BeginAudioPreroll();           \r
 163                 \r
 164                 for(size_t n = 0; n < buffer_size_; ++n)\r
 165                         schedule_next_video(make_safe<core::read_frame>());\r
 166 \r
 167                 if(!config.embedded_audio)\r
 168                         start_playback();\r
 169                                 \r
 170                 CASPAR_LOG(info) << print() << L" Buffer depth: " << buffer_size_;              \r
 171                 CASPAR_LOG(info) << print() << L" Successfully Initialized.";   \r
 172         }\r
 173 \r
 174         ~decklink_consumer()\r
 175         {               \r
 176                 is_running_ = false;\r
 177                 video_frame_buffer_.try_push(std::make_shared<core::read_frame>());\r
 178                 audio_frame_buffer_.try_push(std::make_shared<core::read_frame>());\r
 179 \r
 180                 if(output_ != nullptr) \r
 181                 {\r
 182                         output_->StopScheduledPlayback(0, nullptr, 0);\r
 183                         if(config_.embedded_audio)\r
 184                                 output_->DisableAudioOutput();\r
 185                         output_->DisableVideoOutput();\r
 186                 }\r
 187         }\r
 188                         \r
 189         const core::video_format_desc& get_video_format_desc() const\r
 190         {\r
 191                 return format_desc_;\r
 192         }\r
 193 \r
 194         void set_latency(bool low_latency)\r
 195         {               \r
 196                 if(!low_latency)\r
 197                 {\r
 198                         configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);\r
 199                         CASPAR_LOG(info) << print() << L" Enabled normal-latency mode";\r
 200                 }\r
 201                 else\r
 202                 {                       \r
 203                         configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);\r
 204                         CASPAR_LOG(info) << print() << L" Enabled low-latency mode";\r
 205                 }\r
 206         }\r
 207 \r
 208         void set_keyer(bool external_key)\r
 209         {\r
 210                 if(!external_key) \r
 211                 {\r
 212                         if(FAILED(keyer_->Enable(FALSE)))                       \r
 213                                 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";                   \r
 214                         else if(FAILED(keyer_->SetLevel(255)))                  \r
 215                                 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";\r
 216                         else\r
 217                                 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";             \r
 218                 }\r
 219                 else\r
 220                 {\r
 221                         if(FAILED(keyer_->Enable(TRUE)))                        \r
 222                                 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";   \r
 223                         else if(FAILED(keyer_->SetLevel(255)))                  \r
 224                                 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";\r
 225                         else\r
 226                                 CASPAR_LOG(info) << print() << L" Enabled external keyer.";                     \r
 227                 }\r
 228         }\r
 229         \r
 230         void enable_audio()\r
 231         {\r
 232                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType16bitInteger, 2, bmdAudioOutputStreamTimestamped)))\r
 233                                 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not enable audio output."));\r
 234                                 \r
 235                 if(FAILED(output_->SetAudioCallback(this)))\r
 236                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not set audio callback."));\r
 237 \r
 238                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";\r
 239         }\r
 240 \r
 241         void enable_video(BMDDisplayMode display_mode)\r
 242         {\r
 243                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault))) \r
 244                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not enable video output."));\r
 245                 \r
 246                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))\r
 247                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Failed to set playback completion callback."));\r
 248         }\r
 249 \r
 250         void start_playback()\r
 251         {\r
 252                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0))) \r
 253                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Failed to schedule playback."));\r
 254         }\r
 255         \r
 256         STDMETHOD (QueryInterface(REFIID, LPVOID*))     {return E_NOINTERFACE;}\r
 257         STDMETHOD_(ULONG, AddRef())                                     {return 1;}\r
 258         STDMETHOD_(ULONG, Release())                            {return 1;}\r
 259         \r
 260         STDMETHOD(ScheduledPlaybackHasStopped())\r
 261         {\r
 262                 is_running_ = false;\r
 263                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";\r
 264                 return S_OK;\r
 265         }\r
 266 \r
 267         STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))\r
 268         {\r
 269                 if(!is_running_)\r
 270                         return E_FAIL;\r
 271                 \r
 272                 try\r
 273                 {\r
 274                         if(result == bmdOutputFrameDisplayedLate)\r
 275                                 graph_->add_tag("late-frame");\r
 276                         else if(result == bmdOutputFrameDropped)\r
 277                                 graph_->add_tag("dropped-frame");\r
 278                         else if(result == bmdOutputFrameFlushed)\r
 279                                 graph_->add_tag("flushed-frame");\r
 280 \r
 281                         frame_container_.erase(std::find_if(frame_container_.begin(), frame_container_.end(), [&](const std::shared_ptr<IDeckLinkVideoFrame> frame)\r
 282                         {\r
 283                                 return frame.get() == completed_frame;\r
 284                         }));\r
 285 \r
 286                         std::shared_ptr<const core::read_frame> frame;  \r
 287                         video_frame_buffer_.pop(frame);         \r
 288                         \r
 289                         schedule_next_video(safe_ptr<const core::read_frame>(frame));                   \r
 290                 }\r
 291                 catch(...)\r
 292                 {\r
 293                         exception_ = std::current_exception();\r
 294                         return E_FAIL;\r
 295                 }\r
 296 \r
 297                 return S_OK;\r
 298         }\r
 299                 \r
 300         STDMETHOD(RenderAudioSamples(BOOL preroll))\r
 301         {\r
 302                 if(!is_running_)\r
 303                         return E_FAIL;\r
 304                 \r
 305                 try\r
 306                 {       \r
 307                         if(preroll)\r
 308                         {\r
 309                                 if(++preroll_count_ >= buffer_size_)\r
 310                                 {\r
 311                                         output_->EndAudioPreroll();\r
 312                                         start_playback();                               \r
 313                                 }\r
 314                                 else\r
 315                                         schedule_next_audio(make_safe<core::read_frame>());     \r
 316                         }\r
 317                         else\r
 318                         {\r
 319                                 std::shared_ptr<const core::read_frame> frame;\r
 320                                 audio_frame_buffer_.pop(frame);\r
 321                                 schedule_next_audio(safe_ptr<const core::read_frame>(frame));   \r
 322                         }\r
 323                 }\r
 324                 catch(...)\r
 325                 {\r
 326                         exception_ = std::current_exception();\r
 327                         return E_FAIL;\r
 328                 }\r
 329 \r
 330                 return S_OK;\r
 331         }\r
 332 \r
 333         void schedule_next_audio(const safe_ptr<const core::read_frame>& frame)\r
 334         {\r
 335                 static std::vector<short> silence(48000, 0);\r
 336                 \r
 337                 const int sample_count = format_desc_.audio_samples_per_frame;\r
 338                 const int sample_frame_count = sample_count/2;\r
 339 \r
 340                 const short* frame_audio_data = frame->audio_data().size() == sample_count ? frame->audio_data().begin() : silence.data();\r
 341                 audio_container_.push_back(std::vector<short>(frame_audio_data, frame_audio_data+sample_count));\r
 342 \r
 343                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, (audio_scheduled_++) * sample_frame_count, format_desc_.audio_sample_rate, nullptr)))\r
 344                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";\r
 345         }\r
 346                         \r
 347         void schedule_next_video(const safe_ptr<const core::read_frame>& frame)\r
 348         {\r
 349                 frame_container_.push_back(std::make_shared<decklink_frame_adapter>(frame, format_desc_));\r
 350                 if(FAILED(output_->ScheduleVideoFrame(frame_container_.back().get(), (frames_scheduled_++) * format_desc_.duration, format_desc_.duration, format_desc_.time_scale)))\r
 351                         CASPAR_LOG(error) << print() << L" Failed to schedule video.";\r
 352 \r
 353                 graph_->update_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);\r
 354                 tick_timer_.restart();\r
 355         }\r
 356 \r
 357         void send(const safe_ptr<const core::read_frame>& frame)\r
 358         {\r
 359                 if(exception_ != nullptr)\r
 360                         std::rethrow_exception(exception_);\r
 361 \r
 362                 if(!is_running_)\r
 363                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Is not running."));\r
 364                 \r
 365                 if(config_.embedded_audio)\r
 366                         audio_frame_buffer_.push(frame);        \r
 367                 video_frame_buffer_.push(frame);        \r
 368         }\r
 369         \r
 370         std::wstring print() const\r
 371         {\r
 372                 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(config_.device_index) + L"|" +  format_desc_.name + L"]";\r
 373         }\r
 374 };\r
 375 \r
 376 struct decklink_consumer_proxy : public core::frame_consumer\r
 377 {\r
 378         const configuration config_;\r
 379 \r
 380         com_context<decklink_consumer> context_;\r
 381 public:\r
 382 \r
 383         decklink_consumer_proxy(const configuration& config)\r
 384                 : config_(config)\r
 385                 , context_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]"){}\r
 386         \r
 387         virtual void initialize(const core::video_format_desc& format_desc)\r
 388         {\r
 389                 context_.reset([&]{return new decklink_consumer(config_, format_desc);});\r
 390         }\r
 391         \r
 392         virtual void send(const safe_ptr<const core::read_frame>& frame)\r
 393         {\r
 394                 context_->send(frame);\r
 395         }\r
 396         \r
 397         virtual std::wstring print() const\r
 398         {\r
 399                 return context_->print();\r
 400         }\r
 401 \r
 402         virtual bool key_only() const\r
 403         {\r
 404                 return config_.key_only;\r
 405         }\r
 406                 \r
 407         virtual const core::video_format_desc& get_video_format_desc() const\r
 408         {\r
 409                 return context_->get_video_format_desc();\r
 410         }\r
 411 \r
 412         virtual size_t buffer_depth() const\r
 413         {\r
 414                 return context_->buffer_size_;\r
 415         }\r
 416 };      \r
 417 \r
 418 safe_ptr<core::frame_consumer> create_decklink_consumer(const std::vector<std::wstring>& params) \r
 419 {\r
 420         if(params.size() < 1 || params[0] != L"DECKLINK")\r
 421                 return core::frame_consumer::empty();\r
 422         \r
 423         configuration config;\r
 424                 \r
 425         if(params.size() > 1)\r
 426                 config.device_index = lexical_cast_or_default<int>(params[1], config.device_index);\r
 427         \r
 428         config.external_key             = std::find(params.begin(), params.end(), L"EXTERNAL_KEY")       != params.end();\r
 429         config.low_latency              = std::find(params.begin(), params.end(), L"LOW_LATENCY")        != params.end();\r
 430         config.embedded_audio   = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();\r
 431         config.key_only                 = std::find(params.begin(), params.end(), L"KEY_ONLY")           != params.end();\r
 432 \r
 433         return make_safe<decklink_consumer_proxy>(config);\r
 434 }\r
 435 \r
 436 safe_ptr<core::frame_consumer> create_decklink_consumer(const boost::property_tree::ptree& ptree) \r
 437 {\r
 438         configuration config;\r
 439 \r
 440         config.external_key             = ptree.get("external-key",       config.external_key);\r
 441         config.low_latency              = ptree.get("low-latency",        config.low_latency);\r
 442         config.key_only                 = ptree.get("key-only",           config.key_only);\r
 443         config.device_index             = ptree.get("device",             config.device_index);\r
 444         config.embedded_audio   = ptree.get("embedded-audio", config.embedded_audio);\r
 445 \r
 446         return make_safe<decklink_consumer_proxy>(config);\r
 447 }\r
 448 \r
 449 }\r
 450 \r
 451 /*\r
 452 ##############################################################################\r
 453 Pre-rolling\r
 454 \r
 455 Mail: 2011-05-09\r
 456 \r
 457 Yoshan\r
 458 BMD Developer Support\r
 459 developer@blackmagic-design.com\r
 460 \r
 461 -----------------------------------------------------------------------------\r
 462 \r
 463 Thanks for your inquiry. The minimum number of frames that you can preroll \r
 464 for scheduled playback is three frames for video and four frames for audio. \r
 465 As you mentioned if you preroll less frames then playback will not start or\r
 466 playback will be very sporadic. From our experience with Media Express, we \r
 467 recommended that at least seven frames are prerolled for smooth playback. \r
 468 \r
 469 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:\r
 470 There can be around 3 frames worth of latency on scheduled output.\r
 471 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is\r
 472 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync() \r
 473 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will \r
 474 guarantee that the provided frame will be output as soon the previous \r
 475 frame output has been completed.\r
 476 ################################################################################\r
 477 */\r
 478 \r
 479 /*\r
 480 ##############################################################################\r
 481 Async DMA Transfer without redundant copying\r
 482 \r
 483 Mail: 2011-05-10\r
 484 \r
 485 Yoshan\r
 486 BMD Developer Support\r
 487 developer@blackmagic-design.com\r
 488 \r
 489 -----------------------------------------------------------------------------\r
 490 \r
 491 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame \r
 492 and providing a pointer to your video buffer when GetBytes() is called. \r
 493 This may help to keep copying to a minimum. Please ensure that the pixel \r
 494 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will \r
 495 have to colourspace convert which may result in additional copying.\r
 496 ################################################################################\r
 497 */