git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp

   1 /*\r
   2 * copyright (c) 2010 Sveriges Television AB <info@casparcg.com>\r
   3 *\r
   4 *  This file is part of CasparCG.\r
   5 *\r
   6 *    CasparCG is free software: you can redistribute it and/or modify\r
   7 *    it under the terms of the GNU General Public License as published by\r
   8 *    the Free Software Foundation, either version 3 of the License, or\r
   9 *    (at your option) any later version.\r
  10 *\r
  11 *    CasparCG is distributed in the hope that it will be useful,\r
  12 *    but WITHOUT ANY WARRANTY; without even the implied warranty of\r
  13 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\r
  14 *    GNU General Public License for more details.\r
  15 \r
  16 *    You should have received a copy of the GNU General Public License\r
  17 *    along with CasparCG.  If not, see <http://www.gnu.org/licenses/>.\r
  18 *\r
  19 */\r
  20 \r
  21 #include "../StdAfx.h"\r
  22  \r
  23 #include "decklink_consumer.h"\r
  24 \r
  25 #include "../util/util.h"\r
  26 \r
  27 #include "../interop/DeckLinkAPI_h.h"\r
  28 \r
  29 #include <core/mixer/read_frame.h>\r
  30 \r
  31 #include <common/concurrency/com_context.h>\r
  32 #include <common/diagnostics/graph.h>\r
  33 #include <common/exception/exceptions.h>\r
  34 #include <common/memory/memcpy.h>\r
  35 #include <common/memory/memclr.h>\r
  36 #include <common/memory/memshfl.h>\r
  37 \r
  38 #include <tbb/concurrent_queue.h>\r
  39 \r
  40 #include <boost/circular_buffer.hpp>\r
  41 #include <boost/timer.hpp>\r
  42 \r
  43 namespace caspar { \r
  44         \r
  45 enum key\r
  46 {\r
  47         external_key,\r
  48         internal_key,\r
  49         default_key\r
  50 };\r
  51 \r
  52 enum latency\r
  53 {\r
  54         low_latency,\r
  55         normal_latency,\r
  56         default_latency\r
  57 };\r
  58 \r
  59 enum output_pixels\r
  60 {\r
  61         fill_and_key,\r
  62         key_only\r
  63 };\r
  64 \r
  65 struct configuration\r
  66 {\r
  67         size_t  device_index;\r
  68         bool    embedded_audio;\r
  69         key             keyer;\r
  70         latency latency;\r
  71         output_pixels  output;\r
  72         \r
  73         configuration()\r
  74                 : device_index(1)\r
  75                 , embedded_audio(false)\r
  76                 , keyer(default_key)\r
  77                 , latency(default_latency)\r
  78                 , output(fill_and_key){}\r
  79 };\r
  80 \r
  81 class decklink_frame_adapter : public IDeckLinkVideoFrame\r
  82 {\r
  83         const safe_ptr<const core::read_frame>  frame_;\r
  84         const core::video_format_desc                   format_desc_;\r
  85 public:\r
  86         decklink_frame_adapter(const safe_ptr<const core::read_frame>& frame, const core::video_format_desc& format_desc)\r
  87                 : frame_(frame)\r
  88                 , format_desc_(format_desc){}\r
  89         \r
  90         STDMETHOD (QueryInterface(REFIID, LPVOID*))     {return E_NOINTERFACE;}\r
  91         STDMETHOD_(ULONG, AddRef())                                     {return 1;}\r
  92         STDMETHOD_(ULONG, Release())                            {return 1;}\r
  93 \r
  94         STDMETHOD_(long, GetWidth())                            {return format_desc_.width;}        \r
  95     STDMETHOD_(long, GetHeight())                               {return format_desc_.height;}        \r
  96     STDMETHOD_(long, GetRowBytes())                             {return format_desc_.width*4;}        \r
  97         STDMETHOD_(BMDPixelFormat, GetPixelFormat()){return bmdFormat8BitBGRA;}        \r
  98     STDMETHOD_(BMDFrameFlags, GetFlags())               {return bmdFrameFlagDefault;}\r
  99         \r
 100     STDMETHOD(GetBytes(void** buffer))\r
 101         {\r
 102                 static std::vector<unsigned char> zeros(1920*1080*4, 0);\r
 103                 *buffer = const_cast<unsigned char*>(frame_->image_data().begin());\r
 104                 if(static_cast<size_t>(frame_->image_data().size()) != format_desc_.size)\r
 105                         *buffer = zeros.data();\r
 106                 return S_OK;\r
 107         }\r
 108         \r
 109     STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}        \r
 110     STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary))                 {return S_FALSE;}\r
 111 };\r
 112 \r
 113 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable\r
 114 {               \r
 115         const configuration config_;\r
 116 \r
 117         CComPtr<IDeckLink>                                      decklink_;\r
 118         CComQIPtr<IDeckLinkOutput>                      output_;\r
 119         CComQIPtr<IDeckLinkConfiguration>       configuration_;\r
 120         CComQIPtr<IDeckLinkKeyer>                       keyer_;\r
 121 \r
 122         std::exception_ptr exception_;\r
 123 \r
 124         tbb::atomic<bool> is_running_;\r
 125                 \r
 126         const std::wstring model_name_;\r
 127         const core::video_format_desc format_desc_;\r
 128         const size_t buffer_size_;\r
 129 \r
 130         unsigned long frames_scheduled_;\r
 131         unsigned long audio_scheduled_;\r
 132                 \r
 133         std::list<std::shared_ptr<IDeckLinkVideoFrame>> frame_container_; // Must be std::list in order to guarantee that pointers are always valid.\r
 134         boost::circular_buffer<std::vector<short>> audio_container_;\r
 135 \r
 136         tbb::concurrent_bounded_queue<std::shared_ptr<const core::read_frame>> video_frame_buffer_;\r
 137         tbb::concurrent_bounded_queue<std::shared_ptr<const core::read_frame>> audio_frame_buffer_;\r
 138         \r
 139         std::shared_ptr<diagnostics::graph> graph_;\r
 140         boost::timer tick_timer_;\r
 141 \r
 142 public:\r
 143         decklink_consumer(const configuration& config, const core::video_format_desc& format_desc) \r
 144                 : config_(config)\r
 145                 , decklink_(get_device(config.device_index))\r
 146                 , output_(decklink_)\r
 147                 , configuration_(decklink_)\r
 148                 , keyer_(decklink_)\r
 149                 , model_name_(get_model_name(decklink_))\r
 150                 , format_desc_(format_desc)\r
 151                 , buffer_size_(config.embedded_audio ? 5 : 4) // Minimum buffer-size (3 + 1 tolerance).\r
 152                 , frames_scheduled_(0)\r
 153                 , audio_scheduled_(0)\r
 154                 , audio_container_(buffer_size_+1)\r
 155         {\r
 156                 is_running_ = true;\r
 157                                 \r
 158                 video_frame_buffer_.set_capacity(1);\r
 159                 audio_frame_buffer_.set_capacity(1);\r
 160 \r
 161                 graph_ = diagnostics::create_graph(narrow(print()));\r
 162                 graph_->add_guide("tick-time", 0.5);\r
 163                 graph_->set_color("tick-time", diagnostics::color(0.1f, 0.7f, 0.8f));\r
 164                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));\r
 165                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));\r
 166                 graph_->set_color("flushed-frame", diagnostics::color(0.3f, 0.3f, 0.6f));\r
 167                 \r
 168                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));\r
 169                                 \r
 170                 if(config.embedded_audio)\r
 171                         enable_audio();\r
 172 \r
 173                 set_latency(config.latency);                            \r
 174                 set_keyer(config.keyer);\r
 175                                                                 \r
 176                 for(size_t n = 0; n < buffer_size_; ++n)\r
 177                         schedule_next_video(core::read_frame::empty());\r
 178                 \r
 179                 if(config.embedded_audio)\r
 180                         output_->BeginAudioPreroll();\r
 181                 else\r
 182                         start_playback();\r
 183                 \r
 184                 CASPAR_LOG(info) << print() << L" Buffer depth: " << buffer_size_;              \r
 185                 CASPAR_LOG(info) << print() << L" Successfully Initialized.";   \r
 186         }\r
 187 \r
 188         ~decklink_consumer()\r
 189         {               \r
 190                 is_running_ = false;\r
 191                 video_frame_buffer_.try_push(core::read_frame::empty());\r
 192                 audio_frame_buffer_.try_push(core::read_frame::empty());\r
 193 \r
 194                 if(output_ != nullptr) \r
 195                 {\r
 196                         output_->StopScheduledPlayback(0, nullptr, 0);\r
 197                         if(config_.embedded_audio)\r
 198                                 output_->DisableAudioOutput();\r
 199                         output_->DisableVideoOutput();\r
 200                 }\r
 201         }\r
 202                         \r
 203         void set_latency(latency latency)\r
 204         {               \r
 205                 if(latency == normal_latency)\r
 206                 {\r
 207                         configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);\r
 208                         CASPAR_LOG(info) << print() << L" Enabled normal-latency mode";\r
 209                 }\r
 210                 else if(latency == low_latency)\r
 211                 {                       \r
 212                         configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);\r
 213                         CASPAR_LOG(info) << print() << L" Enabled low-latency mode";\r
 214                 }\r
 215                 else\r
 216                         CASPAR_LOG(info) << print() << L" Uses driver latency settings.";       \r
 217         }\r
 218 \r
 219         void set_keyer(key keyer)\r
 220         {\r
 221                 if(keyer == internal_key) \r
 222                 {\r
 223                         if(FAILED(keyer_->Enable(FALSE)))                       \r
 224                                 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";                   \r
 225                         else if(FAILED(keyer_->SetLevel(255)))                  \r
 226                                 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";\r
 227                         else\r
 228                                 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";             \r
 229                 }\r
 230                 else if(keyer == external_key)\r
 231                 {\r
 232                         if(FAILED(keyer_->Enable(TRUE)))                        \r
 233                                 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";   \r
 234                         else if(FAILED(keyer_->SetLevel(255)))                  \r
 235                                 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";\r
 236                         else\r
 237                                 CASPAR_LOG(info) << print() << L" Enabled external keyer.";                     \r
 238                 }\r
 239                 else\r
 240                         CASPAR_LOG(info) << print() << L" Uses driver keyer settings."; \r
 241         }\r
 242         \r
 243         void enable_audio()\r
 244         {\r
 245                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType16bitInteger, 2, bmdAudioOutputStreamTimestamped)))\r
 246                                 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not enable audio output."));\r
 247                                 \r
 248                 if(FAILED(output_->SetAudioCallback(this)))\r
 249                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not set audio callback."));\r
 250 \r
 251                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";\r
 252         }\r
 253 \r
 254         void enable_video(BMDDisplayMode display_mode)\r
 255         {\r
 256                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault))) \r
 257                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not enable video output."));\r
 258                 \r
 259                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))\r
 260                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Failed to set playback completion callback."));\r
 261         }\r
 262 \r
 263         void start_playback()\r
 264         {\r
 265                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0))) \r
 266                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Failed to schedule playback."));\r
 267         }\r
 268         \r
 269         STDMETHOD (QueryInterface(REFIID, LPVOID*))     {return E_NOINTERFACE;}\r
 270         STDMETHOD_(ULONG, AddRef())                                     {return 1;}\r
 271         STDMETHOD_(ULONG, Release())                            {return 1;}\r
 272         \r
 273         STDMETHOD(ScheduledPlaybackHasStopped())\r
 274         {\r
 275                 is_running_ = false;\r
 276                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";\r
 277                 return S_OK;\r
 278         }\r
 279 \r
 280         STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))\r
 281         {\r
 282                 if(!is_running_)\r
 283                         return E_FAIL;\r
 284                 \r
 285                 try\r
 286                 {\r
 287                         if(result == bmdOutputFrameDisplayedLate)\r
 288                                 graph_->add_tag("late-frame");\r
 289                         else if(result == bmdOutputFrameDropped)\r
 290                                 graph_->add_tag("dropped-frame");\r
 291                         else if(result == bmdOutputFrameFlushed)\r
 292                                 graph_->add_tag("flushed-frame");\r
 293 \r
 294                         frame_container_.erase(std::find_if(frame_container_.begin(), frame_container_.end(), [&](const std::shared_ptr<IDeckLinkVideoFrame> frame)\r
 295                         {\r
 296                                 return frame.get() == completed_frame;\r
 297                         }));\r
 298 \r
 299                         std::shared_ptr<const core::read_frame> frame;  \r
 300                         video_frame_buffer_.pop(frame);         \r
 301                         schedule_next_video(safe_ptr<const core::read_frame>(frame));                   \r
 302                 }\r
 303                 catch(...)\r
 304                 {\r
 305                         exception_ = std::current_exception();\r
 306                         return E_FAIL;\r
 307                 }\r
 308 \r
 309                 return S_OK;\r
 310         }\r
 311                 \r
 312         STDMETHOD(RenderAudioSamples(BOOL preroll))\r
 313         {\r
 314                 if(!is_running_)\r
 315                         return E_FAIL;\r
 316                 \r
 317                 try\r
 318                 {\r
 319                         std::shared_ptr<const core::read_frame> frame;\r
 320                         audio_frame_buffer_.pop(frame);\r
 321                         schedule_next_audio(safe_ptr<const core::read_frame>(frame));           \r
 322 \r
 323                         if(preroll)\r
 324                                 start_playback();\r
 325                 }\r
 326                 catch(...)\r
 327                 {\r
 328                         exception_ = std::current_exception();\r
 329                         return E_FAIL;\r
 330                 }\r
 331 \r
 332                 return S_OK;\r
 333         }\r
 334 \r
 335         void schedule_next_audio(const safe_ptr<const core::read_frame>& frame)\r
 336         {\r
 337                 static std::vector<short> silence(48000, 0);\r
 338                 \r
 339                 const int sample_count = format_desc_.audio_samples_per_frame;\r
 340                 const int sample_frame_count = sample_count/2;\r
 341 \r
 342                 const short* frame_audio_data = frame->audio_data().size() == sample_count ? frame->audio_data().begin() : silence.data();\r
 343                 audio_container_.push_back(std::vector<short>(frame_audio_data, frame_audio_data+sample_count));\r
 344 \r
 345                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, (audio_scheduled_++) * sample_frame_count, format_desc_.audio_sample_rate, nullptr)))\r
 346                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";\r
 347         }\r
 348                         \r
 349         void schedule_next_video(const safe_ptr<const core::read_frame>& frame)\r
 350         {\r
 351                 frame_container_.push_back(std::make_shared<decklink_frame_adapter>(frame, format_desc_));\r
 352                 if(FAILED(output_->ScheduleVideoFrame(frame_container_.back().get(), (frames_scheduled_++) * format_desc_.duration, format_desc_.duration, format_desc_.time_scale)))\r
 353                         CASPAR_LOG(error) << print() << L" Failed to schedule video.";\r
 354 \r
 355                 graph_->update_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);\r
 356                 tick_timer_.restart();\r
 357         }\r
 358 \r
 359         void send(const safe_ptr<const core::read_frame>& frame)\r
 360         {\r
 361                 if(exception_ != nullptr)\r
 362                         std::rethrow_exception(exception_);\r
 363 \r
 364                 if(!is_running_)\r
 365                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Is not running."));\r
 366                 \r
 367                 if(config_.embedded_audio)\r
 368                         audio_frame_buffer_.push(frame);        \r
 369                 video_frame_buffer_.push(frame);        \r
 370         }\r
 371         \r
 372         std::wstring print() const\r
 373         {\r
 374                 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(config_.device_index) + L"|" +  format_desc_.name + L"]";\r
 375         }\r
 376 };\r
 377 \r
 378 struct decklink_consumer_proxy : public core::frame_consumer\r
 379 {\r
 380         const configuration config_;\r
 381 \r
 382         com_context<decklink_consumer> context_;\r
 383 public:\r
 384 \r
 385         decklink_consumer_proxy(const configuration& config)\r
 386                 : config_(config)\r
 387                 , context_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]"){}\r
 388         \r
 389         virtual void initialize(const core::video_format_desc& format_desc)\r
 390         {\r
 391                 context_.reset([&]{return new decklink_consumer(config_, format_desc);});\r
 392         }\r
 393         \r
 394         virtual void send(const safe_ptr<const core::read_frame>& frame)\r
 395         {\r
 396                 context_->send(frame);\r
 397         }\r
 398         \r
 399         virtual std::wstring print() const\r
 400         {\r
 401                 return context_->print();\r
 402         }\r
 403 \r
 404         virtual bool key_only() const\r
 405         {\r
 406                 return (config_.output == caspar::key_only);\r
 407         }\r
 408 };      \r
 409 \r
 410 safe_ptr<core::frame_consumer> create_decklink_consumer(const std::vector<std::wstring>& params) \r
 411 {\r
 412         if(params.size() < 1 || params[0] != L"DECKLINK")\r
 413                 return core::frame_consumer::empty();\r
 414         \r
 415         configuration config;\r
 416                 \r
 417         if(params.size() > 1)\r
 418                 config.device_index = lexical_cast_or_default<int>(params[1], config.device_index);\r
 419                 \r
 420         if(std::find(params.begin(), params.end(), L"INTERNAL_KEY") != params.end())\r
 421                 config.keyer = internal_key;\r
 422         else if(std::find(params.begin(), params.end(), L"EXTERNAL_KEY") != params.end())\r
 423                 config.keyer = external_key;\r
 424         \r
 425         if(std::find(params.begin(), params.end(), L"LOW_LATENCY") != params.end())\r
 426                 config.latency = low_latency;\r
 427         else if(std::find(params.begin(), params.end(), L"NORMAL_LATENCY") != params.end())\r
 428                 config.latency = normal_latency;\r
 429                 \r
 430         config.embedded_audio = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();\r
 431         config.output = std::find(params.begin(), params.end(), L"KEY_ONLY") != params.end() ? key_only : fill_and_key;\r
 432 \r
 433         return make_safe<decklink_consumer_proxy>(config);\r
 434 }\r
 435 \r
 436 safe_ptr<core::frame_consumer> create_decklink_consumer(const boost::property_tree::ptree& ptree) \r
 437 {\r
 438         configuration config;\r
 439 \r
 440         auto key_str = ptree.get("key", "default");\r
 441         if(key_str == "internal")\r
 442                 config.keyer = internal_key;\r
 443         else if(key_str == "external")\r
 444                 config.keyer = external_key;\r
 445 \r
 446         auto latency_str = ptree.get("latency", "default");\r
 447         if(latency_str == "normal")\r
 448                 config.latency = normal_latency;\r
 449         else if(latency_str == "low")\r
 450                 config.latency = low_latency;\r
 451 \r
 452         auto output_str = ptree.get("output", "fill_and_key");\r
 453         if(output_str == "key_only")\r
 454                 config.output = key_only;\r
 455 \r
 456         config.device_index = ptree.get("device", 0);\r
 457         config.embedded_audio  = ptree.get("embedded-audio", false);\r
 458 \r
 459         return make_safe<decklink_consumer_proxy>(config);\r
 460 }\r
 461 \r
 462 }\r
 463 \r
 464 /*\r
 465 ##############################################################################\r
 466 Pre-rolling\r
 467 \r
 468 Mail: 2011-05-09\r
 469 \r
 470 Yoshan\r
 471 BMD Developer Support\r
 472 developer@blackmagic-design.com\r
 473 \r
 474 -----------------------------------------------------------------------------\r
 475 \r
 476 Thanks for your inquiry. The minimum number of frames that you can preroll \r
 477 for scheduled playback is three frames for video and four frames for audio. \r
 478 As you mentioned if you preroll less frames then playback will not start or\r
 479 playback will be very sporadic. From our experience with Media Express, we \r
 480 recommended that at least seven frames are prerolled for smooth playback. \r
 481 \r
 482 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:\r
 483 There can be around 3 frames worth of latency on scheduled output.\r
 484 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is\r
 485 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync() \r
 486 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will \r
 487 guarantee that the provided frame will be output as soon the previous \r
 488 frame output has been completed.\r
 489 ################################################################################\r
 490 */\r
 491 \r
 492 /*\r
 493 ##############################################################################\r
 494 Async DMA Transfer without redundant copying\r
 495 \r
 496 Mail: 2011-05-10\r
 497 \r
 498 Yoshan\r
 499 BMD Developer Support\r
 500 developer@blackmagic-design.com\r
 501 \r
 502 -----------------------------------------------------------------------------\r
 503 \r
 504 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame \r
 505 and providing a pointer to your video buffer when GetBytes() is called. \r
 506 This may help to keep copying to a minimum. Please ensure that the pixel \r
 507 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will \r
 508 have to colourspace convert which may result in additional copying.\r
 509 ################################################################################\r
 510 */