git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp

   1 /*\r
   2 * copyright (c) 2010 Sveriges Television AB <info@casparcg.com>\r
   3 *\r
   4 *  This file is part of CasparCG.\r
   5 *\r
   6 *    CasparCG is free software: you can redistribute it and/or modify\r
   7 *    it under the terms of the GNU General Public License as published by\r
   8 *    the Free Software Foundation, either version 3 of the License, or\r
   9 *    (at your option) any later version.\r
  10 *\r
  11 *    CasparCG is distributed in the hope that it will be useful,\r
  12 *    but WITHOUT ANY WARRANTY; without even the implied warranty of\r
  13 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\r
  14 *    GNU General Public License for more details.\r
  15 \r
  16 *    You should have received a copy of the GNU General Public License\r
  17 *    along with CasparCG.  If not, see <http://www.gnu.org/licenses/>.\r
  18 *\r
  19 */\r
  20 \r
  21 #include "../StdAfx.h"\r
  22  \r
  23 #include "decklink_consumer.h"\r
  24 \r
  25 #include "../util/util.h"\r
  26 \r
  27 #include "../interop/DeckLinkAPI_h.h"\r
  28 \r
  29 #include <core/video_format.h>\r
  30 \r
  31 #include <core/consumer/frame/read_frame.h>\r
  32 \r
  33 #include <common/concurrency/com_context.h>\r
  34 #include <common/diagnostics/graph.h>\r
  35 #include <common/exception/exceptions.h>\r
  36 #include <common/memory/memcpy.h>\r
  37 #include <common/memory/memclr.h>\r
  38 \r
  39 #include <tbb/concurrent_queue.h>\r
  40 \r
  41 #include <boost/circular_buffer.hpp>\r
  42 #include <boost/timer.hpp>\r
  43 \r
  44 #include <vector>\r
  45 \r
  46 #pragma warning(push)\r
  47 #pragma warning(disable : 4996)\r
  48 \r
  49         #include <atlbase.h>\r
  50 \r
  51         #include <atlcom.h>\r
  52         #include <atlhost.h>\r
  53 \r
  54 #pragma warning(push)\r
  55 \r
  56 namespace caspar { \r
  57         \r
  58 enum key\r
  59 {\r
  60         external_key,\r
  61         internal_key,\r
  62         default_key\r
  63 };\r
  64 \r
  65 enum latency\r
  66 {\r
  67         low_latency,\r
  68         normal_latency,\r
  69         default_latency\r
  70 };\r
  71 \r
  72 struct configuration\r
  73 {\r
  74         size_t device_index;\r
  75         bool embedded_audio;\r
  76         key keyer;\r
  77         latency latency;\r
  78         \r
  79         configuration()\r
  80                 : device_index(1)\r
  81                 , embedded_audio(false)\r
  82                 , keyer(default_key)\r
  83                 , latency(default_latency){}\r
  84 };\r
  85 \r
  86 class decklink_frame_adapter : public IDeckLinkVideoFrame\r
  87 {\r
  88         safe_ptr<const core::read_frame> frame_;\r
  89         core::video_format_desc format_desc_;\r
  90 public:\r
  91         decklink_frame_adapter(const safe_ptr<const core::read_frame>& frame, const core::video_format_desc& format_desc)\r
  92                 : frame_(frame)\r
  93                 , format_desc_(format_desc){}\r
  94         \r
  95         STDMETHOD (QueryInterface(REFIID, LPVOID*))     {return E_NOINTERFACE;}\r
  96         STDMETHOD_(ULONG, AddRef())                                     {return 1;}\r
  97         STDMETHOD_(ULONG, Release())                            {return 1;}\r
  98 \r
  99         STDMETHOD_(long, GetWidth())                            {return format_desc_.width;}        \r
 100     STDMETHOD_(long, GetHeight())                               {return format_desc_.height;}        \r
 101     STDMETHOD_(long, GetRowBytes())                             {return format_desc_.width*4;}        \r
 102         STDMETHOD_(BMDPixelFormat, GetPixelFormat()){return bmdFormat8BitBGRA;}        \r
 103     STDMETHOD_(BMDFrameFlags, GetFlags())               {return bmdFrameFlagDefault;}\r
 104         \r
 105     STDMETHOD(GetBytes(void** buffer))\r
 106         {\r
 107                 static std::vector<unsigned char> zeros(1920*1080*4, 0);\r
 108                 *buffer = const_cast<unsigned char*>(frame_->image_data().begin());\r
 109                 if(static_cast<size_t>(frame_->image_data().size()) != format_desc_.size)\r
 110                         *buffer = zeros.data();\r
 111                 return S_OK;\r
 112         }\r
 113         \r
 114     STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}        \r
 115     STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary))                 {return S_FALSE;}\r
 116 };\r
 117 \r
 118 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable\r
 119 {               \r
 120         const configuration config_;\r
 121 \r
 122         CComPtr<IDeckLink>                                      decklink_;\r
 123         CComQIPtr<IDeckLinkOutput>                      output_;\r
 124         CComQIPtr<IDeckLinkConfiguration>       configuration_;\r
 125         CComQIPtr<IDeckLinkKeyer>                       keyer_;\r
 126 \r
 127         std::exception_ptr exception_;\r
 128 \r
 129         tbb::atomic<bool> is_running_;\r
 130                 \r
 131         const std::wstring model_name_;\r
 132         const core::video_format_desc format_desc_;\r
 133         const size_t buffer_size_;\r
 134 \r
 135         unsigned long frames_scheduled_;\r
 136         unsigned long audio_scheduled_;\r
 137                 \r
 138         std::list<decklink_frame_adapter> frame_container_; // Must be std::list in order to guarantee that pointers are always valid.\r
 139         boost::circular_buffer<std::vector<short>> audio_container_;\r
 140 \r
 141         tbb::concurrent_bounded_queue<std::shared_ptr<const core::read_frame>> video_frame_buffer_;\r
 142         tbb::concurrent_bounded_queue<std::shared_ptr<const core::read_frame>> audio_frame_buffer_;\r
 143         \r
 144         std::shared_ptr<diagnostics::graph> graph_;\r
 145         boost::timer tick_timer_;\r
 146 \r
 147 \r
 148 public:\r
 149         decklink_consumer(const configuration& config, const core::video_format_desc& format_desc) \r
 150                 : config_(config)\r
 151                 , decklink_(get_device(config.device_index))\r
 152                 , output_(decklink_)\r
 153                 , configuration_(decklink_)\r
 154                 , keyer_(decklink_)\r
 155                 , model_name_(get_model_name(decklink_))\r
 156                 , format_desc_(format_desc)\r
 157                 , buffer_size_(config.embedded_audio ? 5 : 4) // Minimum buffer-size (3 + 1 tolerance).\r
 158                 , frames_scheduled_(0)\r
 159                 , audio_scheduled_(0)\r
 160                 , audio_container_(buffer_size_+1)\r
 161         {\r
 162                 is_running_ = true;\r
 163                                 \r
 164                 video_frame_buffer_.set_capacity(1);\r
 165                 audio_frame_buffer_.set_capacity(1);\r
 166 \r
 167                 graph_ = diagnostics::create_graph(narrow(print()));\r
 168                 graph_->add_guide("tick-time", 0.5);\r
 169                 graph_->set_color("tick-time", diagnostics::color(0.1f, 0.7f, 0.8f));\r
 170                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));\r
 171                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));\r
 172                 graph_->set_color("flushed-frame", diagnostics::color(0.3f, 0.3f, 0.6f));\r
 173                 \r
 174                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));\r
 175                                 \r
 176                 if(config.embedded_audio)\r
 177                         enable_audio();\r
 178 \r
 179                 set_latency(config.latency);                            \r
 180                 set_keyer(config.keyer);\r
 181                                                                 \r
 182                 for(size_t n = 0; n < buffer_size_; ++n)\r
 183                         schedule_next_video(core::read_frame::empty());\r
 184                 \r
 185                 if(config.embedded_audio)\r
 186                         output_->BeginAudioPreroll();\r
 187                 else\r
 188                 {\r
 189                         if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0))) \r
 190                                 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Failed to schedule playback."));\r
 191                 }\r
 192                 \r
 193                 CASPAR_LOG(info) << print() << L" Buffer depth: " << buffer_size_;              \r
 194                 CASPAR_LOG(info) << print() << L" Successfully initialized for " << format_desc_.name;  \r
 195         }\r
 196 \r
 197         ~decklink_consumer()\r
 198         {               \r
 199                 is_running_ = false;\r
 200                 video_frame_buffer_.clear();\r
 201                 audio_frame_buffer_.clear();\r
 202                 video_frame_buffer_.try_push(core::read_frame::empty());\r
 203                 audio_frame_buffer_.try_push(core::read_frame::empty());\r
 204 \r
 205                 if(output_ != nullptr) \r
 206                 {\r
 207                         output_->StopScheduledPlayback(0, nullptr, 0);\r
 208                         if(config_.embedded_audio)\r
 209                                 output_->DisableAudioOutput();\r
 210                         output_->DisableVideoOutput();\r
 211                 }\r
 212         }\r
 213                         \r
 214         void set_latency(latency latency)\r
 215         {               \r
 216                 if(latency == normal_latency)\r
 217                 {\r
 218                         configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);\r
 219                         CASPAR_LOG(info) << print() << L" Enabled normal-latency mode";\r
 220                 }\r
 221                 else if(latency == low_latency)\r
 222                 {                       \r
 223                         configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);\r
 224                         CASPAR_LOG(info) << print() << L" Enabled low-latency mode";\r
 225                 }\r
 226                 else\r
 227                         CASPAR_LOG(info) << print() << L" Uses driver latency settings.";       \r
 228         }\r
 229 \r
 230         void set_keyer(key keyer)\r
 231         {\r
 232                 if(keyer == internal_key) \r
 233                 {\r
 234                         if(FAILED(keyer_->Enable(FALSE)))                       \r
 235                                 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";                   \r
 236                         else if(FAILED(keyer_->SetLevel(255)))                  \r
 237                                 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";\r
 238                         else\r
 239                                 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";             \r
 240                 }\r
 241                 else if(keyer == external_key)\r
 242                 {\r
 243                         if(FAILED(keyer_->Enable(TRUE)))                        \r
 244                                 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";   \r
 245                         else if(FAILED(keyer_->SetLevel(255)))                  \r
 246                                 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";\r
 247                         else\r
 248                                 CASPAR_LOG(info) << print() << L" Enabled external keyer.";                     \r
 249                 }\r
 250                 else\r
 251                         CASPAR_LOG(info) << print() << L" Uses driver keyer settings."; \r
 252         }\r
 253         \r
 254         void enable_audio()\r
 255         {\r
 256                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType16bitInteger, 2, bmdAudioOutputStreamTimestamped)))\r
 257                                 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not enable audio output."));\r
 258                                 \r
 259                 if(FAILED(output_->SetAudioCallback(this)))\r
 260                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not set audio callback."));\r
 261 \r
 262                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";\r
 263         }\r
 264 \r
 265         void enable_video(BMDDisplayMode display_mode)\r
 266         {\r
 267                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault))) \r
 268                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not enable video output."));\r
 269                 \r
 270                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))\r
 271                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Failed to set playback completion callback."));\r
 272         }\r
 273         \r
 274         STDMETHOD (QueryInterface(REFIID, LPVOID*))     {return E_NOINTERFACE;}\r
 275         STDMETHOD_(ULONG, AddRef())                                     {return 1;}\r
 276         STDMETHOD_(ULONG, Release())                            {return 1;}\r
 277         \r
 278         STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))\r
 279         {\r
 280                 if(!is_running_)\r
 281                         return E_FAIL;\r
 282                 \r
 283                 try\r
 284                 {\r
 285                         if(result == bmdOutputFrameDisplayedLate)\r
 286                                 graph_->add_tag("late-frame");\r
 287                         else if(result == bmdOutputFrameDropped)\r
 288                                 graph_->add_tag("dropped-frame");\r
 289                         else if(result == bmdOutputFrameFlushed)\r
 290                                 graph_->add_tag("flushed-frame");\r
 291 \r
 292                         frame_container_.erase(std::remove_if(frame_container_.begin(), frame_container_.end(), [&](const decklink_frame_adapter& frame)\r
 293                         {\r
 294                                 return &frame == completed_frame;\r
 295                         }), frame_container_.end());\r
 296 \r
 297                         std::shared_ptr<const core::read_frame> frame;  \r
 298                         video_frame_buffer_.pop(frame);         \r
 299                         schedule_next_video(safe_ptr<const core::read_frame>(frame));                   \r
 300                 }\r
 301                 catch(...)\r
 302                 {\r
 303                         exception_ = std::current_exception();\r
 304                         return E_FAIL;\r
 305                 }\r
 306 \r
 307                 return S_OK;\r
 308         }\r
 309 \r
 310         STDMETHOD(ScheduledPlaybackHasStopped())\r
 311         {\r
 312                 is_running_ = false;\r
 313                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";\r
 314                 return S_OK;\r
 315         }\r
 316                 \r
 317         STDMETHOD(RenderAudioSamples(BOOL preroll))\r
 318         {\r
 319                 if(!is_running_)\r
 320                         return E_FAIL;\r
 321                 \r
 322                 try\r
 323                 {\r
 324                         if(preroll)\r
 325                         {\r
 326                                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0)))\r
 327                                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Failed to schedule playback."));\r
 328                         }\r
 329 \r
 330                         std::shared_ptr<const core::read_frame> frame;\r
 331                         audio_frame_buffer_.pop(frame);\r
 332                         schedule_next_audio(safe_ptr<const core::read_frame>(frame));           \r
 333                 }\r
 334                 catch(...)\r
 335                 {\r
 336                         exception_ = std::current_exception();\r
 337                         return E_FAIL;\r
 338                 }\r
 339 \r
 340                 return S_OK;\r
 341         }\r
 342 \r
 343         void schedule_next_audio(const safe_ptr<const core::read_frame>& frame)\r
 344         {\r
 345                 static std::vector<short> silence(48000, 0);\r
 346 \r
 347                 int audio_samples = static_cast<size_t>(48000.0 / format_desc_.fps)*2; // Audio samples per channel\r
 348 \r
 349                 const short* frame_audio_data = frame->audio_data().size() == audio_samples ? frame->audio_data().begin() : silence.data();\r
 350                 audio_container_.push_back(std::vector<short>(frame_audio_data, frame_audio_data+audio_samples));\r
 351 \r
 352                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), audio_samples/2, (audio_scheduled_++) * audio_samples/2, 48000, nullptr)))\r
 353                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";\r
 354         }\r
 355                         \r
 356         void schedule_next_video(const safe_ptr<const core::read_frame>& frame)\r
 357         {\r
 358                 frame_container_.push_back(decklink_frame_adapter(frame, format_desc_));\r
 359                 if(FAILED(output_->ScheduleVideoFrame(&frame_container_.back(), (frames_scheduled_++) * format_desc_.duration, format_desc_.duration, format_desc_.time_scale)))\r
 360                         CASPAR_LOG(error) << print() << L" Failed to schedule video.";\r
 361 \r
 362                 graph_->update_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);\r
 363                 tick_timer_.restart();\r
 364         }\r
 365 \r
 366         void send(const safe_ptr<const core::read_frame>& frame)\r
 367         {\r
 368                 if(exception_ != nullptr)\r
 369                         std::rethrow_exception(exception_);\r
 370 \r
 371                 if(!is_running_)\r
 372                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Is not running."));\r
 373                 \r
 374                 if(config_.embedded_audio)\r
 375                         audio_frame_buffer_.push(frame);        \r
 376                 video_frame_buffer_.push(frame);        \r
 377         }\r
 378         \r
 379         std::wstring print() const\r
 380         {\r
 381                 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(config_.device_index) + L"]";\r
 382         }\r
 383 };\r
 384 \r
 385 struct decklink_consumer_proxy : public core::frame_consumer\r
 386 {\r
 387         const configuration config_;\r
 388 \r
 389         com_context<decklink_consumer> context_;\r
 390 public:\r
 391 \r
 392         decklink_consumer_proxy(const configuration& config)\r
 393                 : config_(config)\r
 394                 , context_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]"){}\r
 395         \r
 396         virtual void initialize(const core::video_format_desc& format_desc)\r
 397         {\r
 398                 context_.reset([&]{return new decklink_consumer(config_, format_desc);});\r
 399         }\r
 400         \r
 401         virtual void send(const safe_ptr<const core::read_frame>& frame)\r
 402         {\r
 403                 context_->send(frame);\r
 404         }\r
 405         \r
 406         virtual std::wstring print() const\r
 407         {\r
 408                 return context_->print();\r
 409         }\r
 410 };      \r
 411 \r
 412 safe_ptr<core::frame_consumer> create_decklink_consumer(const std::vector<std::wstring>& params) \r
 413 {\r
 414         if(params.size() < 1 || params[0] != L"DECKLINK")\r
 415                 return core::frame_consumer::empty();\r
 416         \r
 417         configuration config;\r
 418                 \r
 419         if(params.size() > 1)\r
 420                 config.device_index = lexical_cast_or_default<int>(params[1], config.device_index);\r
 421                 \r
 422         if(std::find(params.begin(), params.end(), L"INTERNAL_KEY") != params.end())\r
 423                 config.keyer = internal_key;\r
 424         else if(std::find(params.begin(), params.end(), L"EXTERNAL_KEY") != params.end())\r
 425                 config.keyer = external_key;\r
 426         \r
 427         if(std::find(params.begin(), params.end(), L"LOW_LATENCY") != params.end())\r
 428                 config.latency = low_latency;\r
 429         else if(std::find(params.begin(), params.end(), L"NORMAL_LATENCY") != params.end())\r
 430                 config.latency = normal_latency;\r
 431                 \r
 432         config.embedded_audio = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();\r
 433 \r
 434         return make_safe<decklink_consumer_proxy>(config);\r
 435 }\r
 436 \r
 437 safe_ptr<core::frame_consumer> create_decklink_consumer(const boost::property_tree::ptree& ptree) \r
 438 {\r
 439         configuration config;\r
 440 \r
 441         auto key_str = ptree.get("key", "default");\r
 442         if(key_str == "internal")\r
 443                 config.keyer = internal_key;\r
 444         else if(key_str == "external")\r
 445                 config.keyer = external_key;\r
 446 \r
 447         auto latency_str = ptree.get("latency", "default");\r
 448         if(latency_str == "normal")\r
 449                 config.latency = normal_latency;\r
 450         else if(latency_str == "low")\r
 451                 config.latency = low_latency;\r
 452 \r
 453         config.device_index = ptree.get("device", 0);\r
 454         config.embedded_audio  = ptree.get("embedded-audio", false);\r
 455 \r
 456         return make_safe<decklink_consumer_proxy>(config);\r
 457 }\r
 458 \r
 459 }\r
 460 \r
 461 /*\r
 462 ##############################################################################\r
 463 Pre-rolling\r
 464 \r
 465 Mail: 2011-05-09\r
 466 \r
 467 Yoshan\r
 468 BMD Developer Support\r
 469 developer@blackmagic-design.com\r
 470 \r
 471 -----------------------------------------------------------------------------\r
 472 \r
 473 Thanks for your inquiry. The minimum number of frames that you can preroll \r
 474 for scheduled playback is three frames for video and four frames for audio. \r
 475 As you mentioned if you preroll less frames then playback will not start or\r
 476 playback will be very sporadic. From our experience with Media Express, we \r
 477 recommended that at least seven frames are prerolled for smooth playback. \r
 478 \r
 479 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:\r
 480 There can be around 3 frames worth of latency on scheduled output.\r
 481 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is\r
 482 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync() \r
 483 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will \r
 484 guarantee that the provided frame will be output as soon the previous \r
 485 frame output has been completed.\r
 486 ################################################################################\r
 487 */\r
 488 \r
 489 /*\r
 490 ##############################################################################\r
 491 Async DMA Transfer without redundant copying\r
 492 \r
 493 Mail: 2011-05-10\r
 494 \r
 495 Yoshan\r
 496 BMD Developer Support\r
 497 developer@blackmagic-design.com\r
 498 \r
 499 -----------------------------------------------------------------------------\r
 500 \r
 501 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame \r
 502 and providing a pointer to your video buffer when GetBytes() is called. \r
 503 This may help to keep copying to a minimum. Please ensure that the pixel \r
 504 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will \r
 505 have to colourspace convert which may result in additional copying.\r
 506 ################################################################################\r
 507 */