]> git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp
2.0. Optimized key-only.
[casparcg] / modules / decklink / consumer / decklink_consumer.cpp
1 /*\r
2 * copyright (c) 2010 Sveriges Television AB <info@casparcg.com>\r
3 *\r
4 *  This file is part of CasparCG.\r
5 *\r
6 *    CasparCG is free software: you can redistribute it and/or modify\r
7 *    it under the terms of the GNU General Public License as published by\r
8 *    the Free Software Foundation, either version 3 of the License, or\r
9 *    (at your option) any later version.\r
10 *\r
11 *    CasparCG is distributed in the hope that it will be useful,\r
12 *    but WITHOUT ANY WARRANTY; without even the implied warranty of\r
13 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\r
14 *    GNU General Public License for more details.\r
15 \r
16 *    You should have received a copy of the GNU General Public License\r
17 *    along with CasparCG.  If not, see <http://www.gnu.org/licenses/>.\r
18 *\r
19 */\r
20 \r
21 #include "../StdAfx.h"\r
22  \r
23 #include "decklink_consumer.h"\r
24 \r
25 #include "../util/util.h"\r
26 \r
27 #include "../interop/DeckLinkAPI_h.h"\r
28 \r
29 #include <core/mixer/read_frame.h>\r
30 \r
31 #include <common/concurrency/com_context.h>\r
32 #include <common/diagnostics/graph.h>\r
33 #include <common/exception/exceptions.h>\r
34 #include <common/memory/memcpy.h>\r
35 #include <common/memory/memclr.h>\r
36 #include <common/memory/memshfl.h>\r
37 \r
38 #include <core/consumer/frame_consumer.h>\r
39 \r
40 #include <tbb/concurrent_queue.h>\r
41 #include <tbb/cache_aligned_allocator.h>\r
42 \r
43 #include <boost/circular_buffer.hpp>\r
44 #include <boost/timer.hpp>\r
45 \r
46 namespace caspar { \r
47         \r
48 struct configuration\r
49 {\r
50         size_t  device_index;\r
51         bool    embedded_audio;\r
52         bool    internal_key;\r
53         bool    low_latency;\r
54         bool    key_only;\r
55         size_t  buffer_depth;\r
56         \r
57         configuration()\r
58                 : device_index(1)\r
59                 , embedded_audio(false)\r
60                 , internal_key(false)\r
61                 , low_latency(false)\r
62                 , key_only(false)\r
63                 , buffer_depth(core::consumer_buffer_depth()){}\r
64 };\r
65 \r
66 class decklink_frame : public IDeckLinkVideoFrame\r
67 {\r
68         const std::shared_ptr<core::read_frame>                                         frame_;\r
69         const core::video_format_desc                                                           format_desc_;\r
70 \r
71         bool                                                                                                            key_only_;\r
72         std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>> key_data_;\r
73 public:\r
74         decklink_frame(const safe_ptr<core::read_frame>& frame, const core::video_format_desc& format_desc, bool key_only)\r
75                 : frame_(frame)\r
76                 , format_desc_(format_desc)\r
77                 , key_only_(key_only){}\r
78         \r
79         STDMETHOD (QueryInterface(REFIID, LPVOID*))             {return E_NOINTERFACE;}\r
80         STDMETHOD_(ULONG,                       AddRef())                       {return 1;}\r
81         STDMETHOD_(ULONG,                       Release())                      {return 1;}\r
82 \r
83         STDMETHOD_(long,                        GetWidth())                     {return format_desc_.width;}        \r
84     STDMETHOD_(long,                    GetHeight())            {return format_desc_.height;}        \r
85     STDMETHOD_(long,                    GetRowBytes())          {return format_desc_.width*4;}        \r
86         STDMETHOD_(BMDPixelFormat,      GetPixelFormat())       {return bmdFormat8BitBGRA;}        \r
87     STDMETHOD_(BMDFrameFlags,   GetFlags())                     {return bmdFrameFlagDefault;}\r
88         \r
89     STDMETHOD(GetBytes(void** buffer))\r
90         {\r
91                 static std::vector<uint8_t> zeros(1920*1080*4, 0);\r
92                 if(static_cast<size_t>(frame_->image_data().size()) != format_desc_.size)\r
93                 {\r
94                         *buffer = zeros.data();\r
95                         return S_OK;\r
96                 }\r
97 \r
98                 if(!key_only_)\r
99                         *buffer = const_cast<uint8_t*>(frame_->image_data().begin());\r
100                 else\r
101                 {\r
102                         if(key_data_.empty())\r
103                         {\r
104                                 key_data_.resize(frame_->image_data().size());\r
105                                 fast_memshfl(key_data_.data(), frame_->image_data().begin(), frame_->image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);\r
106                         }\r
107                         *buffer = key_data_.data();\r
108                 }\r
109 \r
110                 return S_OK;\r
111         }\r
112         \r
113     STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}        \r
114     STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary))                 {return S_FALSE;}\r
115 };\r
116 \r
117 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable\r
118 {               \r
119         const configuration                                     config_;\r
120 \r
121         CComPtr<IDeckLink>                                      decklink_;\r
122         CComQIPtr<IDeckLinkOutput>                      output_;\r
123         CComQIPtr<IDeckLinkConfiguration>       configuration_;\r
124         CComQIPtr<IDeckLinkKeyer>                       keyer_;\r
125 \r
126         tbb::spin_mutex                                         exception_mutex_;\r
127         std::exception_ptr                                      exception_;\r
128 \r
129         tbb::atomic<bool>                                       is_running_;\r
130                 \r
131         const std::wstring                                      model_name_;\r
132         const core::video_format_desc           format_desc_;\r
133         const size_t                                            buffer_size_;\r
134 \r
135         long long                                                       frames_scheduled_;\r
136         long long                                                       audio_scheduled_;\r
137 \r
138         size_t                                                          preroll_count_;\r
139                 \r
140         std::list<std::shared_ptr<IDeckLinkVideoFrame>> frame_container_; // Must be std::list in order to guarantee that pointers are always valid.\r
141         boost::circular_buffer<std::vector<int16_t>>    audio_container_;\r
142 \r
143         tbb::concurrent_bounded_queue<std::shared_ptr<core::read_frame>> video_frame_buffer_;\r
144         tbb::concurrent_bounded_queue<std::shared_ptr<core::read_frame>> audio_frame_buffer_;\r
145         \r
146         std::shared_ptr<diagnostics::graph> graph_;\r
147         boost::timer tick_timer_;\r
148 \r
149 public:\r
150         decklink_consumer(const configuration& config, const core::video_format_desc& format_desc) \r
151                 : config_(config)\r
152                 , decklink_(get_device(config.device_index))\r
153                 , output_(decklink_)\r
154                 , configuration_(decklink_)\r
155                 , keyer_(decklink_)\r
156                 , model_name_(get_model_name(decklink_))\r
157                 , format_desc_(format_desc)\r
158                 , buffer_size_(config.embedded_audio ? config.buffer_depth + 1 : config.buffer_depth) // Minimum buffer-size 3.\r
159                 , frames_scheduled_(0)\r
160                 , audio_scheduled_(0)\r
161                 , preroll_count_(0)\r
162                 , audio_container_(buffer_size_+1)\r
163         {\r
164                 is_running_ = true;\r
165                                 \r
166                 video_frame_buffer_.set_capacity(1);\r
167                 audio_frame_buffer_.set_capacity(1);\r
168 \r
169                 graph_ = diagnostics::create_graph(narrow(print()));\r
170                 graph_->add_guide("tick-time", 0.5);\r
171                 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));   \r
172                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));\r
173                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));\r
174                 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));\r
175                 \r
176                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));\r
177                                 \r
178                 if(config.embedded_audio)\r
179                         enable_audio();\r
180 \r
181                 set_latency(config.low_latency);                                \r
182                 set_keyer(config.internal_key);\r
183                                 \r
184                 if(config.embedded_audio)               \r
185                         output_->BeginAudioPreroll();           \r
186                 \r
187                 for(size_t n = 0; n < buffer_size_; ++n)\r
188                         schedule_next_video(make_safe<core::read_frame>());\r
189 \r
190                 if(!config.embedded_audio)\r
191                         start_playback();\r
192         }\r
193 \r
194         ~decklink_consumer()\r
195         {               \r
196                 is_running_ = false;\r
197                 video_frame_buffer_.try_push(std::make_shared<core::read_frame>());\r
198                 audio_frame_buffer_.try_push(std::make_shared<core::read_frame>());\r
199 \r
200                 if(output_ != nullptr) \r
201                 {\r
202                         output_->StopScheduledPlayback(0, nullptr, 0);\r
203                         if(config_.embedded_audio)\r
204                                 output_->DisableAudioOutput();\r
205                         output_->DisableVideoOutput();\r
206                 }\r
207         }\r
208                         \r
209         const core::video_format_desc& get_video_format_desc() const\r
210         {\r
211                 return format_desc_;\r
212         }\r
213 \r
214         void set_latency(bool low_latency)\r
215         {               \r
216                 if(!low_latency)\r
217                 {\r
218                         configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);\r
219                         CASPAR_LOG(info) << print() << L" Enabled normal-latency mode";\r
220                 }\r
221                 else\r
222                 {                       \r
223                         configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);\r
224                         CASPAR_LOG(info) << print() << L" Enabled low-latency mode";\r
225                 }\r
226         }\r
227 \r
228         void set_keyer(bool internal_key)\r
229         {\r
230                 if(internal_key) \r
231                 {\r
232                         if(FAILED(keyer_->Enable(FALSE)))                       \r
233                                 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";                   \r
234                         else if(FAILED(keyer_->SetLevel(255)))                  \r
235                                 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";\r
236                         else\r
237                                 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";             \r
238                 }\r
239                 else\r
240                 {\r
241                         if(FAILED(keyer_->Enable(TRUE)))                        \r
242                                 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";   \r
243                         else if(FAILED(keyer_->SetLevel(255)))                  \r
244                                 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";\r
245                         else\r
246                                 CASPAR_LOG(info) << print() << L" Enabled external keyer.";                     \r
247                 }\r
248         }\r
249         \r
250         void enable_audio()\r
251         {\r
252                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType16bitInteger, 2, bmdAudioOutputStreamTimestamped)))\r
253                                 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not enable audio output."));\r
254                                 \r
255                 if(FAILED(output_->SetAudioCallback(this)))\r
256                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not set audio callback."));\r
257 \r
258                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";\r
259         }\r
260 \r
261         void enable_video(BMDDisplayMode display_mode)\r
262         {\r
263                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault))) \r
264                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not enable video output."));\r
265                 \r
266                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))\r
267                         BOOST_THROW_EXCEPTION(caspar_exception() \r
268                                                                         << msg_info(narrow(print()) + " Failed to set playback completion callback.")\r
269                                                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));\r
270         }\r
271 \r
272         void start_playback()\r
273         {\r
274                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0))) \r
275                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Failed to schedule playback."));\r
276         }\r
277         \r
278         STDMETHOD (QueryInterface(REFIID, LPVOID*))     {return E_NOINTERFACE;}\r
279         STDMETHOD_(ULONG, AddRef())                                     {return 1;}\r
280         STDMETHOD_(ULONG, Release())                            {return 1;}\r
281         \r
282         STDMETHOD(ScheduledPlaybackHasStopped())\r
283         {\r
284                 is_running_ = false;\r
285                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";\r
286                 return S_OK;\r
287         }\r
288 \r
289         STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))\r
290         {\r
291                 if(!is_running_)\r
292                         return E_FAIL;\r
293                 \r
294                 try\r
295                 {\r
296                         if(result == bmdOutputFrameDisplayedLate)\r
297                         {\r
298                                 graph_->add_tag("late-frame");\r
299                                 ++frames_scheduled_;\r
300                                 ++audio_scheduled_;\r
301                         }\r
302                         else if(result == bmdOutputFrameDropped)\r
303                                 graph_->add_tag("dropped-frame");\r
304                         else if(result == bmdOutputFrameFlushed)\r
305                                 graph_->add_tag("flushed-frame");\r
306 \r
307                         frame_container_.erase(std::find_if(frame_container_.begin(), frame_container_.end(), [&](const std::shared_ptr<IDeckLinkVideoFrame>& frame)\r
308                         {\r
309                                 return frame.get() == completed_frame;\r
310                         }));\r
311 \r
312                         std::shared_ptr<core::read_frame> frame;        \r
313                         video_frame_buffer_.pop(frame);                                 \r
314                         schedule_next_video(make_safe(frame));  \r
315                 }\r
316                 catch(...)\r
317                 {\r
318                         tbb::spin_mutex::scoped_lock lock(exception_mutex_);\r
319                         exception_ = std::current_exception();\r
320                         return E_FAIL;\r
321                 }\r
322 \r
323                 return S_OK;\r
324         }\r
325                 \r
326         STDMETHOD(RenderAudioSamples(BOOL preroll))\r
327         {\r
328                 if(!is_running_)\r
329                         return E_FAIL;\r
330                 \r
331                 try\r
332                 {       \r
333                         if(preroll)\r
334                         {\r
335                                 if(++preroll_count_ >= buffer_size_)\r
336                                 {\r
337                                         output_->EndAudioPreroll();\r
338                                         start_playback();                               \r
339                                 }\r
340                                 else\r
341                                         schedule_next_audio(make_safe<core::read_frame>());     \r
342                         }\r
343                         else\r
344                         {\r
345                                 std::shared_ptr<core::read_frame> frame;\r
346                                 audio_frame_buffer_.pop(frame);\r
347                                 schedule_next_audio(make_safe(frame));  \r
348                         }\r
349                 }\r
350                 catch(...)\r
351                 {\r
352                         tbb::spin_mutex::scoped_lock lock(exception_mutex_);\r
353                         exception_ = std::current_exception();\r
354                         return E_FAIL;\r
355                 }\r
356 \r
357                 return S_OK;\r
358         }\r
359 \r
360         void schedule_next_audio(const safe_ptr<core::read_frame>& frame)\r
361         {\r
362                 const int sample_frame_count = frame->audio_data().size()/format_desc_.audio_channels;\r
363 \r
364                 audio_container_.push_back(std::vector<int16_t>(frame->audio_data().begin(), frame->audio_data().end()));\r
365 \r
366                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, (audio_scheduled_++) * sample_frame_count, format_desc_.audio_sample_rate, nullptr)))\r
367                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";\r
368         }\r
369                         \r
370         void schedule_next_video(const safe_ptr<core::read_frame>& frame)\r
371         {\r
372                 frame_container_.push_back(std::make_shared<decklink_frame>(frame, format_desc_, config_.key_only));\r
373                 if(FAILED(output_->ScheduleVideoFrame(frame_container_.back().get(), (frames_scheduled_++) * format_desc_.duration, format_desc_.duration, format_desc_.time_scale)))\r
374                         CASPAR_LOG(error) << print() << L" Failed to schedule video.";\r
375 \r
376                 graph_->update_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);\r
377                 tick_timer_.restart();\r
378         }\r
379 \r
380         void send(const safe_ptr<core::read_frame>& frame)\r
381         {\r
382                 {\r
383                         tbb::spin_mutex::scoped_lock lock(exception_mutex_);\r
384                         if(exception_ != nullptr)\r
385                                 std::rethrow_exception(exception_);\r
386                 }\r
387 \r
388                 if(!is_running_)\r
389                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Is not running."));\r
390                 \r
391                 if(config_.embedded_audio)\r
392                         audio_frame_buffer_.push(frame);        \r
393                 video_frame_buffer_.push(frame);        \r
394         }\r
395         \r
396         std::wstring print() const\r
397         {\r
398                 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(config_.device_index) + L"|" +  format_desc_.name + L"]";\r
399         }\r
400 };\r
401 \r
402 struct decklink_consumer_proxy : public core::frame_consumer\r
403 {\r
404         const configuration                             config_;\r
405         com_context<decklink_consumer>  context_;\r
406         core::video_format_desc                 format_desc_;\r
407         size_t                                                  fail_count_;\r
408 public:\r
409 \r
410         decklink_consumer_proxy(const configuration& config)\r
411                 : config_(config)\r
412                 , context_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]")\r
413                 , fail_count_(0)\r
414         {\r
415         }\r
416 \r
417         ~decklink_consumer_proxy()\r
418         {\r
419                 auto str = print();\r
420                 context_.reset();\r
421                 CASPAR_LOG(info) << str << L" Successfully Uninitialized.";     \r
422         }\r
423         \r
424         virtual void initialize(const core::video_format_desc& format_desc)\r
425         {\r
426                 format_desc_ = format_desc;\r
427                 context_.reset([&]{return new decklink_consumer(config_, format_desc_);});              \r
428                                 \r
429                 CASPAR_LOG(info) << print() << L" Successfully Initialized.";   \r
430         }\r
431         \r
432         virtual bool send(const safe_ptr<core::read_frame>& frame)\r
433         {\r
434                 if(!context_)\r
435                         context_.reset([&]{return new decklink_consumer(config_, format_desc_);});\r
436 \r
437                 try\r
438                 {\r
439                         context_->send(frame);\r
440                         fail_count_ = 0;\r
441                 }\r
442                 catch(...)\r
443                 {\r
444                         context_.reset();\r
445 \r
446                         if(fail_count_++ > 3)\r
447                                 return false;  // Outside didn't handle exception properly, just give up.\r
448                         \r
449                         throw;\r
450                 }\r
451 \r
452                 return true;\r
453         }\r
454         \r
455         virtual std::wstring print() const\r
456         {\r
457                 return context_ ? context_->print() : L"decklink_consumer";\r
458         }\r
459 \r
460         virtual bool key_only() const\r
461         {\r
462                 return config_.key_only;\r
463         }\r
464                 \r
465         virtual const core::video_format_desc& get_video_format_desc() const\r
466         {\r
467                 return format_desc_;\r
468         }\r
469 };      \r
470 \r
471 safe_ptr<core::frame_consumer> create_decklink_consumer(const std::vector<std::wstring>& params) \r
472 {\r
473         if(params.size() < 1 || params[0] != L"DECKLINK")\r
474                 return core::frame_consumer::empty();\r
475         \r
476         configuration config;\r
477                 \r
478         if(params.size() > 1)\r
479                 config.device_index = lexical_cast_or_default<int>(params[1], config.device_index);\r
480         \r
481         config.internal_key             = std::find(params.begin(), params.end(), L"INTERNAL_KEY")       != params.end();\r
482         config.low_latency              = std::find(params.begin(), params.end(), L"LOW_LATENCY")        != params.end();\r
483         config.embedded_audio   = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();\r
484         config.key_only                 = std::find(params.begin(), params.end(), L"KEY_ONLY")           != params.end();\r
485 \r
486         return make_safe<decklink_consumer_proxy>(config);\r
487 }\r
488 \r
489 safe_ptr<core::frame_consumer> create_decklink_consumer(const boost::property_tree::ptree& ptree) \r
490 {\r
491         configuration config;\r
492 \r
493         config.internal_key             = ptree.get("internal-key",       config.internal_key);\r
494         config.low_latency              = ptree.get("low-latency",        config.low_latency);\r
495         config.key_only                 = ptree.get("key-only",           config.key_only);\r
496         config.device_index             = ptree.get("device",             config.device_index);\r
497         config.embedded_audio   = ptree.get("embedded-audio", config.embedded_audio);\r
498 \r
499         return make_safe<decklink_consumer_proxy>(config);\r
500 }\r
501 \r
502 }\r
503 \r
504 /*\r
505 ##############################################################################\r
506 Pre-rolling\r
507 \r
508 Mail: 2011-05-09\r
509 \r
510 Yoshan\r
511 BMD Developer Support\r
512 developer@blackmagic-design.com\r
513 \r
514 -----------------------------------------------------------------------------\r
515 \r
516 Thanks for your inquiry. The minimum number of frames that you can preroll \r
517 for scheduled playback is three frames for video and four frames for audio. \r
518 As you mentioned if you preroll less frames then playback will not start or\r
519 playback will be very sporadic. From our experience with Media Express, we \r
520 recommended that at least seven frames are prerolled for smooth playback. \r
521 \r
522 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:\r
523 There can be around 3 frames worth of latency on scheduled output.\r
524 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is\r
525 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync() \r
526 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will \r
527 guarantee that the provided frame will be output as soon the previous \r
528 frame output has been completed.\r
529 ################################################################################\r
530 */\r
531 \r
532 /*\r
533 ##############################################################################\r
534 Async DMA Transfer without redundant copying\r
535 \r
536 Mail: 2011-05-10\r
537 \r
538 Yoshan\r
539 BMD Developer Support\r
540 developer@blackmagic-design.com\r
541 \r
542 -----------------------------------------------------------------------------\r
543 \r
544 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame \r
545 and providing a pointer to your video buffer when GetBytes() is called. \r
546 This may help to keep copying to a minimum. Please ensure that the pixel \r
547 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will \r
548 have to colourspace convert which may result in additional copying.\r
549 ################################################################################\r
550 */