]> git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp
concrt-exp: Improved diagnostic graphs API.
[casparcg] / modules / decklink / consumer / decklink_consumer.cpp
1 /*\r
2 * copyright (c) 2010 Sveriges Television AB <info@casparcg.com>\r
3 *\r
4 *  This file is part of CasparCG.\r
5 *\r
6 *    CasparCG is free software: you can redistribute it and/or modify\r
7 *    it under the terms of the GNU General Public License as published by\r
8 *    the Free Software Foundation, either version 3 of the License, or\r
9 *    (at your option) any later version.\r
10 *\r
11 *    CasparCG is distributed in the hope that it will be useful,\r
12 *    but WITHOUT ANY WARRANTY; without even the implied warranty of\r
13 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\r
14 *    GNU General Public License for more details.\r
15 \r
16 *    You should have received a copy of the GNU General Public License\r
17 *    along with CasparCG.  If not, see <http://www.gnu.org/licenses/>.\r
18 *\r
19 */\r
20 \r
21 #include "../StdAfx.h"\r
22  \r
23 #include "decklink_consumer.h"\r
24 \r
25 #include "../util/util.h"\r
26 \r
27 #include "../interop/DeckLinkAPI_h.h"\r
28 \r
29 #include <core/mixer/read_frame.h>\r
30 \r
31 #include <common/diagnostics/graph.h>\r
32 #include <common/exception/exceptions.h>\r
33 #include <common/memory/memcpy.h>\r
34 #include <common/memory/memclr.h>\r
35 #include <common/memory/memshfl.h>\r
36 \r
37 #include <core/consumer/frame_consumer.h>\r
38 \r
39 #include <tbb/concurrent_queue.h>\r
40 #include <tbb/cache_aligned_allocator.h>\r
41 \r
42 #include <boost/circular_buffer.hpp>\r
43 #include <boost/timer.hpp>\r
44 \r
45 #include <concrt_extras.h>\r
46 \r
47 namespace caspar { namespace decklink { \r
48         \r
49 struct configuration\r
50 {\r
51         size_t  device_index;\r
52         bool    embedded_audio;\r
53         bool    internal_key;\r
54         bool    low_latency;\r
55         bool    key_only;\r
56         size_t  buffer_depth;\r
57         \r
58         configuration()\r
59                 : device_index(1)\r
60                 , embedded_audio(false)\r
61                 , internal_key(false)\r
62                 , low_latency(false)\r
63                 , key_only(false)\r
64                 , buffer_depth(core::consumer_buffer_depth()){}\r
65 };\r
66 \r
67 class decklink_frame : public IDeckLinkVideoFrame\r
68 {\r
69         tbb::atomic<int>                                                                                        ref_count_;\r
70         std::shared_ptr<core::read_frame>                                                       frame_;\r
71         const core::video_format_desc                                                           format_desc_;\r
72 \r
73         bool                                                                                                            key_only_;\r
74         std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>> key_data_;\r
75 public:\r
76         decklink_frame(const safe_ptr<core::read_frame>& frame, const core::video_format_desc& format_desc, bool key_only)\r
77                 : frame_(frame)\r
78                 , format_desc_(format_desc)\r
79                 , key_only_(key_only)\r
80         {\r
81                 ref_count_ = 0;\r
82         }\r
83         \r
84         STDMETHOD (QueryInterface(REFIID, LPVOID*))             {return E_NOINTERFACE;}\r
85         STDMETHOD_(ULONG,                       AddRef())                       \r
86         {\r
87                 return ++ref_count_;\r
88         }\r
89         STDMETHOD_(ULONG,                       Release())                      \r
90         {\r
91                 --ref_count_;\r
92                 if(ref_count_ == 0)\r
93                         delete this;\r
94                 return ref_count_;\r
95         }\r
96 \r
97         STDMETHOD_(long,                        GetWidth())                     {return format_desc_.width;}        \r
98     STDMETHOD_(long,                    GetHeight())            {return format_desc_.height;}        \r
99     STDMETHOD_(long,                    GetRowBytes())          {return format_desc_.width*4;}        \r
100         STDMETHOD_(BMDPixelFormat,      GetPixelFormat())       {return bmdFormat8BitBGRA;}        \r
101     STDMETHOD_(BMDFrameFlags,   GetFlags())                     {return bmdFrameFlagDefault;}\r
102         \r
103     STDMETHOD(GetBytes(void** buffer))\r
104         {\r
105                 static std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>> zeros(1920*1080*4, 0);\r
106                 if(static_cast<size_t>(frame_->image_data().size()) != format_desc_.size)\r
107                 {\r
108                         *buffer = zeros.data();\r
109                         return S_OK;\r
110                 }\r
111 \r
112                 if(!key_only_)\r
113                         *buffer = const_cast<uint8_t*>(frame_->image_data().begin());\r
114                 else\r
115                 {\r
116                         if(key_data_.empty())\r
117                         {\r
118                                 key_data_.resize(frame_->image_data().size());\r
119                                 fast_memshfl(key_data_.data(), frame_->image_data().begin(), frame_->image_data().size(), 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);\r
120                                 frame_.reset();\r
121                         }\r
122                         *buffer = key_data_.data();\r
123                 }\r
124 \r
125                 return S_OK;\r
126         }\r
127         \r
128     STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}        \r
129     STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary))                 {return S_FALSE;}\r
130 };\r
131 \r
132 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable\r
133 {               \r
134         const configuration                                     config_;\r
135 \r
136         CComPtr<IDeckLink>                                      decklink_;\r
137         CComQIPtr<IDeckLinkOutput>                      output_;\r
138         CComQIPtr<IDeckLinkConfiguration>       configuration_;\r
139         CComQIPtr<IDeckLinkKeyer>                       keyer_;\r
140 \r
141         tbb::spin_mutex                                         exception_mutex_;\r
142         std::exception_ptr                                      exception_;\r
143 \r
144         tbb::atomic<bool>                                       is_running_;\r
145                 \r
146         const std::wstring                                      model_name_;\r
147         const core::video_format_desc           format_desc_;\r
148         const size_t                                            buffer_size_;\r
149 \r
150         long long                                                       frames_scheduled_;\r
151         long long                                                       audio_scheduled_;\r
152 \r
153         size_t                                                          preroll_count_;\r
154                 \r
155         boost::circular_buffer<std::vector<int32_t>>    audio_container_;\r
156 \r
157         tbb::concurrent_bounded_queue<std::shared_ptr<core::read_frame>> video_frame_buffer_;\r
158         tbb::concurrent_bounded_queue<std::shared_ptr<core::read_frame>> audio_frame_buffer_;\r
159         \r
160         safe_ptr<diagnostics::graph> graph_;\r
161         boost::timer tick_timer_;\r
162 \r
163 public:\r
164         decklink_consumer(const configuration& config, const core::video_format_desc& format_desc) \r
165                 : config_(config)\r
166                 , decklink_(get_device(config.device_index))\r
167                 , output_(decklink_)\r
168                 , configuration_(decklink_)\r
169                 , keyer_(decklink_)\r
170                 , model_name_(get_model_name(decklink_))\r
171                 , format_desc_(format_desc)\r
172                 , buffer_size_(config.embedded_audio ? config.buffer_depth + 1 : config.buffer_depth) // Minimum buffer-size 3.\r
173                 , frames_scheduled_(0)\r
174                 , audio_scheduled_(0)\r
175                 , preroll_count_(0)\r
176                 , audio_container_(buffer_size_+1)\r
177         {\r
178                 is_running_ = true;\r
179                                 \r
180                 video_frame_buffer_.set_capacity(1);\r
181                 audio_frame_buffer_.set_capacity(1);\r
182 \r
183                 graph_->add_guide("tick-time", 0.5);\r
184                 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));   \r
185                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));\r
186                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));\r
187                 graph_->set_color("flushed-frame", diagnostics::color(0.4f, 0.3f, 0.8f));\r
188                 graph_->set_text(print());\r
189                 diagnostics::register_graph(graph_);\r
190                 \r
191                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));\r
192                                 \r
193                 if(config.embedded_audio)\r
194                         enable_audio();\r
195 \r
196                 set_latency(config.low_latency);                                \r
197                 set_keyer(config.internal_key);\r
198                                 \r
199                 if(config.embedded_audio)               \r
200                         output_->BeginAudioPreroll();           \r
201                 \r
202                 for(size_t n = 0; n < buffer_size_; ++n)\r
203                         schedule_next_video(make_safe<core::read_frame>());\r
204 \r
205                 if(!config.embedded_audio)\r
206                         start_playback();\r
207         }\r
208 \r
209         ~decklink_consumer()\r
210         {               \r
211                 is_running_ = false;\r
212                 video_frame_buffer_.try_push(std::make_shared<core::read_frame>());\r
213                 audio_frame_buffer_.try_push(std::make_shared<core::read_frame>());\r
214 \r
215                 if(output_ != nullptr) \r
216                 {\r
217                         output_->StopScheduledPlayback(0, nullptr, 0);\r
218                         if(config_.embedded_audio)\r
219                                 output_->DisableAudioOutput();\r
220                         output_->DisableVideoOutput();\r
221                 }\r
222         }\r
223                         \r
224         const core::video_format_desc& get_video_format_desc() const\r
225         {\r
226                 return format_desc_;\r
227         }\r
228 \r
229         void set_latency(bool low_latency)\r
230         {               \r
231                 if(!low_latency)\r
232                 {\r
233                         configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);\r
234                         CASPAR_LOG(info) << print() << L" Enabled normal-latency mode";\r
235                 }\r
236                 else\r
237                 {                       \r
238                         configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);\r
239                         CASPAR_LOG(info) << print() << L" Enabled low-latency mode";\r
240                 }\r
241         }\r
242 \r
243         void set_keyer(bool internal_key)\r
244         {\r
245                 if(internal_key) \r
246                 {\r
247                         if(FAILED(keyer_->Enable(FALSE)))                       \r
248                                 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";                   \r
249                         else if(FAILED(keyer_->SetLevel(255)))                  \r
250                                 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";\r
251                         else\r
252                                 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";             \r
253                 }\r
254                 else\r
255                 {\r
256                         if(FAILED(keyer_->Enable(TRUE)))                        \r
257                                 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";   \r
258                         else if(FAILED(keyer_->SetLevel(255)))                  \r
259                                 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";\r
260                         else\r
261                                 CASPAR_LOG(info) << print() << L" Enabled external keyer.";                     \r
262                 }\r
263         }\r
264         \r
265         void enable_audio()\r
266         {\r
267                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped)))\r
268                                 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not enable audio output."));\r
269                                 \r
270                 if(FAILED(output_->SetAudioCallback(this)))\r
271                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not set audio callback."));\r
272 \r
273                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";\r
274         }\r
275 \r
276         void enable_video(BMDDisplayMode display_mode)\r
277         {\r
278                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault))) \r
279                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not enable video output."));\r
280                 \r
281                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))\r
282                         BOOST_THROW_EXCEPTION(caspar_exception() \r
283                                                                         << msg_info(narrow(print()) + " Failed to set playback completion callback.")\r
284                                                                         << boost::errinfo_api_function("SetScheduledFrameCompletionCallback"));\r
285         }\r
286 \r
287         void start_playback()\r
288         {\r
289                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0))) \r
290                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Failed to schedule playback."));\r
291         }\r
292         \r
293         STDMETHOD (QueryInterface(REFIID, LPVOID*))     {return E_NOINTERFACE;}\r
294         STDMETHOD_(ULONG, AddRef())                                     {return 1;}\r
295         STDMETHOD_(ULONG, Release())                            {return 1;}\r
296         \r
297         STDMETHOD(ScheduledPlaybackHasStopped())\r
298         {\r
299                 is_running_ = false;\r
300                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";\r
301                 return S_OK;\r
302         }\r
303 \r
304         STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))\r
305         {\r
306                 if(!is_running_)\r
307                         return E_FAIL;\r
308                 \r
309                 try\r
310                 {\r
311                         if(result == bmdOutputFrameDisplayedLate)\r
312                         {\r
313                                 graph_->add_tag("late-frame");\r
314                                 ++frames_scheduled_;\r
315                                 ++audio_scheduled_;\r
316                         }\r
317                         else if(result == bmdOutputFrameDropped)\r
318                                 graph_->add_tag("dropped-frame");\r
319                         else if(result == bmdOutputFrameFlushed)\r
320                                 graph_->add_tag("flushed-frame");\r
321 \r
322                         std::shared_ptr<core::read_frame> frame;        \r
323                         video_frame_buffer_.pop(frame);                                 \r
324                         schedule_next_video(make_safe_ptr(frame));      \r
325                 }\r
326                 catch(...)\r
327                 {\r
328                         tbb::spin_mutex::scoped_lock lock(exception_mutex_);\r
329                         exception_ = std::current_exception();\r
330                         return E_FAIL;\r
331                 }\r
332 \r
333                 return S_OK;\r
334         }\r
335                 \r
336         STDMETHOD(RenderAudioSamples(BOOL preroll))\r
337         {\r
338                 if(!is_running_)\r
339                         return E_FAIL;\r
340                 \r
341                 try\r
342                 {       \r
343                         if(preroll)\r
344                         {\r
345                                 if(++preroll_count_ >= buffer_size_)\r
346                                 {\r
347                                         output_->EndAudioPreroll();\r
348                                         start_playback();                               \r
349                                 }\r
350                                 else\r
351                                         schedule_next_audio(make_safe<core::read_frame>());     \r
352                         }\r
353                         else\r
354                         {\r
355                                 std::shared_ptr<core::read_frame> frame;\r
356                                 audio_frame_buffer_.pop(frame);\r
357                                 schedule_next_audio(make_safe_ptr(frame));      \r
358                         }\r
359                 }\r
360                 catch(...)\r
361                 {\r
362                         tbb::spin_mutex::scoped_lock lock(exception_mutex_);\r
363                         exception_ = std::current_exception();\r
364                         return E_FAIL;\r
365                 }\r
366 \r
367                 return S_OK;\r
368         }\r
369 \r
370         void schedule_next_audio(const safe_ptr<core::read_frame>& frame)\r
371         {\r
372                 const int sample_frame_count = frame->audio_data().size()/format_desc_.audio_channels;\r
373 \r
374                 audio_container_.push_back(std::vector<int32_t>(frame->audio_data().begin(), frame->audio_data().end()));\r
375 \r
376                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, (audio_scheduled_++) * sample_frame_count, format_desc_.audio_sample_rate, nullptr)))\r
377                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";\r
378         }\r
379                         \r
380         void schedule_next_video(const safe_ptr<core::read_frame>& frame)\r
381         {\r
382                 CComPtr<IDeckLinkVideoFrame> frame2(new decklink_frame(frame, format_desc_, config_.key_only));\r
383                 if(FAILED(output_->ScheduleVideoFrame(frame2, (frames_scheduled_++) * format_desc_.duration, format_desc_.duration, format_desc_.time_scale)))\r
384                         CASPAR_LOG(error) << print() << L" Failed to schedule video.";\r
385 \r
386                 graph_->update_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);\r
387                 tick_timer_.restart();\r
388         }\r
389 \r
390         void send(const safe_ptr<core::read_frame>& frame)\r
391         {\r
392                 {\r
393                         tbb::spin_mutex::scoped_lock lock(exception_mutex_);\r
394                         if(exception_ != nullptr)\r
395                                 std::rethrow_exception(exception_);\r
396                 }\r
397 \r
398                 if(!is_running_)\r
399                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Is not running."));\r
400                 \r
401                 if(config_.embedded_audio)\r
402                         audio_frame_buffer_.push(frame);        \r
403                 video_frame_buffer_.push(frame);        \r
404         }\r
405         \r
406         std::wstring print() const\r
407         {\r
408                 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(config_.device_index) + L"|" +  format_desc_.name + L"]";\r
409         }\r
410 };\r
411 \r
412 struct decklink_consumer_proxy : public core::frame_consumer\r
413 {\r
414         const configuration                                     config_;\r
415         std::unique_ptr<decklink_consumer>      context_;\r
416         core::video_format_desc                         format_desc_;\r
417 public:\r
418 \r
419         decklink_consumer_proxy(const configuration& config)\r
420                 : config_(config)\r
421         {\r
422         }\r
423 \r
424         ~decklink_consumer_proxy()\r
425         {\r
426                 auto str = print();\r
427                 struct co_init\r
428                 {\r
429                         co_init(){CoInitialize(nullptr);}\r
430                         ~co_init(){CoUninitialize();}\r
431                 } init;         \r
432                 context_.reset();\r
433                 CASPAR_LOG(info) << str << L" Successfully Uninitialized.";     \r
434         }\r
435         \r
436         virtual void initialize(const core::video_format_desc& format_desc)\r
437         {\r
438                 Concurrency::scoped_oversubcription_token oversubscribe;\r
439                 format_desc_ = format_desc;\r
440                 struct co_init\r
441                 {\r
442                         co_init(){CoInitialize(nullptr);}\r
443                         ~co_init(){CoUninitialize();}\r
444                 } init;         \r
445                 context_.reset(new decklink_consumer(config_, format_desc_));           \r
446                                 \r
447                 CASPAR_LOG(info) << print() << L" Successfully Initialized.";   \r
448         }\r
449         \r
450         virtual bool send(const safe_ptr<core::read_frame>& frame)\r
451         {\r
452                 Concurrency::scoped_oversubcription_token oversubscribe;\r
453                 context_->send(frame);\r
454                 return true;\r
455         }\r
456         \r
457         virtual std::wstring print() const\r
458         {\r
459                 return context_ ? context_->print() : L"decklink_consumer";\r
460         }\r
461                         \r
462         virtual const core::video_format_desc& get_video_format_desc() const\r
463         {\r
464                 return format_desc_;\r
465         }\r
466 };      \r
467 \r
468 safe_ptr<core::frame_consumer> create_consumer(const std::vector<std::wstring>& params) \r
469 {\r
470         if(params.size() < 1 || params[0] != L"DECKLINK")\r
471                 return core::frame_consumer::empty();\r
472         \r
473         configuration config;\r
474                 \r
475         if(params.size() > 1)\r
476                 config.device_index = lexical_cast_or_default<int>(params[1], config.device_index);\r
477         \r
478         config.internal_key             = std::find(params.begin(), params.end(), L"INTERNAL_KEY")       != params.end();\r
479         config.low_latency              = std::find(params.begin(), params.end(), L"LOW_LATENCY")        != params.end();\r
480         config.embedded_audio   = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();\r
481         config.key_only                 = std::find(params.begin(), params.end(), L"KEY_ONLY")           != params.end();\r
482 \r
483         return make_safe<decklink_consumer_proxy>(config);\r
484 }\r
485 \r
486 safe_ptr<core::frame_consumer> create_consumer(const boost::property_tree::ptree& ptree) \r
487 {\r
488         configuration config;\r
489 \r
490         config.internal_key             = ptree.get("internal-key",       config.internal_key);\r
491         config.low_latency              = ptree.get("low-latency",        config.low_latency);\r
492         config.key_only                 = ptree.get("key-only",           config.key_only);\r
493         config.device_index             = ptree.get("device",             config.device_index);\r
494         config.embedded_audio   = ptree.get("embedded-audio", config.embedded_audio);\r
495 \r
496         return make_safe<decklink_consumer_proxy>(config);\r
497 }\r
498 \r
499 }}\r
500 \r
501 /*\r
502 ##############################################################################\r
503 Pre-rolling\r
504 \r
505 Mail: 2011-05-09\r
506 \r
507 Yoshan\r
508 BMD Developer Support\r
509 developer@blackmagic-design.com\r
510 \r
511 -----------------------------------------------------------------------------\r
512 \r
513 Thanks for your inquiry. The minimum number of frames that you can preroll \r
514 for scheduled playback is three frames for video and four frames for audio. \r
515 As you mentioned if you preroll less frames then playback will not start or\r
516 playback will be very sporadic. From our experience with Media Express, we \r
517 recommended that at least seven frames are prerolled for smooth playback. \r
518 \r
519 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:\r
520 There can be around 3 frames worth of latency on scheduled output.\r
521 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is\r
522 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync() \r
523 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will \r
524 guarantee that the provided frame will be output as soon the previous \r
525 frame output has been completed.\r
526 ################################################################################\r
527 */\r
528 \r
529 /*\r
530 ##############################################################################\r
531 Async DMA Transfer without redundant copying\r
532 \r
533 Mail: 2011-05-10\r
534 \r
535 Yoshan\r
536 BMD Developer Support\r
537 developer@blackmagic-design.com\r
538 \r
539 -----------------------------------------------------------------------------\r
540 \r
541 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame \r
542 and providing a pointer to your video buffer when GetBytes() is called. \r
543 This may help to keep copying to a minimum. Please ensure that the pixel \r
544 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will \r
545 have to colourspace convert which may result in additional copying.\r
546 ################################################################################\r
547 */