]> git.sesse.net Git - casparcg/blob - modules/decklink/consumer/decklink_consumer.cpp
2.0.0.2: Refactored key_only implementation.
[casparcg] / modules / decklink / consumer / decklink_consumer.cpp
1 /*\r
2 * copyright (c) 2010 Sveriges Television AB <info@casparcg.com>\r
3 *\r
4 *  This file is part of CasparCG.\r
5 *\r
6 *    CasparCG is free software: you can redistribute it and/or modify\r
7 *    it under the terms of the GNU General Public License as published by\r
8 *    the Free Software Foundation, either version 3 of the License, or\r
9 *    (at your option) any later version.\r
10 *\r
11 *    CasparCG is distributed in the hope that it will be useful,\r
12 *    but WITHOUT ANY WARRANTY; without even the implied warranty of\r
13 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\r
14 *    GNU General Public License for more details.\r
15 \r
16 *    You should have received a copy of the GNU General Public License\r
17 *    along with CasparCG.  If not, see <http://www.gnu.org/licenses/>.\r
18 *\r
19 */\r
20 \r
21 #include "../StdAfx.h"\r
22  \r
23 #include "decklink_consumer.h"\r
24 \r
25 #include "../util/util.h"\r
26 \r
27 #include "../interop/DeckLinkAPI_h.h"\r
28 \r
29 #include <core/consumer/frame/read_frame.h>\r
30 \r
31 #include <common/concurrency/com_context.h>\r
32 #include <common/diagnostics/graph.h>\r
33 #include <common/exception/exceptions.h>\r
34 #include <common/memory/memcpy.h>\r
35 #include <common/memory/memclr.h>\r
36 #include <common/memory/memshfl.h>\r
37 \r
38 #include <tbb/concurrent_queue.h>\r
39 \r
40 #include <boost/circular_buffer.hpp>\r
41 #include <boost/timer.hpp>\r
42 \r
43 namespace caspar { \r
44         \r
45 enum key\r
46 {\r
47         external_key,\r
48         internal_key,\r
49         default_key\r
50 };\r
51 \r
52 enum latency\r
53 {\r
54         low_latency,\r
55         normal_latency,\r
56         default_latency\r
57 };\r
58 \r
59 enum output_pixels\r
60 {\r
61         fill_and_key,\r
62         key_only\r
63 };\r
64 \r
65 struct configuration\r
66 {\r
67         size_t  device_index;\r
68         bool    embedded_audio;\r
69         key             keyer;\r
70         latency latency;\r
71         output_pixels  output;\r
72         \r
73         configuration()\r
74                 : device_index(1)\r
75                 , embedded_audio(false)\r
76                 , keyer(default_key)\r
77                 , latency(default_latency)\r
78                 , output(fill_and_key){}\r
79 };\r
80 \r
81 class decklink_frame_adapter : public IDeckLinkVideoFrame\r
82 {\r
83         const safe_ptr<const core::read_frame>  frame_;\r
84         const core::video_format_desc                   format_desc_;\r
85 public:\r
86         decklink_frame_adapter(const safe_ptr<const core::read_frame>& frame, const core::video_format_desc& format_desc)\r
87                 : frame_(frame)\r
88                 , format_desc_(format_desc){}\r
89         \r
90         STDMETHOD (QueryInterface(REFIID, LPVOID*))     {return E_NOINTERFACE;}\r
91         STDMETHOD_(ULONG, AddRef())                                     {return 1;}\r
92         STDMETHOD_(ULONG, Release())                            {return 1;}\r
93 \r
94         STDMETHOD_(long, GetWidth())                            {return format_desc_.width;}        \r
95     STDMETHOD_(long, GetHeight())                               {return format_desc_.height;}        \r
96     STDMETHOD_(long, GetRowBytes())                             {return format_desc_.width*4;}        \r
97         STDMETHOD_(BMDPixelFormat, GetPixelFormat()){return bmdFormat8BitBGRA;}        \r
98     STDMETHOD_(BMDFrameFlags, GetFlags())               {return bmdFrameFlagDefault;}\r
99         \r
100     STDMETHOD(GetBytes(void** buffer))\r
101         {\r
102                 static std::vector<unsigned char> zeros(1920*1080*4, 0);\r
103                 *buffer = const_cast<unsigned char*>(frame_->image_data().begin());\r
104                 if(static_cast<size_t>(frame_->image_data().size()) != format_desc_.size)\r
105                         *buffer = zeros.data();\r
106                 return S_OK;\r
107         }\r
108         \r
109     STDMETHOD(GetTimecode(BMDTimecodeFormat format, IDeckLinkTimecode** timecode)){return S_FALSE;}        \r
110     STDMETHOD(GetAncillaryData(IDeckLinkVideoFrameAncillary** ancillary))                 {return S_FALSE;}\r
111 };\r
112 \r
113 std::shared_ptr<IDeckLinkVideoFrame> make_alpha_only_frame(const CComQIPtr<IDeckLinkOutput>& decklink, const safe_ptr<const core::read_frame>& frame, const core::video_format_desc& format_desc)\r
114 {\r
115         if(static_cast<size_t>(frame->image_data().size()) != format_desc.size)\r
116                 return std::make_shared<decklink_frame_adapter>(frame, format_desc);\r
117 \r
118         IDeckLinkMutableVideoFrame* result;\r
119 \r
120         if(FAILED(decklink->CreateVideoFrame(format_desc.width, format_desc.height, format_desc.size/format_desc.height, bmdFormat8BitBGRA, bmdFrameFlagDefault, &result)))\r
121                 BOOST_THROW_EXCEPTION(caspar_exception());\r
122 \r
123         void* bytes = nullptr;\r
124         if(FAILED(result->GetBytes(&bytes)))\r
125                 BOOST_THROW_EXCEPTION(caspar_exception());\r
126 \r
127         fast_memsfhl(reinterpret_cast<unsigned char*>(bytes), frame->image_data().begin(), frame->image_data().size(), 0x03030303, 0x07070707, 0x0B0B0B0B, 0x0F0F0F0F);\r
128 \r
129         return std::shared_ptr<IDeckLinkVideoFrame>(result, [](IDeckLinkMutableVideoFrame* p) {p->Release();});\r
130 }\r
131 \r
132 struct decklink_consumer : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback, boost::noncopyable\r
133 {               \r
134         const configuration config_;\r
135 \r
136         CComPtr<IDeckLink>                                      decklink_;\r
137         CComQIPtr<IDeckLinkOutput>                      output_;\r
138         CComQIPtr<IDeckLinkConfiguration>       configuration_;\r
139         CComQIPtr<IDeckLinkKeyer>                       keyer_;\r
140 \r
141         std::exception_ptr exception_;\r
142 \r
143         tbb::atomic<bool> is_running_;\r
144                 \r
145         const std::wstring model_name_;\r
146         const core::video_format_desc format_desc_;\r
147         const size_t buffer_size_;\r
148 \r
149         unsigned long frames_scheduled_;\r
150         unsigned long audio_scheduled_;\r
151                 \r
152         std::list<std::shared_ptr<IDeckLinkVideoFrame>> frame_container_; // Must be std::list in order to guarantee that pointers are always valid.\r
153         boost::circular_buffer<std::vector<short>> audio_container_;\r
154 \r
155         tbb::concurrent_bounded_queue<std::shared_ptr<const core::read_frame>> video_frame_buffer_;\r
156         tbb::concurrent_bounded_queue<std::shared_ptr<const core::read_frame>> audio_frame_buffer_;\r
157         \r
158         std::shared_ptr<diagnostics::graph> graph_;\r
159         boost::timer tick_timer_;\r
160 \r
161 public:\r
162         decklink_consumer(const configuration& config, const core::video_format_desc& format_desc) \r
163                 : config_(config)\r
164                 , decklink_(get_device(config.device_index))\r
165                 , output_(decklink_)\r
166                 , configuration_(decklink_)\r
167                 , keyer_(decklink_)\r
168                 , model_name_(get_model_name(decklink_))\r
169                 , format_desc_(format_desc)\r
170                 , buffer_size_(config.embedded_audio ? 5 : 4) // Minimum buffer-size (3 + 1 tolerance).\r
171                 , frames_scheduled_(0)\r
172                 , audio_scheduled_(0)\r
173                 , audio_container_(buffer_size_+1)\r
174         {\r
175                 is_running_ = true;\r
176                                 \r
177                 video_frame_buffer_.set_capacity(1);\r
178                 audio_frame_buffer_.set_capacity(1);\r
179 \r
180                 graph_ = diagnostics::create_graph(narrow(print()));\r
181                 graph_->add_guide("tick-time", 0.5);\r
182                 graph_->set_color("tick-time", diagnostics::color(0.1f, 0.7f, 0.8f));\r
183                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.3f));\r
184                 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));\r
185                 graph_->set_color("flushed-frame", diagnostics::color(0.3f, 0.3f, 0.6f));\r
186                 \r
187                 enable_video(get_display_mode(output_, format_desc_.format, bmdFormat8BitBGRA, bmdVideoOutputFlagDefault));\r
188                                 \r
189                 if(config.embedded_audio)\r
190                         enable_audio();\r
191 \r
192                 set_latency(config.latency);                            \r
193                 set_keyer(config.keyer);\r
194                                                                 \r
195                 for(size_t n = 0; n < buffer_size_; ++n)\r
196                         schedule_next_video(core::read_frame::empty());\r
197                 \r
198                 if(config.embedded_audio)\r
199                         output_->BeginAudioPreroll();\r
200                 else\r
201                         start_playback();\r
202                 \r
203                 CASPAR_LOG(info) << print() << L" Buffer depth: " << buffer_size_;              \r
204                 CASPAR_LOG(info) << print() << L" Successfully initialized for " << format_desc_.name;  \r
205         }\r
206 \r
207         ~decklink_consumer()\r
208         {               \r
209                 is_running_ = false;\r
210                 video_frame_buffer_.try_push(core::read_frame::empty());\r
211                 audio_frame_buffer_.try_push(core::read_frame::empty());\r
212 \r
213                 if(output_ != nullptr) \r
214                 {\r
215                         output_->StopScheduledPlayback(0, nullptr, 0);\r
216                         if(config_.embedded_audio)\r
217                                 output_->DisableAudioOutput();\r
218                         output_->DisableVideoOutput();\r
219                 }\r
220         }\r
221                         \r
222         void set_latency(latency latency)\r
223         {               \r
224                 if(latency == normal_latency)\r
225                 {\r
226                         configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, false);\r
227                         CASPAR_LOG(info) << print() << L" Enabled normal-latency mode";\r
228                 }\r
229                 else if(latency == low_latency)\r
230                 {                       \r
231                         configuration_->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true);\r
232                         CASPAR_LOG(info) << print() << L" Enabled low-latency mode";\r
233                 }\r
234                 else\r
235                         CASPAR_LOG(info) << print() << L" Uses driver latency settings.";       \r
236         }\r
237 \r
238         void set_keyer(key keyer)\r
239         {\r
240                 if(keyer == internal_key) \r
241                 {\r
242                         if(FAILED(keyer_->Enable(FALSE)))                       \r
243                                 CASPAR_LOG(error) << print() << L" Failed to enable internal keyer.";                   \r
244                         else if(FAILED(keyer_->SetLevel(255)))                  \r
245                                 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";\r
246                         else\r
247                                 CASPAR_LOG(info) << print() << L" Enabled internal keyer.";             \r
248                 }\r
249                 else if(keyer == external_key)\r
250                 {\r
251                         if(FAILED(keyer_->Enable(TRUE)))                        \r
252                                 CASPAR_LOG(error) << print() << L" Failed to enable external keyer.";   \r
253                         else if(FAILED(keyer_->SetLevel(255)))                  \r
254                                 CASPAR_LOG(error) << print() << L" Failed to set key-level to max.";\r
255                         else\r
256                                 CASPAR_LOG(info) << print() << L" Enabled external keyer.";                     \r
257                 }\r
258                 else\r
259                         CASPAR_LOG(info) << print() << L" Uses driver keyer settings."; \r
260         }\r
261         \r
262         void enable_audio()\r
263         {\r
264                 if(FAILED(output_->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType16bitInteger, 2, bmdAudioOutputStreamTimestamped)))\r
265                                 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not enable audio output."));\r
266                                 \r
267                 if(FAILED(output_->SetAudioCallback(this)))\r
268                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not set audio callback."));\r
269 \r
270                 CASPAR_LOG(info) << print() << L" Enabled embedded-audio.";\r
271         }\r
272 \r
273         void enable_video(BMDDisplayMode display_mode)\r
274         {\r
275                 if(FAILED(output_->EnableVideoOutput(display_mode, bmdVideoOutputFlagDefault))) \r
276                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Could not enable video output."));\r
277                 \r
278                 if(FAILED(output_->SetScheduledFrameCompletionCallback(this)))\r
279                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Failed to set playback completion callback."));\r
280         }\r
281 \r
282         void start_playback()\r
283         {\r
284                 if(FAILED(output_->StartScheduledPlayback(0, format_desc_.time_scale, 1.0))) \r
285                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Failed to schedule playback."));\r
286         }\r
287         \r
288         STDMETHOD (QueryInterface(REFIID, LPVOID*))     {return E_NOINTERFACE;}\r
289         STDMETHOD_(ULONG, AddRef())                                     {return 1;}\r
290         STDMETHOD_(ULONG, Release())                            {return 1;}\r
291         \r
292         STDMETHOD(ScheduledPlaybackHasStopped())\r
293         {\r
294                 is_running_ = false;\r
295                 CASPAR_LOG(info) << print() << L" Scheduled playback has stopped.";\r
296                 return S_OK;\r
297         }\r
298 \r
299         STDMETHOD(ScheduledFrameCompleted(IDeckLinkVideoFrame* completed_frame, BMDOutputFrameCompletionResult result))\r
300         {\r
301                 if(!is_running_)\r
302                         return E_FAIL;\r
303                 \r
304                 try\r
305                 {\r
306                         if(result == bmdOutputFrameDisplayedLate)\r
307                                 graph_->add_tag("late-frame");\r
308                         else if(result == bmdOutputFrameDropped)\r
309                                 graph_->add_tag("dropped-frame");\r
310                         else if(result == bmdOutputFrameFlushed)\r
311                                 graph_->add_tag("flushed-frame");\r
312 \r
313                         frame_container_.erase(std::find_if(frame_container_.begin(), frame_container_.end(), [&](const std::shared_ptr<IDeckLinkVideoFrame> frame)\r
314                         {\r
315                                 return frame.get() == completed_frame;\r
316                         }));\r
317 \r
318                         std::shared_ptr<const core::read_frame> frame;  \r
319                         video_frame_buffer_.pop(frame);         \r
320                         schedule_next_video(safe_ptr<const core::read_frame>(frame));                   \r
321                 }\r
322                 catch(...)\r
323                 {\r
324                         exception_ = std::current_exception();\r
325                         return E_FAIL;\r
326                 }\r
327 \r
328                 return S_OK;\r
329         }\r
330                 \r
331         STDMETHOD(RenderAudioSamples(BOOL preroll))\r
332         {\r
333                 if(!is_running_)\r
334                         return E_FAIL;\r
335                 \r
336                 try\r
337                 {\r
338                         std::shared_ptr<const core::read_frame> frame;\r
339                         audio_frame_buffer_.pop(frame);\r
340                         schedule_next_audio(safe_ptr<const core::read_frame>(frame));           \r
341 \r
342                         if(preroll)\r
343                                 start_playback();\r
344                 }\r
345                 catch(...)\r
346                 {\r
347                         exception_ = std::current_exception();\r
348                         return E_FAIL;\r
349                 }\r
350 \r
351                 return S_OK;\r
352         }\r
353 \r
354         void schedule_next_audio(const safe_ptr<const core::read_frame>& frame)\r
355         {\r
356                 static std::vector<short> silence(48000, 0);\r
357                 \r
358                 const int sample_count = format_desc_.audio_samples_per_frame;\r
359                 const int sample_frame_count = sample_count/2;\r
360 \r
361                 const short* frame_audio_data = frame->audio_data().size() == sample_count ? frame->audio_data().begin() : silence.data();\r
362                 audio_container_.push_back(std::vector<short>(frame_audio_data, frame_audio_data+sample_count));\r
363 \r
364                 if(FAILED(output_->ScheduleAudioSamples(audio_container_.back().data(), sample_frame_count, (audio_scheduled_++) * sample_frame_count, format_desc_.audio_sample_rate, nullptr)))\r
365                         CASPAR_LOG(error) << print() << L" Failed to schedule audio.";\r
366         }\r
367                         \r
368         void schedule_next_video(const safe_ptr<const core::read_frame>& frame)\r
369         {\r
370                 std::shared_ptr<IDeckLinkVideoFrame> deck_frame;\r
371                 if(config_.output == key_only)\r
372                         deck_frame = make_alpha_only_frame(output_, frame, format_desc_);\r
373                 else \r
374                         deck_frame = std::make_shared<decklink_frame_adapter>(frame, format_desc_);\r
375 \r
376                 frame_container_.push_back(deck_frame);\r
377                 if(FAILED(output_->ScheduleVideoFrame(frame_container_.back().get(), (frames_scheduled_++) * format_desc_.duration, format_desc_.duration, format_desc_.time_scale)))\r
378                         CASPAR_LOG(error) << print() << L" Failed to schedule video.";\r
379 \r
380                 graph_->update_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);\r
381                 tick_timer_.restart();\r
382         }\r
383 \r
384         void send(const safe_ptr<const core::read_frame>& frame)\r
385         {\r
386                 if(exception_ != nullptr)\r
387                         std::rethrow_exception(exception_);\r
388 \r
389                 if(!is_running_)\r
390                         BOOST_THROW_EXCEPTION(caspar_exception() << msg_info(narrow(print()) + " Is not running."));\r
391                 \r
392                 if(config_.embedded_audio)\r
393                         audio_frame_buffer_.push(frame);        \r
394                 video_frame_buffer_.push(frame);        \r
395         }\r
396         \r
397         std::wstring print() const\r
398         {\r
399                 return model_name_ + L" [" + boost::lexical_cast<std::wstring>(config_.device_index) + L"]";\r
400         }\r
401 };\r
402 \r
403 struct decklink_consumer_proxy : public core::frame_consumer\r
404 {\r
405         const configuration config_;\r
406 \r
407         com_context<decklink_consumer> context_;\r
408 public:\r
409 \r
410         decklink_consumer_proxy(const configuration& config)\r
411                 : config_(config)\r
412                 , context_(L"decklink_consumer[" + boost::lexical_cast<std::wstring>(config.device_index) + L"]"){}\r
413         \r
414         virtual void initialize(const core::video_format_desc& format_desc)\r
415         {\r
416                 context_.reset([&]{return new decklink_consumer(config_, format_desc);});\r
417         }\r
418         \r
419         virtual void send(const safe_ptr<const core::read_frame>& frame)\r
420         {\r
421                 context_->send(frame);\r
422         }\r
423         \r
424         virtual std::wstring print() const\r
425         {\r
426                 return context_->print();\r
427         }\r
428 };      \r
429 \r
430 safe_ptr<core::frame_consumer> create_decklink_consumer(const std::vector<std::wstring>& params) \r
431 {\r
432         if(params.size() < 1 || params[0] != L"DECKLINK")\r
433                 return core::frame_consumer::empty();\r
434         \r
435         configuration config;\r
436                 \r
437         if(params.size() > 1)\r
438                 config.device_index = lexical_cast_or_default<int>(params[1], config.device_index);\r
439                 \r
440         if(std::find(params.begin(), params.end(), L"INTERNAL_KEY") != params.end())\r
441                 config.keyer = internal_key;\r
442         else if(std::find(params.begin(), params.end(), L"EXTERNAL_KEY") != params.end())\r
443                 config.keyer = external_key;\r
444         \r
445         if(std::find(params.begin(), params.end(), L"LOW_LATENCY") != params.end())\r
446                 config.latency = low_latency;\r
447         else if(std::find(params.begin(), params.end(), L"NORMAL_LATENCY") != params.end())\r
448                 config.latency = normal_latency;\r
449                 \r
450         config.embedded_audio = std::find(params.begin(), params.end(), L"EMBEDDED_AUDIO") != params.end();\r
451 \r
452         return make_safe<decklink_consumer_proxy>(config);\r
453 }\r
454 \r
455 safe_ptr<core::frame_consumer> create_decklink_consumer(const boost::property_tree::ptree& ptree) \r
456 {\r
457         configuration config;\r
458 \r
459         auto key_str = ptree.get("key", "default");\r
460         if(key_str == "internal")\r
461                 config.keyer = internal_key;\r
462         else if(key_str == "external")\r
463                 config.keyer = external_key;\r
464 \r
465         auto latency_str = ptree.get("latency", "default");\r
466         if(latency_str == "normal")\r
467                 config.latency = normal_latency;\r
468         else if(latency_str == "low")\r
469                 config.latency = low_latency;\r
470 \r
471         auto output_str = ptree.get("output", "fill_and_key");\r
472         if(output_str == "key_only")\r
473                 config.output = key_only;\r
474 \r
475         config.device_index = ptree.get("device", 0);\r
476         config.embedded_audio  = ptree.get("embedded-audio", false);\r
477 \r
478         return make_safe<decklink_consumer_proxy>(config);\r
479 }\r
480 \r
481 }\r
482 \r
483 /*\r
484 ##############################################################################\r
485 Pre-rolling\r
486 \r
487 Mail: 2011-05-09\r
488 \r
489 Yoshan\r
490 BMD Developer Support\r
491 developer@blackmagic-design.com\r
492 \r
493 -----------------------------------------------------------------------------\r
494 \r
495 Thanks for your inquiry. The minimum number of frames that you can preroll \r
496 for scheduled playback is three frames for video and four frames for audio. \r
497 As you mentioned if you preroll less frames then playback will not start or\r
498 playback will be very sporadic. From our experience with Media Express, we \r
499 recommended that at least seven frames are prerolled for smooth playback. \r
500 \r
501 Regarding the bmdDeckLinkConfigLowLatencyVideoOutput flag:\r
502 There can be around 3 frames worth of latency on scheduled output.\r
503 When the bmdDeckLinkConfigLowLatencyVideoOutput flag is used this latency is\r
504 reduced  or removed for scheduled playback. If the DisplayVideoFrameSync() \r
505 method is used, the bmdDeckLinkConfigLowLatencyVideoOutput setting will \r
506 guarantee that the provided frame will be output as soon the previous \r
507 frame output has been completed.\r
508 ################################################################################\r
509 */\r
510 \r
511 /*\r
512 ##############################################################################\r
513 Async DMA Transfer without redundant copying\r
514 \r
515 Mail: 2011-05-10\r
516 \r
517 Yoshan\r
518 BMD Developer Support\r
519 developer@blackmagic-design.com\r
520 \r
521 -----------------------------------------------------------------------------\r
522 \r
523 Thanks for your inquiry. You could try subclassing IDeckLinkMutableVideoFrame \r
524 and providing a pointer to your video buffer when GetBytes() is called. \r
525 This may help to keep copying to a minimum. Please ensure that the pixel \r
526 format is in bmdFormat10BitYUV, otherwise the DeckLink API / driver will \r
527 have to colourspace convert which may result in additional copying.\r
528 ################################################################################\r
529 */