]> git.sesse.net Git - mlt/blob - src/modules/decklink/consumer_decklink.cpp
Log when decklink audio buffer level changes.
[mlt] / src / modules / decklink / consumer_decklink.cpp
1 /*
2  * consumer_decklink.c -- output through Blackmagic Design DeckLink
3  * Copyright (C) 2010 Dan Dennedy <dan@dennedy.org>
4  *
5  * This library is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU Lesser General Public
7  * License as published by the Free Software Foundation; either
8  * version 2.1 of the License, or (at your option) any later version.
9  *
10  * This library is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13  * Lesser General Public License for more details.
14  *
15  * You should have received a copy of the GNU Lesser General Public
16  * License along with consumer library; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
18  */
19
20 #define __STDC_FORMAT_MACROS  /* see inttypes.h */
21 #include <framework/mlt.h>
22 #include <stdlib.h>
23 #include <string.h>
24 #include <pthread.h>
25 #include <unistd.h>
26 #include <sys/time.h>
27 #include <limits.h>
28 #ifdef WIN32
29 #include <objbase.h>
30 #include "DeckLinkAPI_h.h"
31 #else
32 #include "DeckLinkAPI.h"
33 #endif
34
35 static const unsigned PREROLL_MINIMUM = 3;
36
37 class DeckLinkConsumer
38         : public IDeckLinkVideoOutputCallback
39 {
40 private:
41         mlt_consumer_s              m_consumer;
42         IDeckLink*                  m_deckLink;
43         IDeckLinkOutput*            m_deckLinkOutput;
44         IDeckLinkDisplayMode*       m_displayMode;
45         int                         m_width;
46         int                         m_height;
47         BMDTimeValue                m_duration;
48         BMDTimeScale                m_timescale;
49         double                      m_fps;
50         uint64_t                    m_count;
51         int                         m_channels;
52         unsigned                    m_dropped;
53         IDeckLinkMutableVideoFrame* m_decklinkFrame;
54         bool                        m_isAudio;
55         int                         m_isKeyer;
56         IDeckLinkKeyer*             m_deckLinkKeyer;
57         bool                        m_terminate_on_pause;
58         uint32_t                    m_preroll;
59         uint32_t                    m_acnt;
60
61         IDeckLinkDisplayMode* getDisplayMode()
62         {
63                 mlt_profile profile = mlt_service_profile( MLT_CONSUMER_SERVICE( getConsumer() ) );
64                 IDeckLinkDisplayModeIterator* iter;
65                 IDeckLinkDisplayMode* mode;
66                 IDeckLinkDisplayMode* result = 0;
67                 
68                 if ( m_deckLinkOutput->GetDisplayModeIterator( &iter ) == S_OK )
69                 {
70                         while ( !result && iter->Next( &mode ) == S_OK )
71                         {
72                                 m_width = mode->GetWidth();
73                                 m_height = mode->GetHeight();
74                                 mode->GetFrameRate( &m_duration, &m_timescale );
75                                 m_fps = (double) m_timescale / m_duration;
76                                 int p = mode->GetFieldDominance() == bmdProgressiveFrame;
77                                 mlt_log_verbose( getConsumer(), "BMD mode %dx%d %.3f fps prog %d\n", m_width, m_height, m_fps, p );
78                                 
79                                 if ( m_width == profile->width && m_height == profile->height && p == profile->progressive
80                                          && m_fps == mlt_profile_fps( profile ) )
81                                         result = mode;
82                         }
83                 }
84                 
85                 return result;
86         }
87         
88 public:
89         mlt_consumer getConsumer()
90                 { return &m_consumer; }
91         
92         ~DeckLinkConsumer()
93         {
94                 if ( m_deckLinkKeyer )
95                         m_deckLinkKeyer->Release();
96                 if ( m_deckLinkOutput )
97                         m_deckLinkOutput->Release();
98                 if ( m_deckLink )
99                         m_deckLink->Release();
100         }
101         
102         bool open( unsigned card = 0 )
103         {
104                 unsigned i = 0;
105 #ifdef WIN32
106                 IDeckLinkIterator* deckLinkIterator = NULL;
107                 HRESULT result =  CoInitialize( NULL );
108                 if ( FAILED( result ) )
109                 {
110                         mlt_log_error( getConsumer(), "COM initialization failed\n" );
111                         return false;
112                 }
113                 result = CoCreateInstance( CLSID_CDeckLinkIterator, NULL, CLSCTX_ALL, IID_IDeckLinkIterator, (void**) &deckLinkIterator );
114                 if ( FAILED( result ) )
115                 {
116                         mlt_log_error( getConsumer(), "The DeckLink drivers not installed.\n" );
117                         return false;
118                 }
119 #else
120                 IDeckLinkIterator* deckLinkIterator = CreateDeckLinkIteratorInstance();
121                 
122                 if ( !deckLinkIterator )
123                 {
124                         mlt_log_error( getConsumer(), "The DeckLink drivers not installed.\n" );
125                         return false;
126                 }
127 #endif
128                 
129                 // Connect to the Nth DeckLink instance
130                 do {
131                         if ( deckLinkIterator->Next( &m_deckLink ) != S_OK )
132                         {
133                                 mlt_log_error( getConsumer(), "DeckLink card not found\n" );
134                                 deckLinkIterator->Release();
135                                 return false;
136                         }
137                 } while ( ++i <= card );
138                 deckLinkIterator->Release();
139                 
140                 // Obtain the audio/video output interface (IDeckLinkOutput)
141                 if ( m_deckLink->QueryInterface( IID_IDeckLinkOutput, (void**)&m_deckLinkOutput ) != S_OK )
142                 {
143                         mlt_log_error( getConsumer(), "No DeckLink cards support output\n" );
144                         m_deckLink->Release();
145                         m_deckLink = 0;
146                         return false;
147                 }
148                 
149                 // Get the keyer interface
150                 IDeckLinkAttributes *deckLinkAttributes = 0;
151                 m_deckLinkKeyer = 0;
152                 if ( m_deckLink->QueryInterface( IID_IDeckLinkAttributes, (void**) &deckLinkAttributes ) == S_OK )
153                 {
154 #ifdef WIN32
155                         BOOL flag = FALSE;
156 #else
157                         bool flag = false;
158 #endif
159                         if ( deckLinkAttributes->GetFlag( BMDDeckLinkSupportsInternalKeying, &flag ) == S_OK && flag )
160                         {
161                                 if ( m_deckLink->QueryInterface( IID_IDeckLinkKeyer, (void**) &m_deckLinkKeyer ) != S_OK )
162                                 {
163                                         mlt_log_error( getConsumer(), "Failed to get keyer\n" );
164                                         m_deckLinkOutput->Release();
165                                         m_deckLinkOutput = 0;
166                                         m_deckLink->Release();
167                                         m_deckLink = 0;
168                                         return false;
169                                 }
170                         }
171                         deckLinkAttributes->Release();
172                 }
173
174                 // Provide this class as a delegate to the audio and video output interfaces
175                 m_deckLinkOutput->SetScheduledFrameCompletionCallback( this );
176                 
177                 return true;
178         }
179         
180         bool start( unsigned preroll )
181         {
182                 unsigned i;
183                 mlt_properties properties = MLT_CONSUMER_PROPERTIES( getConsumer() );
184
185                 // Initialize members
186                 m_count = 0;
187                 m_dropped = 0;
188                 m_decklinkFrame = NULL;
189                 preroll = preroll < PREROLL_MINIMUM ? PREROLL_MINIMUM : preroll;
190                 m_channels = mlt_properties_get_int( properties, "channels" );
191                 m_isAudio = !mlt_properties_get_int( properties, "audio_off" );
192                 m_terminate_on_pause = mlt_properties_get_int( properties, "terminate_on_pause" );
193
194
195                 m_displayMode = getDisplayMode();
196                 if ( !m_displayMode )
197                 {
198                         mlt_log_error( getConsumer(), "Profile is not compatible with decklink.\n" );
199                         return false;
200                 }
201                 
202                 // Set the keyer
203                 if ( m_deckLinkKeyer && ( m_isKeyer = mlt_properties_get_int( properties, "keyer" ) ) )
204                 {
205                         bool external = (m_isKeyer == 2);
206                         double level = mlt_properties_get_double( properties, "keyer_level" );
207
208                         if ( m_deckLinkKeyer->Enable( external ) != S_OK )
209                                 mlt_log_error( getConsumer(), "Failed to enable %s keyer\n",
210                                         external ? "external" : "internal" );
211                         m_deckLinkKeyer->SetLevel( level <= 1 ? ( level > 0 ? 255 * level : 255 ) : 255 );
212                 }
213                 else if ( m_deckLinkKeyer )
214                 {
215                         m_deckLinkKeyer->Disable();
216                 }
217
218                 // Set the video output mode
219                 if ( S_OK != m_deckLinkOutput->EnableVideoOutput( m_displayMode->GetDisplayMode(), bmdVideoOutputFlagDefault) )
220                 {
221                         mlt_log_error( getConsumer(), "Failed to enable video output\n" );
222                         return false;
223                 }
224
225                 // Set the audio output mode
226                 if ( !m_isAudio )
227                 {
228                         m_deckLinkOutput->DisableAudioOutput();
229                         return true;
230                 }
231                 if ( S_OK != m_deckLinkOutput->EnableAudioOutput( bmdAudioSampleRate48kHz, bmdAudioSampleType16bitInteger,
232                         m_channels, bmdAudioOutputStreamTimestamped ) )
233                 {
234                         mlt_log_error( getConsumer(), "Failed to enable audio output\n" );
235                         stop();
236                         return false;
237                 }
238
239                 m_preroll = preroll;
240
241                 // preroll frames
242                 for( i = 0; i < preroll; i++ )
243                         ScheduleNextFrame( true );
244
245                 // start scheduled playback
246                 m_deckLinkOutput->StartScheduledPlayback( 0, m_timescale, 1.0 );
247
248                 // Set the running state
249                 mlt_properties_set_int( properties, "running", 1 );
250
251                 return true;
252         }
253         
254         bool stop()
255         {
256                 mlt_properties properties = MLT_CONSUMER_PROPERTIES( getConsumer() );
257
258                 // set running state is 0
259                 mlt_properties_set_int( properties, "running", 0 );
260                 mlt_consumer_stopped( getConsumer() );
261
262                 // release decklink frame
263                 if ( m_decklinkFrame )
264                         m_decklinkFrame->Release();
265                 m_decklinkFrame = NULL;
266
267                 // Stop the audio and video output streams immediately
268                 if ( m_deckLinkOutput )
269                 {
270                         m_deckLinkOutput->StopScheduledPlayback( 0, 0, 0 );
271                         m_deckLinkOutput->DisableAudioOutput();
272                         m_deckLinkOutput->DisableVideoOutput();
273                 }
274
275                 return true;
276         }
277
278         void renderAudio( mlt_frame frame )
279         {
280                 mlt_audio_format format = mlt_audio_s16;
281                 int frequency = bmdAudioSampleRate48kHz;
282                 int samples = mlt_sample_calculator( m_fps, frequency, m_count );
283                 int16_t *pcm = 0;
284
285                 if ( !mlt_frame_get_audio( frame, (void**) &pcm, &format, &frequency, &m_channels, &samples ) )
286                 {
287                         uint32_t written = 0;
288                         BMDTimeValue streamTime = m_count * frequency * m_duration / m_timescale;
289                         m_deckLinkOutput->GetBufferedAudioSampleFrameCount(&written);
290                         if ( written > (m_preroll + 1) * samples )
291                         {
292                                 mlt_log_verbose( getConsumer(), "renderAudio: will flush %d audiosamples\n", written);
293                                 m_deckLinkOutput->FlushBufferedAudioSamples();
294                         };
295 #ifdef WIN32
296                         m_deckLinkOutput->ScheduleAudioSamples( pcm, samples, streamTime, frequency, (unsigned long*) &written );
297 #else
298                         m_deckLinkOutput->ScheduleAudioSamples( pcm, samples, streamTime, frequency, &written );
299 #endif
300
301                         if ( written != (uint32_t) samples )
302                                 mlt_log_verbose( getConsumer(), "renderAudio: samples=%d, written=%u\n", samples, written );
303                 }
304         }
305
306         bool createFrame( IDeckLinkMutableVideoFrame** decklinkFrame )
307         {
308                 BMDPixelFormat format = m_isKeyer? bmdFormat8BitARGB : bmdFormat8BitYUV;
309                 IDeckLinkMutableVideoFrame* frame = 0;
310                 uint8_t *buffer = 0;
311                 int stride = m_width * ( m_isKeyer? 4 : 2 );
312
313                 *decklinkFrame = NULL;
314
315                 // Generate a DeckLink video frame
316                 if ( S_OK != m_deckLinkOutput->CreateVideoFrame( m_width, m_height,
317                         stride, format, bmdFrameFlagDefault, &frame ) )
318                 {
319                         mlt_log_verbose( getConsumer(), "Failed to create video frame\n" );
320                         stop();
321                         return false;
322                 }
323                 
324                 // Make the first line black for field order correction.
325                 if ( S_OK == frame->GetBytes( (void**) &buffer ) && buffer )
326                 {
327                         if ( m_isKeyer )
328                         {
329                                 memset( buffer, 0, stride );
330                         }
331                         else for ( int i = 0; i < m_width; i++ )
332                         {
333                                 *buffer++ = 128;
334                                 *buffer++ = 16;
335                         }
336                 }
337
338                 *decklinkFrame = frame;
339
340                 return true;
341         }
342
343         void renderVideo( mlt_frame frame )
344         {
345                 mlt_image_format format = m_isKeyer? mlt_image_rgb24a : mlt_image_yuv422;
346                 uint8_t* image = 0;
347                 int rendered = mlt_properties_get_int( MLT_FRAME_PROPERTIES(frame), "rendered");
348
349                 if ( rendered && !mlt_frame_get_image( frame, &image, &format, &m_width, &m_height, 0 ) )
350                 {
351                         uint8_t* buffer = 0;
352                         int stride = m_width * ( m_isKeyer? 4 : 2 );
353
354                         if ( m_decklinkFrame )
355                                 m_decklinkFrame->Release();
356                         if ( createFrame( &m_decklinkFrame ) )
357                                 m_decklinkFrame->GetBytes( (void**) &buffer );
358
359                         if ( buffer )
360                         {
361                                 int progressive = mlt_properties_get_int( MLT_FRAME_PROPERTIES( frame ), "progressive" );
362
363                                 if ( !m_isKeyer )
364                                 {
365                                         // Normal non-keyer playout - needs byte swapping
366                                         if ( !progressive && m_displayMode->GetFieldDominance() == bmdUpperFieldFirst )
367                                                 // convert lower field first to top field first
368                                                 swab( (char*) image, (char*) buffer + stride, stride * ( m_height - 1 ) );
369                                         else
370                                                 swab( (char*) image, (char*) buffer, stride * m_height );
371                                 }
372                                 else if ( !mlt_properties_get_int( MLT_FRAME_PROPERTIES( frame ), "test_image" ) )
373                                 {
374                                         // Normal keyer output
375                                         int y = m_height + 1;
376                                         uint32_t* s = (uint32_t*) image;
377                                         uint32_t* d = (uint32_t*) buffer;
378
379                                         if ( !progressive && m_displayMode->GetFieldDominance() == bmdUpperFieldFirst )
380                                         {
381                                                 // Correct field order
382                                                 m_height--;
383                                                 y--;
384                                                 d += m_width;
385                                         }
386
387                                         // Need to relocate alpha channel RGBA => ARGB
388                                         while ( --y )
389                                         {
390                                                 int x = m_width + 1;
391                                                 while ( --x )
392                                                 {
393                                                         *d++ = ( *s << 8 ) | ( *s >> 24 );
394                                                         s++;
395                                                 }
396                                         }
397                                 }
398                                 else
399                                 {
400                                         // Keying blank frames - nullify alpha
401                                         memset( buffer, 0, stride * m_height );
402                                 }
403                         }
404                 }
405                 if ( m_decklinkFrame )
406                         m_deckLinkOutput->ScheduleVideoFrame( m_decklinkFrame, m_count * m_duration, m_duration, m_timescale );
407
408                 if ( !rendered )
409                         mlt_log_verbose( getConsumer(), "dropped video frame %u\n", ++m_dropped );
410         }
411
412         HRESULT render( mlt_frame frame )
413         {
414                 HRESULT result = S_OK;
415
416                 // Get the audio
417                 double speed = mlt_properties_get_double( MLT_FRAME_PROPERTIES(frame), "_speed" );
418                 if ( m_isAudio && speed == 1.0 )
419                         renderAudio( frame );
420
421                 // Get the video
422                 renderVideo( frame );
423                 ++m_count;
424
425                 return result;
426         }
427         
428         // *** DeckLink API implementation of IDeckLinkVideoOutputCallback IDeckLinkAudioOutputCallback *** //
429
430         // IUnknown needs only a dummy implementation
431         virtual HRESULT STDMETHODCALLTYPE QueryInterface( REFIID iid, LPVOID *ppv )
432                 { return E_NOINTERFACE; }
433         virtual ULONG STDMETHODCALLTYPE AddRef()
434                 { return 1; }
435         virtual ULONG STDMETHODCALLTYPE Release()
436                 { return 1; }
437         
438         /************************* DeckLink API Delegate Methods *****************************/
439         
440         virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted( IDeckLinkVideoFrame* completedFrame, BMDOutputFrameCompletionResult completed )
441         {
442                 uint32_t cnt;
443                 m_deckLinkOutput->GetBufferedAudioSampleFrameCount(&cnt);
444                 if(cnt != m_acnt)
445                 {
446                         mlt_log_verbose( getConsumer(),
447                                 "ScheduledFrameCompleted: GetBufferedAudioSampleFrameCount %u -> %u, m_count=%"PRIu64"\n",
448                                 m_acnt, cnt, m_count );
449                         m_acnt = cnt;
450                 };
451
452                 // When a video frame has been released by the API, schedule another video frame to be output
453
454                 // ignore handler if frame was flushed
455                 if(bmdOutputFrameFlushed == completed)
456                         return S_OK;
457
458                 // schedule next frame
459                 ScheduleNextFrame(false);
460
461                 // step forward frames counter if underrun
462                 if(bmdOutputFrameDisplayedLate == completed)
463                 {
464                         mlt_log_verbose( getConsumer(), "ScheduledFrameCompleted: bmdOutputFrameDisplayedLate == completed\n");
465                         m_count++;
466                 }
467                 if(bmdOutputFrameDropped == completed)
468                 {
469                         mlt_log_verbose( getConsumer(), "ScheduledFrameCompleted: bmdOutputFrameDropped == completed\n");
470                         m_count++;
471                 }
472
473                 return S_OK;
474         }
475
476         virtual HRESULT STDMETHODCALLTYPE ScheduledPlaybackHasStopped()
477         {
478                 return mlt_consumer_is_stopped( getConsumer() ) ? S_FALSE : S_OK;
479         }
480         
481
482         void ScheduleNextFrame(bool preroll)
483         {
484                 // get the consumer
485                 mlt_consumer consumer = getConsumer();
486
487                 // Get the properties
488                 mlt_properties properties = MLT_CONSUMER_PROPERTIES( consumer );
489
490                 // Frame and size
491                 mlt_frame frame = NULL;
492
493                 if( mlt_properties_get_int( properties, "running" ) || preroll )
494                 {
495                         frame = mlt_consumer_rt_frame( consumer );
496                         if ( frame != NULL )
497                         {
498                                 render( frame );
499
500                                 mlt_events_fire( properties, "consumer-frame-show", frame, NULL );
501
502                                 // terminate on pause
503                                 if (m_terminate_on_pause &&
504                                         mlt_properties_get_double( MLT_FRAME_PROPERTIES( frame ), "_speed" ) == 0.0)
505                                         stop();
506
507                                 mlt_frame_close( frame );
508                         }
509                 }
510         }
511 };
512
513 /** Start the consumer.
514  */
515
516 static int start( mlt_consumer consumer )
517 {
518         // Get the properties
519         mlt_properties properties = MLT_CONSUMER_PROPERTIES( consumer );
520         DeckLinkConsumer* decklink = (DeckLinkConsumer*) consumer->child;
521         return decklink->start( mlt_properties_get_int( properties, "preroll" ) ) ? 0 : 1;
522 }
523
524 /** Stop the consumer.
525  */
526
527 static int stop( mlt_consumer consumer )
528 {
529         // Get the properties
530         DeckLinkConsumer* decklink = (DeckLinkConsumer*) consumer->child;
531         return decklink->stop();
532 }
533
534 /** Determine if the consumer is stopped.
535  */
536
537 static int is_stopped( mlt_consumer consumer )
538 {
539         // Get the properties
540         mlt_properties properties = MLT_CONSUMER_PROPERTIES( consumer );
541         return !mlt_properties_get_int( properties, "running" );
542 }
543
544 /** Close the consumer.
545  */
546
547 static void close( mlt_consumer consumer )
548 {
549         // Stop the consumer
550         mlt_consumer_stop( consumer );
551
552         // Close the parent
553         consumer->close = NULL;
554         mlt_consumer_close( consumer );
555
556         // Free the memory
557         delete (DeckLinkConsumer*) consumer->child;
558 }
559
560 extern "C" {
561
562 /** Initialise the consumer.
563  */
564
565 mlt_consumer consumer_decklink_init( mlt_profile profile, mlt_service_type type, const char *id, char *arg )
566 {
567         // Allocate the consumer
568         DeckLinkConsumer* decklink = new DeckLinkConsumer();
569         mlt_consumer consumer = NULL;
570
571         // If allocated
572         if ( decklink && !mlt_consumer_init( decklink->getConsumer(), decklink, profile ) )
573         {
574                 // If initialises without error
575                 if ( decklink->open( arg? atoi(arg) : 0 ) )
576                 {
577                         consumer = decklink->getConsumer();
578                         
579                         // Setup callbacks
580                         consumer->close = close;
581                         consumer->start = start;
582                         consumer->stop = stop;
583                         consumer->is_stopped = is_stopped;
584                         mlt_properties_set( MLT_CONSUMER_PROPERTIES(consumer), "deinterlace_method", "onefield" );
585                 }
586         }
587
588         // Return consumer
589         return consumer;
590 }
591
592 extern mlt_producer producer_decklink_init( mlt_profile profile, mlt_service_type type, const char *id, char *arg );
593
594 static mlt_properties metadata( mlt_service_type type, const char *id, void *data )
595 {
596         char file[ PATH_MAX ];
597         const char *service_type = NULL;
598         switch ( type )
599         {
600                 case consumer_type:
601                         service_type = "consumer";
602                         break;
603                 case producer_type:
604                         service_type = "producer";
605                         break;
606                 default:
607                         return NULL;
608         }
609         snprintf( file, PATH_MAX, "%s/decklink/%s_%s.yml", mlt_environment( "MLT_DATA" ), service_type, id );
610         return mlt_properties_parse_yaml( file );
611 }
612
613 MLT_REPOSITORY
614 {
615         MLT_REGISTER( consumer_type, "decklink", consumer_decklink_init );
616         MLT_REGISTER( producer_type, "decklink", producer_decklink_init );
617         MLT_REGISTER_METADATA( consumer_type, "decklink", metadata, NULL );
618         MLT_REGISTER_METADATA( producer_type, "decklink", metadata, NULL );
619 }
620
621 } // extern C