2 * consumer_blipflash.c -- a consumer to measure A/V sync from a blip/flash
4 * Copyright (C) 2013 Brian Matherly
5 * Author: Brian Matherly <pez4brian@yahoo.com>
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23 #include <framework/mlt_consumer.h>
24 #include <framework/mlt_frame.h>
25 #include <framework/mlt_deque.h>
27 // System header files
35 #define SAMPLE_FREQ 48000
36 #define FLASH_LUMA_THRESHOLD 150
37 #define BLIP_THRESHOLD 0.5
42 int64_t flash_history[2];
43 int flash_history_count;
44 int64_t blip_history[2];
45 int blip_history_count;
47 int samples_since_blip;
55 // Forward references.
56 static int consumer_start( mlt_consumer consumer );
57 static int consumer_stop( mlt_consumer consumer );
58 static int consumer_is_stopped( mlt_consumer consumer );
59 static void *consumer_thread( void *arg );
60 static void consumer_close( mlt_consumer consumer );
62 /** Initialize the consumer.
65 mlt_consumer consumer_blipflash_init( mlt_profile profile, mlt_service_type type, const char *id, char *arg )
67 // Allocate the consumer
68 mlt_consumer consumer = mlt_consumer_new( profile );
69 mlt_properties consumer_properties = MLT_CONSUMER_PROPERTIES( consumer );
70 avsync_stats* stats = NULL;
72 // If memory allocated and initializes without error
73 if ( consumer != NULL )
75 // Set up start/stop/terminated callbacks
76 consumer->close = consumer_close;
77 consumer->start = consumer_start;
78 consumer->stop = consumer_stop;
79 consumer->is_stopped = consumer_is_stopped;
81 stats = mlt_pool_alloc( sizeof( avsync_stats ) );
82 stats->flash_history_count = 0;
83 stats->blip_history_count = 0;
84 stats->blip_in_progress = 0;
85 stats->samples_since_blip = 0;
88 stats->sample_offset = INT_MAX;
89 stats->report_frames = 0;
90 stats->out_file = stdout;
93 FILE* out_file = fopen( arg, "w" );
94 if ( out_file != NULL )
95 stats->out_file = out_file;
97 mlt_properties_set_data( consumer_properties, "_stats", stats, 0, NULL, NULL );
99 mlt_properties_set( consumer_properties, "report", "blip" );
106 /** Start the consumer.
109 static int consumer_start( mlt_consumer consumer )
111 // Get the properties
112 mlt_properties properties = MLT_CONSUMER_PROPERTIES( consumer );
114 // Check that we're not already running
115 if ( !mlt_properties_get_int( properties, "_running" ) )
118 pthread_t *thread = calloc( 1, sizeof( pthread_t ) );
120 // Assign the thread to properties
121 mlt_properties_set_data( properties, "_thread", thread, sizeof( pthread_t ), free, NULL );
123 // Set the running state
124 mlt_properties_set_int( properties, "_running", 1 );
127 pthread_create( thread, NULL, consumer_thread, consumer );
132 /** Stop the consumer.
135 static int consumer_stop( mlt_consumer consumer )
137 // Get the properties
138 mlt_properties properties = MLT_CONSUMER_PROPERTIES( consumer );
140 // Check that we're running
141 if ( mlt_properties_get_int( properties, "_running" ) )
144 pthread_t *thread = mlt_properties_get_data( properties, "_thread", NULL );
147 mlt_properties_set_int( properties, "_running", 0 );
149 // Wait for termination
151 pthread_join( *thread, NULL );
157 /** Determine if the consumer is stopped.
160 static int consumer_is_stopped( mlt_consumer consumer )
162 // Get the properties
163 mlt_properties properties = MLT_CONSUMER_PROPERTIES( consumer );
164 return !mlt_properties_get_int( properties, "_running" );
167 static void detect_flash( mlt_frame frame, mlt_position pos, double fps, avsync_stats* stats )
171 mlt_image_format format = mlt_image_yuv422;
172 uint8_t* image = NULL;
173 int error = mlt_frame_get_image( frame, &image, &format, &width, &height, 0 );
175 if ( !error && format == mlt_image_yuv422 && image != NULL )
178 int y_accumulator = 0;
180 // Add up the luma values from 4 samples in 4 different quadrants.
181 for( i = 1; i < 3; i++ )
183 int x = ( width / 3 ) * i;
184 x = x - x % 2; // Make sure this is a luma sample
185 for( j = 1; j < 3; j++ )
187 int y = ( height / 3 ) * j;
188 y_accumulator += image[ y * height * 2 + x * 2 ];
191 // If the average luma value is > 150, assume it is a flash.
192 stats->flash = ( y_accumulator / 4 ) > FLASH_LUMA_THRESHOLD;
197 stats->flash_history[1] = stats->flash_history[0];
198 stats->flash_history[0] = mlt_sample_calculator_to_now( fps, SAMPLE_FREQ, pos );
199 if( stats->flash_history_count < 2 )
201 stats->flash_history_count++;
206 static void detect_blip( mlt_frame frame, mlt_position pos, double fps, avsync_stats* stats )
208 int frequency = SAMPLE_FREQ;
210 int samples = mlt_sample_calculator( fps, frequency, pos );
211 mlt_audio_format format = mlt_audio_float;
212 float* buffer = NULL;
213 int error = mlt_frame_get_audio( frame, (void**) &buffer, &format, &frequency, &channels, &samples );
215 if ( !error && format == mlt_audio_float && buffer != NULL )
219 for( i = 0; i < samples; i++ )
221 if( !stats->blip_in_progress )
223 if( buffer[i] > BLIP_THRESHOLD || buffer[i] < -BLIP_THRESHOLD )
225 // This sample must start a blip
226 stats->blip_in_progress = 1;
227 stats->samples_since_blip = 0;
229 stats->blip_history[1] = stats->blip_history[0];
230 stats->blip_history[0] = mlt_sample_calculator_to_now( fps, SAMPLE_FREQ, pos );
231 stats->blip_history[0] += i;
232 if( stats->blip_history_count < 2 )
234 stats->blip_history_count++;
241 if( buffer[i] > -BLIP_THRESHOLD && buffer[i] < BLIP_THRESHOLD )
243 if( ++stats->samples_since_blip > frequency / 1000 )
245 // One ms of silence means the blip is over
246 stats->blip_in_progress = 0;
247 stats->samples_since_blip = 0;
252 stats->samples_since_blip = 0;
259 static void calculate_sync( avsync_stats* stats )
261 if( stats->blip || stats->flash )
263 if( stats->flash_history_count > 0 &&
264 stats->blip_history_count > 0 &&
265 stats->blip_history[0] == stats->flash_history[0] )
267 // The flash and blip occurred at the same time.
268 stats->sample_offset = 0;
270 if( stats->flash_history_count > 1 &&
271 stats->blip_history_count > 0 &&
272 stats->blip_history[0] <= stats->flash_history[0] &&
273 stats->blip_history[0] >= stats->flash_history[1] )
275 // The latest blip occurred between two flashes
276 if( stats->flash_history[0] - stats->blip_history[0] >
277 stats->blip_history[0] - stats->flash_history[1] )
279 // Blip is closer to the previous flash.
282 // Video leads audio (negative number).
283 stats->sample_offset = (int)(stats->flash_history[1] - stats->blip_history[0] );
287 // Blip is closer to the current flash.
290 // Audio leads video (positive number).
291 stats->sample_offset = (int)(stats->flash_history[0] - stats->blip_history[0]);
294 else if( stats->blip_history_count > 1 &&
295 stats->flash_history_count > 0 &&
296 stats->flash_history[0] <= stats->blip_history[0] &&
297 stats->flash_history[0] >= stats->blip_history[1] )
299 // The latest flash occurred between two blips
300 if( stats->blip_history[0] - stats->flash_history[0] >
301 stats->flash_history[0] - stats->blip_history[1] )
303 // Flash is closer to the previous blip.
306 // Audio leads video (positive number).
307 stats->sample_offset = (int)(stats->flash_history[0] - stats->blip_history[1]);
311 // Flash is closer to the latest blip.
314 // Video leads audio (negative number).
315 stats->sample_offset = (int)(stats->flash_history[0] - stats->blip_history[0] );
321 static void report_results( avsync_stats* stats, mlt_position pos )
323 if( stats->report_frames || stats->blip )
325 if( stats->sample_offset == INT_MAX )
327 fprintf( stats->out_file, MLT_POSITION_FMT "\t??\n", pos );
331 // Convert to milliseconds.
332 double ms_offset = (double)stats->sample_offset * 1000.0 / (double)SAMPLE_FREQ;
333 fprintf( stats->out_file, MLT_POSITION_FMT "\t%02.02f\n", pos, ms_offset );
340 /** The main thread - the argument is simply the consumer.
343 static void *consumer_thread( void *arg )
345 // Map the argument to the object
346 mlt_consumer consumer = arg;
348 // Get the properties
349 mlt_properties properties = MLT_CONSUMER_PROPERTIES( consumer );
351 // Convenience functionality
352 int terminate_on_pause = mlt_properties_get_int( properties, "terminate_on_pause" );
356 mlt_frame frame = NULL;
358 // Loop while running
359 while( !terminated && mlt_properties_get_int( properties, "_running" ) )
362 frame = mlt_consumer_rt_frame( consumer );
364 // Check for termination
365 if ( terminate_on_pause && frame != NULL )
366 terminated = mlt_properties_get_double( MLT_FRAME_PROPERTIES( frame ), "_speed" ) == 0.0;
368 // Check that we have a frame to work with
371 avsync_stats* stats = mlt_properties_get_data( properties, "_stats", NULL );
372 double fps = mlt_properties_get_double( properties, "fps" );
373 mlt_position pos = mlt_frame_get_position( frame );
375 if( !strcmp( mlt_properties_get( properties, "report" ), "frame" ) )
377 stats->report_frames = 1;
381 stats->report_frames = 0;
384 detect_flash( frame, pos, fps, stats );
385 detect_blip( frame, pos, fps, stats );
386 calculate_sync( stats );
387 report_results( stats, pos );
390 mlt_events_fire( properties, "consumer-frame-show", frame, NULL );
391 mlt_frame_close( frame );
395 // Indicate that the consumer is stopped
396 mlt_properties_set_int( properties, "_running", 0 );
397 mlt_consumer_stopped( consumer );
402 /** Close the consumer.
405 static void consumer_close( mlt_consumer consumer )
407 mlt_properties consumer_properties = MLT_CONSUMER_PROPERTIES( consumer );
408 avsync_stats* stats = mlt_properties_get_data( consumer_properties, "_stats", NULL );
411 mlt_consumer_stop( consumer );
414 if( stats->out_file != stdout )
416 fclose( stats->out_file );
420 mlt_pool_release( stats );
423 mlt_consumer_close( consumer );