git.sesse.net Git - mlt/blob - src/modules/plus/consumer_blipflash.c

   1 /*
   2  * consumer_blipflash.c -- a consumer to measure A/V sync from a blip/flash
   3  *                         source
   4  * Copyright (C) 2013 Brian Matherly
   5  * Author: Brian Matherly <pez4brian@yahoo.com>
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2.1 of the License, or (at your option) any later version.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with this library; if not, write to the Free Software
  19  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20  */
  21
  22 // mlt Header files
  23 #include <framework/mlt_consumer.h>
  24 #include <framework/mlt_frame.h>
  25 #include <framework/mlt_deque.h>
  26
  27 // System header files
  28 #include <limits.h>
  29 #include <stdio.h>
  30 #include <stdlib.h>
  31 #include <string.h>
  32 #include <pthread.h>
  33
  34 // Private constants
  35 #define SAMPLE_FREQ 48000
  36 #define FLASH_LUMA_THRESHOLD 150
  37 #define BLIP_THRESHOLD 0.5
  38
  39 // Private types
  40 typedef struct
  41 {
  42         int64_t flash_history[2];
  43         int flash_history_count;
  44         int64_t blip_history[2];
  45         int blip_history_count;
  46         int blip_in_progress;
  47         int samples_since_blip;
  48         int blip;
  49         int flash;
  50         int sample_offset;
  51         FILE* out_file;
  52         int report_frames;
  53 } avsync_stats;
  54
  55 // Forward references.
  56 static int consumer_start( mlt_consumer consumer );
  57 static int consumer_stop( mlt_consumer consumer );
  58 static int consumer_is_stopped( mlt_consumer consumer );
  59 static void *consumer_thread( void *arg );
  60 static void consumer_close( mlt_consumer consumer );
  61
  62 /** Initialize the consumer.
  63 */
  64
  65 mlt_consumer consumer_blipflash_init( mlt_profile profile, mlt_service_type type, const char *id, char *arg )
  66 {
  67         // Allocate the consumer
  68         mlt_consumer consumer = mlt_consumer_new( profile );
  69         mlt_properties consumer_properties = MLT_CONSUMER_PROPERTIES( consumer );
  70         avsync_stats* stats = NULL;
  71
  72         // If memory allocated and initializes without error
  73         if ( consumer != NULL )
  74         {
  75                 // Set up start/stop/terminated callbacks
  76                 consumer->close = consumer_close;
  77                 consumer->start = consumer_start;
  78                 consumer->stop = consumer_stop;
  79                 consumer->is_stopped = consumer_is_stopped;
  80
  81                 stats = mlt_pool_alloc( sizeof( avsync_stats ) );
  82                 stats->flash_history_count = 0;
  83                 stats->blip_history_count = 0;
  84                 stats->blip_in_progress = 0;
  85                 stats->samples_since_blip = 0;
  86                 stats->blip = 0;
  87                 stats->flash = 0;
  88                 stats->sample_offset = INT_MAX;
  89                 stats->report_frames = 0;
  90                 stats->out_file = stdout;
  91                 if ( arg != NULL )
  92                 {
  93                         FILE* out_file = fopen( arg, "w" );
  94                         if ( out_file != NULL )
  95                                 stats->out_file = out_file;
  96                 }
  97                 mlt_properties_set_data( consumer_properties, "_stats", stats, 0, NULL, NULL );
  98
  99                 mlt_properties_set( consumer_properties, "report", "blip" );
 100         }
 101
 102         // Return this
 103         return consumer;
 104 }
 105
 106 /** Start the consumer.
 107 */
 108
 109 static int consumer_start( mlt_consumer consumer )
 110 {
 111         // Get the properties
 112         mlt_properties properties = MLT_CONSUMER_PROPERTIES( consumer );
 113
 114         // Check that we're not already running
 115         if ( !mlt_properties_get_int( properties, "_running" ) )
 116         {
 117                 // Allocate a thread
 118                 pthread_t *thread = calloc( 1, sizeof( pthread_t ) );
 119
 120                 // Assign the thread to properties
 121                 mlt_properties_set_data( properties, "_thread", thread, sizeof( pthread_t ), free, NULL );
 122
 123                 // Set the running state
 124                 mlt_properties_set_int( properties, "_running", 1 );
 125
 126                 // Create the thread
 127                 pthread_create( thread, NULL, consumer_thread, consumer );
 128         }
 129         return 0;
 130 }
 131
 132 /** Stop the consumer.
 133 */
 134
 135 static int consumer_stop( mlt_consumer consumer )
 136 {
 137         // Get the properties
 138         mlt_properties properties = MLT_CONSUMER_PROPERTIES( consumer );
 139
 140         // Check that we're running
 141         if ( mlt_properties_get_int( properties, "_running" ) )
 142         {
 143                 // Get the thread
 144                 pthread_t *thread = mlt_properties_get_data( properties, "_thread", NULL );
 145
 146                 // Stop the thread
 147                 mlt_properties_set_int( properties, "_running", 0 );
 148
 149                 // Wait for termination
 150                 if ( thread )
 151                         pthread_join( *thread, NULL );
 152         }
 153
 154         return 0;
 155 }
 156
 157 /** Determine if the consumer is stopped.
 158 */
 159
 160 static int consumer_is_stopped( mlt_consumer consumer )
 161 {
 162         // Get the properties
 163         mlt_properties properties = MLT_CONSUMER_PROPERTIES( consumer );
 164         return !mlt_properties_get_int( properties, "_running" );
 165 }
 166
 167 static void detect_flash( mlt_frame frame, mlt_position pos, double fps, avsync_stats* stats )
 168 {
 169         int width = 0;
 170         int height = 0;
 171         mlt_image_format format = mlt_image_yuv422;
 172         uint8_t* image = NULL;
 173         int error = mlt_frame_get_image( frame, &image, &format, &width, &height, 0 );
 174
 175         if ( !error && format == mlt_image_yuv422 && image != NULL )
 176         {
 177                 int i, j = 0;
 178                 int y_accumulator = 0;
 179
 180                 // Add up the luma values from 4 samples in 4 different quadrants.
 181                 for( i = 1; i < 3; i++ )
 182                 {
 183                         int x = ( width / 3 ) * i;
 184                         x = x - x % 2; // Make sure this is a luma sample
 185                         for( j = 1; j < 3; j++ )
 186                         {
 187                                 int y = ( height / 3 ) * j;
 188                                 y_accumulator += image[ y * height * 2 + x * 2 ];
 189                         }
 190                 }
 191                 // If the average luma value is > 150, assume it is a flash.
 192                 stats->flash = ( y_accumulator / 4 ) > FLASH_LUMA_THRESHOLD;
 193         }
 194
 195         if( stats->flash )
 196         {
 197                 stats->flash_history[1] = stats->flash_history[0];
 198                 stats->flash_history[0] = mlt_sample_calculator_to_now( fps, SAMPLE_FREQ, pos );
 199                 if( stats->flash_history_count < 2 )
 200                 {
 201                         stats->flash_history_count++;
 202                 }
 203         }
 204 }
 205
 206 static void detect_blip( mlt_frame frame, mlt_position pos, double fps, avsync_stats* stats )
 207 {
 208         int frequency = SAMPLE_FREQ;
 209         int channels = 1;
 210         int samples = mlt_sample_calculator( fps, frequency, pos );
 211         mlt_audio_format format = mlt_audio_float;
 212         float* buffer = NULL;
 213         int error = mlt_frame_get_audio( frame, (void**) &buffer, &format, &frequency, &channels, &samples );
 214
 215         if ( !error && format == mlt_audio_float && buffer != NULL )
 216         {
 217                 int i = 0;
 218
 219                 for( i = 0; i < samples; i++ )
 220                 {
 221                         if( !stats->blip_in_progress )
 222                         {
 223                                 if( buffer[i] > BLIP_THRESHOLD || buffer[i] < -BLIP_THRESHOLD )
 224                                 {
 225                                         // This sample must start a blip
 226                                         stats->blip_in_progress = 1;
 227                                         stats->samples_since_blip = 0;
 228
 229                                         stats->blip_history[1] = stats->blip_history[0];
 230                                         stats->blip_history[0] = mlt_sample_calculator_to_now( fps, SAMPLE_FREQ, pos );
 231                                         stats->blip_history[0] += i;
 232                                         if( stats->blip_history_count < 2 )
 233                                         {
 234                                                 stats->blip_history_count++;
 235                                         }
 236                                         stats->blip = 1;
 237                                 }
 238                         }
 239                         else
 240                         {
 241                                 if( buffer[i] > -BLIP_THRESHOLD && buffer[i] < BLIP_THRESHOLD )
 242                                 {
 243                                         if( ++stats->samples_since_blip > frequency / 1000 )
 244                                         {
 245                                                 // One ms of silence means the blip is over
 246                                                 stats->blip_in_progress = 0;
 247                                                 stats->samples_since_blip = 0;
 248                                         }
 249                                 }
 250                                 else
 251                                 {
 252                                         stats->samples_since_blip = 0;
 253                                 }
 254                         }
 255                 }
 256         }
 257 }
 258
 259 static void calculate_sync( avsync_stats* stats )
 260 {
 261         if( stats->blip || stats->flash )
 262         {
 263                 if( stats->flash_history_count > 0 &&
 264                         stats->blip_history_count > 0 &&
 265                         stats->blip_history[0] == stats->flash_history[0] )
 266                 {
 267                         // The flash and blip occurred at the same time.
 268                         stats->sample_offset = 0;
 269                 }
 270                 if( stats->flash_history_count > 1 &&
 271                         stats->blip_history_count > 0 &&
 272                         stats->blip_history[0] <= stats->flash_history[0] &&
 273                         stats->blip_history[0] >= stats->flash_history[1] )
 274                 {
 275                         // The latest blip occurred between two flashes
 276                         if( stats->flash_history[0] - stats->blip_history[0] >
 277                             stats->blip_history[0] - stats->flash_history[1] )
 278                         {
 279                                 // Blip is closer to the previous flash.
 280                                 // F1---B0--------F0
 281                                 // ^----^
 282                                 // Video leads audio (negative number).
 283                                 stats->sample_offset = (int)(stats->flash_history[1] - stats->blip_history[0] );
 284                         }
 285                         else
 286                         {
 287                                 // Blip is closer to the current flash.
 288                                 // F1--------B0---F0
 289                                 //           ^----^
 290                                 // Audio leads video (positive number).
 291                                 stats->sample_offset = (int)(stats->flash_history[0] - stats->blip_history[0]);
 292                         }
 293                 }
 294                 else if( stats->blip_history_count > 1 &&
 295                                  stats->flash_history_count > 0 &&
 296                                  stats->flash_history[0] <= stats->blip_history[0] &&
 297                                  stats->flash_history[0] >= stats->blip_history[1] )
 298                 {
 299                         // The latest flash occurred between two blips
 300                         if( stats->blip_history[0] - stats->flash_history[0] >
 301                             stats->flash_history[0] - stats->blip_history[1] )
 302                         {
 303                                 // Flash is closer to the previous blip.
 304                                 // B1---F0--------B0
 305                                 // ^----^
 306                                 // Audio leads video (positive number).
 307                                 stats->sample_offset = (int)(stats->flash_history[0] - stats->blip_history[1]);
 308                         }
 309                         else
 310                         {
 311                                 // Flash is closer to the latest blip.
 312                                 // B1--------F0---B0
 313                                 //           ^----^
 314                                 // Video leads audio (negative number).
 315                                 stats->sample_offset = (int)(stats->flash_history[0] - stats->blip_history[0] );
 316                         }
 317                 }
 318         }
 319 }
 320
 321 static void report_results( avsync_stats* stats, mlt_position pos )
 322 {
 323         if( stats->report_frames || stats->blip )
 324         {
 325                 if( stats->sample_offset == INT_MAX )
 326                 {
 327                         fprintf( stats->out_file, MLT_POSITION_FMT "\t??\n", pos );
 328                 }
 329                 else
 330                 {
 331                         // Convert to milliseconds.
 332                         double ms_offset = (double)stats->sample_offset * 1000.0 / (double)SAMPLE_FREQ;
 333                         fprintf( stats->out_file, MLT_POSITION_FMT "\t%02.02f\n", pos, ms_offset );
 334                 }
 335         }
 336         stats->blip = 0;
 337         stats->flash = 0;
 338 }
 339
 340 /** The main thread - the argument is simply the consumer.
 341 */
 342
 343 static void *consumer_thread( void *arg )
 344 {
 345         // Map the argument to the object
 346         mlt_consumer consumer = arg;
 347
 348         // Get the properties
 349         mlt_properties properties = MLT_CONSUMER_PROPERTIES( consumer );
 350
 351         // Convenience functionality
 352         int terminate_on_pause = mlt_properties_get_int( properties, "terminate_on_pause" );
 353         int terminated = 0;
 354
 355         // Frame and size
 356         mlt_frame frame = NULL;
 357
 358         // Loop while running
 359         while( !terminated && mlt_properties_get_int( properties, "_running" ) )
 360         {
 361                 // Get the frame
 362                 frame = mlt_consumer_rt_frame( consumer );
 363
 364                 // Check for termination
 365                 if ( terminate_on_pause && frame != NULL )
 366                         terminated = mlt_properties_get_double( MLT_FRAME_PROPERTIES( frame ), "_speed" ) == 0.0;
 367
 368                 // Check that we have a frame to work with
 369                 if ( frame )
 370                 {
 371                         avsync_stats* stats = mlt_properties_get_data( properties, "_stats", NULL );
 372                         double fps = mlt_properties_get_double( properties, "fps" );
 373                         mlt_position pos = mlt_frame_get_position( frame );
 374
 375                          if( !strcmp( mlt_properties_get( properties, "report" ), "frame" ) )
 376                          {
 377                                  stats->report_frames = 1;
 378                          }
 379                          else
 380                          {
 381                                  stats->report_frames = 0;
 382                          }
 383
 384                         detect_flash( frame, pos, fps, stats );
 385                         detect_blip( frame, pos, fps, stats );
 386                         calculate_sync( stats );
 387                         report_results( stats, pos );
 388
 389                         // Close the frame
 390                         mlt_events_fire( properties, "consumer-frame-show", frame, NULL );
 391                         mlt_frame_close( frame );
 392                 }
 393         }
 394
 395         // Indicate that the consumer is stopped
 396         mlt_properties_set_int( properties, "_running", 0 );
 397         mlt_consumer_stopped( consumer );
 398
 399         return NULL;
 400 }
 401
 402 /** Close the consumer.
 403 */
 404
 405 static void consumer_close( mlt_consumer consumer )
 406 {
 407         mlt_properties consumer_properties = MLT_CONSUMER_PROPERTIES( consumer );
 408         avsync_stats* stats = mlt_properties_get_data( consumer_properties, "_stats", NULL );
 409
 410         // Stop the consumer
 411         mlt_consumer_stop( consumer );
 412
 413         // Close the file
 414         if( stats->out_file != stdout )
 415         {
 416                 fclose( stats->out_file );
 417         }
 418
 419         // Clean up memory
 420         mlt_pool_release( stats );
 421
 422         // Close the parent
 423         mlt_consumer_close( consumer );
 424
 425         // Free the memory
 426         free( consumer );
 427 }