git.sesse.net Git - mlt/blob - src/framework/mlt_consumer.c

   1 /**
   2  * \file mlt_consumer.c
   3  * \brief abstraction for all consumer services
   4  * \see mlt_consumer_s
   5  *
   6  * Copyright (C) 2003-2010 Ushodaya Enterprises Limited
   7  * \author Charles Yates <charles.yates@pandora.be>
   8  * \author Dan Dennedy <dan@dennedy.org>
   9  *
  10  * This library is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU Lesser General Public
  12  * License as published by the Free Software Foundation; either
  13  * version 2.1 of the License, or (at your option) any later version.
  14  *
  15  * This library is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18  * Lesser General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU Lesser General Public
  21  * License along with this library; if not, write to the Free Software
  22  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  23  */
  24
  25 #include "mlt_consumer.h"
  26 #include "mlt_factory.h"
  27 #include "mlt_producer.h"
  28 #include "mlt_frame.h"
  29 #include "mlt_profile.h"
  30 #include "mlt_log.h"
  31
  32 #include <stdio.h>
  33 #include <string.h>
  34 #include <stdlib.h>
  35 #include <sys/time.h>
  36
  37 /** Define this if you want an automatic deinterlace (if necessary) when the
  38  * consumer's producer is not running at normal speed.
  39  */
  40 #undef DEINTERLACE_ON_NOT_NORMAL_SPEED
  41
  42 /** This is not the ideal place for this, but it is needed by VDPAU as well.
  43  */
  44 pthread_mutex_t mlt_sdl_mutex = PTHREAD_MUTEX_INITIALIZER;
  45
  46 /** \brief private members of mlt_consumer */
  47
  48 typedef struct
  49 {
  50         int real_time;
  51         int ahead;
  52         mlt_image_format format;
  53         mlt_deque queue;
  54         pthread_t ahead_thread;
  55         pthread_mutex_t queue_mutex;
  56         pthread_cond_t queue_cond;
  57         pthread_mutex_t put_mutex;
  58         pthread_cond_t put_cond;
  59         mlt_frame put;
  60         int put_active;
  61         mlt_event event_listener;
  62         mlt_position position;
  63         int is_purge;
  64
  65         /* additional fields added for the parallel work queue */
  66         mlt_deque worker_threads;
  67         pthread_mutex_t done_mutex;
  68         pthread_cond_t done_cond;
  69         int consecutive_dropped;
  70         int consecutive_rendered;
  71         int process_head;
  72         int started;
  73         pthread_t *threads; /**< used to deallocate all threads */
  74 }
  75 consumer_private;
  76
  77 static void mlt_consumer_frame_render( mlt_listener listener, mlt_properties owner, mlt_service self, void **args );
  78 static void mlt_consumer_frame_show( mlt_listener listener, mlt_properties owner, mlt_service self, void **args );
  79 static void mlt_consumer_property_changed( mlt_properties owner, mlt_consumer self, char *name );
  80 static void apply_profile_properties( mlt_consumer self, mlt_profile profile, mlt_properties properties );
  81 static void on_consumer_frame_show( mlt_properties owner, mlt_consumer self, mlt_frame frame );
  82
  83 /** Initialize a consumer service.
  84  *
  85  * \public \memberof mlt_consumer_s
  86  * \param self the consumer to initialize
  87  * \param child a pointer to the object for the subclass
  88  * \param profile the \p mlt_profile_s to use (optional but recommended,
  89  * uses the environment variable MLT if self is NULL)
  90  * \return true if there was an error
  91  */
  92
  93 int mlt_consumer_init( mlt_consumer self, void *child, mlt_profile profile )
  94 {
  95         int error = 0;
  96         memset( self, 0, sizeof( struct mlt_consumer_s ) );
  97         self->child = child;
  98         consumer_private *priv = self->local = calloc( 1, sizeof( consumer_private ) );
  99
 100         error = mlt_service_init( &self->parent, self );
 101         if ( error == 0 )
 102         {
 103                 // Get the properties from the service
 104                 mlt_properties properties = MLT_SERVICE_PROPERTIES( &self->parent );
 105
 106                 // Apply profile to properties
 107                 if ( profile == NULL )
 108                 {
 109                         // Normally the application creates the profile and controls its lifetime
 110                         // This is the fallback exception handling
 111                         profile = mlt_profile_init( NULL );
 112                         mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
 113                         mlt_properties_set_data( properties, "_profile", profile, 0, (mlt_destructor)mlt_profile_close, NULL );
 114                 }
 115                 apply_profile_properties( self, profile, properties );
 116
 117                 // Default rescaler for all consumers
 118                 mlt_properties_set( properties, "rescale", "bilinear" );
 119
 120                 // Default read ahead buffer size
 121                 mlt_properties_set_int( properties, "buffer", 25 );
 122                 mlt_properties_set_int( properties, "drop_max", 5 );
 123
 124                 // Default audio frequency and channels
 125                 mlt_properties_set_int( properties, "frequency", 48000 );
 126                 mlt_properties_set_int( properties, "channels", 2 );
 127
 128                 // Default of all consumers is real time
 129                 mlt_properties_set_int( properties, "real_time", 1 );
 130
 131                 // Default to environment test card
 132                 mlt_properties_set( properties, "test_card", mlt_environment( "MLT_TEST_CARD" ) );
 133
 134                 // Hmm - default all consumers to yuv422 :-/
 135                 priv->format = mlt_image_yuv422;
 136                 mlt_properties_set( properties, "mlt_image_format", mlt_image_format_name( priv->format ) );
 137                 mlt_properties_set( properties, "mlt_audio_format", mlt_audio_format_name( mlt_audio_s16 ) );
 138
 139                 mlt_events_register( properties, "consumer-frame-show", ( mlt_transmitter )mlt_consumer_frame_show );
 140                 mlt_events_register( properties, "consumer-frame-render", ( mlt_transmitter )mlt_consumer_frame_render );
 141                 mlt_events_register( properties, "consumer-thread-started", NULL );
 142                 mlt_events_register( properties, "consumer-thread-stopped", NULL );
 143                 mlt_events_register( properties, "consumer-stopping", NULL );
 144                 mlt_events_register( properties, "consumer-stopped", NULL );
 145                 mlt_events_listen( properties, self, "consumer-frame-show", ( mlt_listener )on_consumer_frame_show );
 146
 147                 // Register a property-changed listener to handle the profile property -
 148                 // subsequent properties can override the profile
 149                 priv->event_listener = mlt_events_listen( properties, self, "property-changed", ( mlt_listener )mlt_consumer_property_changed );
 150
 151                 // Create the push mutex and condition
 152                 pthread_mutex_init( &priv->put_mutex, NULL );
 153                 pthread_cond_init( &priv->put_cond, NULL );
 154
 155         }
 156         return error;
 157 }
 158
 159 /** Convert the profile into properties on the consumer.
 160  *
 161  * \private \memberof mlt_consumer_s
 162  * \param self a consumer
 163  * \param profile a profile
 164  * \param properties a properties list (typically, the consumer's)
 165  */
 166
 167 static void apply_profile_properties( mlt_consumer self, mlt_profile profile, mlt_properties properties )
 168 {
 169         consumer_private *priv = self->local;
 170         mlt_event_block( priv->event_listener );
 171         mlt_properties_set_double( properties, "fps", mlt_profile_fps( profile ) );
 172         mlt_properties_set_int( properties, "frame_rate_num", profile->frame_rate_num );
 173         mlt_properties_set_int( properties, "frame_rate_den", profile->frame_rate_den );
 174         mlt_properties_set_int( properties, "width", profile->width );
 175         mlt_properties_set_int( properties, "height", profile->height );
 176         mlt_properties_set_int( properties, "progressive", profile->progressive );
 177         mlt_properties_set_double( properties, "aspect_ratio", mlt_profile_sar( profile )  );
 178         mlt_properties_set_int( properties, "sample_aspect_num", profile->sample_aspect_num );
 179         mlt_properties_set_int( properties, "sample_aspect_den", profile->sample_aspect_den );
 180         mlt_properties_set_double( properties, "display_ratio", mlt_profile_dar( profile )  );
 181         mlt_properties_set_int( properties, "display_aspect_num", profile->display_aspect_num );
 182         mlt_properties_set_int( properties, "display_aspect_den", profile->display_aspect_den );
 183         mlt_properties_set_int( properties, "colorspace", profile->colorspace );
 184         mlt_event_unblock( priv->event_listener );
 185 }
 186
 187 /** The property-changed event listener
 188  *
 189  * \private \memberof mlt_consumer_s
 190  * \param owner the events object
 191  * \param self the consumer
 192  * \param name the name of the property that changed
 193  */
 194
 195 static void mlt_consumer_property_changed( mlt_properties owner, mlt_consumer self, char *name )
 196 {
 197         if ( !strcmp( name, "mlt_profile" ) )
 198         {
 199                 // Get the properies
 200                 mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
 201
 202                 // Get the current profile
 203                 mlt_profile profile = mlt_service_profile( MLT_CONSUMER_SERVICE( self ) );
 204
 205                 // Load the new profile
 206                 mlt_profile new_profile = mlt_profile_init( mlt_properties_get( properties, name ) );
 207
 208                 if ( new_profile )
 209                 {
 210                         // Copy the profile
 211                         if ( profile != NULL )
 212                         {
 213                                 free( profile->description );
 214                                 memcpy( profile, new_profile, sizeof( struct mlt_profile_s ) );
 215                                 profile->description = strdup( new_profile->description );
 216                         }
 217                         else
 218                         {
 219                                 profile = new_profile;
 220                         }
 221
 222                         // Apply to properties
 223                         apply_profile_properties( self, profile, properties );
 224                         mlt_profile_close( new_profile );
 225                 }
 226         }
 227         else if ( !strcmp( name, "frame_rate_num" ) )
 228         {
 229                 mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
 230                 mlt_profile profile = mlt_service_profile( MLT_CONSUMER_SERVICE( self ) );
 231                 if ( profile )
 232                 {
 233                         profile->frame_rate_num = mlt_properties_get_int( properties, "frame_rate_num" );
 234                         mlt_properties_set_double( properties, "fps", mlt_profile_fps( profile ) );
 235                 }
 236         }
 237         else if ( !strcmp( name, "frame_rate_den" ) )
 238         {
 239                 mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
 240                 mlt_profile profile = mlt_service_profile( MLT_CONSUMER_SERVICE( self ) );
 241                 if ( profile )
 242                 {
 243                         profile->frame_rate_den = mlt_properties_get_int( properties, "frame_rate_den" );
 244                         mlt_properties_set_double( properties, "fps", mlt_profile_fps( profile ) );
 245                 }
 246         }
 247         else if ( !strcmp( name, "width" ) )
 248         {
 249                 mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
 250                 mlt_profile profile = mlt_service_profile( MLT_CONSUMER_SERVICE( self ) );
 251                 if ( profile )
 252                         profile->width = mlt_properties_get_int( properties, "width" );
 253         }
 254         else if ( !strcmp( name, "height" ) )
 255         {
 256                 mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
 257                 mlt_profile profile = mlt_service_profile( MLT_CONSUMER_SERVICE( self ) );
 258                 if ( profile )
 259                         profile->height = mlt_properties_get_int( properties, "height" );
 260         }
 261         else if ( !strcmp( name, "progressive" ) )
 262         {
 263                 mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
 264                 mlt_profile profile = mlt_service_profile( MLT_CONSUMER_SERVICE( self ) );
 265                 if ( profile )
 266                         profile->progressive = mlt_properties_get_int( properties, "progressive" );
 267         }
 268         else if ( !strcmp( name, "sample_aspect_num" ) )
 269         {
 270                 mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
 271                 mlt_profile profile = mlt_service_profile( MLT_CONSUMER_SERVICE( self ) );
 272                 if ( profile )
 273                 {
 274                         profile->sample_aspect_num = mlt_properties_get_int( properties, "sample_aspect_num" );
 275                         mlt_properties_set_double( properties, "aspect_ratio", mlt_profile_sar( profile )  );
 276                 }
 277         }
 278         else if ( !strcmp( name, "sample_aspect_den" ) )
 279         {
 280                 mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
 281                 mlt_profile profile = mlt_service_profile( MLT_CONSUMER_SERVICE( self ) );
 282                 if ( profile )
 283                 {
 284                         profile->sample_aspect_den = mlt_properties_get_int( properties, "sample_aspect_den" );
 285                         mlt_properties_set_double( properties, "aspect_ratio", mlt_profile_sar( profile )  );
 286                 }
 287         }
 288         else if ( !strcmp( name, "display_aspect_num" ) )
 289         {
 290                 mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
 291                 mlt_profile profile = mlt_service_profile( MLT_CONSUMER_SERVICE( self ) );
 292                 if ( profile )
 293                 {
 294                         profile->display_aspect_num = mlt_properties_get_int( properties, "display_aspect_num" );
 295                         mlt_properties_set_double( properties, "display_ratio", mlt_profile_dar( profile )  );
 296                 }
 297         }
 298         else if ( !strcmp( name, "display_aspect_den" ) )
 299         {
 300                 mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
 301                 mlt_profile profile = mlt_service_profile( MLT_CONSUMER_SERVICE( self ) );
 302                 if ( profile )
 303                 {
 304                         profile->display_aspect_den = mlt_properties_get_int( properties, "display_aspect_den" );
 305                         mlt_properties_set_double( properties, "display_ratio", mlt_profile_dar( profile )  );
 306                 }
 307         }
 308         else if ( !strcmp( name, "colorspace" ) )
 309         {
 310                 mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
 311                 mlt_profile profile = mlt_service_profile( MLT_CONSUMER_SERVICE( self ) );
 312                 if ( profile )
 313                         profile->colorspace = mlt_properties_get_int( properties, "colorspace" );
 314         }
 315 }
 316
 317 /** The transmitter for the consumer-frame-show event
 318  *
 319  * Invokes the listener.
 320  *
 321  * \private \memberof mlt_consumer_s
 322  * \param listener a function pointer that will be invoked
 323  * \param owner the events object that will be passed to \p listener
 324  * \param self a service that will be passed to \p listener
 325  * \param args an array of pointers - the first entry is passed as a frame to \p listener
 326  */
 327
 328 static void mlt_consumer_frame_show( mlt_listener listener, mlt_properties owner, mlt_service self, void **args )
 329 {
 330         if ( listener != NULL )
 331                 listener( owner, self, ( mlt_frame )args[ 0 ] );
 332 }
 333
 334 /** The transmitter for the consumer-frame-render event
 335  *
 336  * Invokes the listener.
 337  *
 338  * \private \memberof mlt_consumer_s
 339  * \param listener a function pointer that will be invoked
 340  * \param owner the events object that will be passed to \p listener
 341  * \param self a service that will be passed to \p listener
 342  * \param args an array of pointers - the first entry is passed as a frame to \p listener
 343  */
 344
 345 static void mlt_consumer_frame_render( mlt_listener listener, mlt_properties owner, mlt_service self, void **args )
 346 {
 347         if ( listener != NULL )
 348                 listener( owner, self, ( mlt_frame )args[ 0 ] );
 349 }
 350
 351 /** A listener on the consumer-frame-show event
 352  *
 353  * Saves the position of the frame shown.
 354  *
 355  * \private \memberof mlt_consumer_s
 356  * \param owner the events object
 357  * \param consumer the consumer on which this event occurred
 358  * \param frame the frame that was shown
 359  */
 360
 361 static void on_consumer_frame_show( mlt_properties owner, mlt_consumer consumer, mlt_frame frame )
 362 {
 363         if ( frame )
 364                 ( ( consumer_private*) consumer->local )->position = mlt_frame_get_position( frame );
 365 }
 366
 367 /** Create a new consumer.
 368  *
 369  * \public \memberof mlt_consumer_s
 370  * \param profile a profile (optional, but recommended)
 371  * \return a new consumer
 372  */
 373
 374 mlt_consumer mlt_consumer_new( mlt_profile profile )
 375 {
 376         // Create the memory for the structure
 377         mlt_consumer self = malloc( sizeof( struct mlt_consumer_s ) );
 378
 379         // Initialise it
 380         if ( self != NULL && mlt_consumer_init( self, NULL, profile ) == 0 )
 381         {
 382                 // Return it
 383                 return self;
 384         }
 385         else
 386         {
 387                 free(self);
 388                 return NULL;
 389         }
 390 }
 391
 392 /** Get the parent service object.
 393  *
 394  * \public \memberof mlt_consumer_s
 395  * \param self a consumer
 396  * \return the parent service class
 397  * \see MLT_CONSUMER_SERVICE
 398  */
 399
 400 mlt_service mlt_consumer_service( mlt_consumer self )
 401 {
 402         return self != NULL ? &self->parent : NULL;
 403 }
 404
 405 /** Get the consumer properties.
 406  *
 407  * \public \memberof mlt_consumer_s
 408  * \param self a consumer
 409  * \return the consumer's properties list
 410  * \see MLT_CONSUMER_PROPERTIES
 411  */
 412
 413 mlt_properties mlt_consumer_properties( mlt_consumer self )
 414 {
 415         return self != NULL ? MLT_SERVICE_PROPERTIES( &self->parent ) : NULL;
 416 }
 417
 418 /** Connect the consumer to the producer.
 419  *
 420  * \public \memberof mlt_consumer_s
 421  * \param self a consumer
 422  * \param producer a producer
 423  * \return > 0 warning, == 0 success, < 0 serious error,
 424  *         1 = this service does not accept input,
 425  *         2 = the producer is invalid,
 426  *         3 = the producer is already registered with this consumer
 427  */
 428
 429 int mlt_consumer_connect( mlt_consumer self, mlt_service producer )
 430 {
 431         return mlt_service_connect_producer( &self->parent, producer, 0 );
 432 }
 433
 434 /** Start the consumer.
 435  *
 436  * \public \memberof mlt_consumer_s
 437  * \param self a consumer
 438  * \return true if there was an error
 439  */
 440
 441 int mlt_consumer_start( mlt_consumer self )
 442 {
 443         if ( !mlt_consumer_is_stopped( self ) )
 444                 return 0;
 445
 446         consumer_private *priv = self->local;
 447
 448         // Stop listening to the property-changed event
 449         mlt_event_block( priv->event_listener );
 450
 451         // Get the properies
 452         mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
 453
 454         // Determine if there's a test card producer
 455         char *test_card = mlt_properties_get( properties, "test_card" );
 456
 457         // Just to make sure nothing is hanging around...
 458         pthread_mutex_lock( &priv->put_mutex );
 459         priv->put = NULL;
 460         priv->put_active = 1;
 461         pthread_mutex_unlock( &priv->put_mutex );
 462
 463         // Deal with it now.
 464         if ( test_card != NULL )
 465         {
 466                 if ( mlt_properties_get_data( properties, "test_card_producer", NULL ) == NULL )
 467                 {
 468                         // Create a test card producer
 469                         mlt_profile profile = mlt_service_profile( MLT_CONSUMER_SERVICE( self ) );
 470                         mlt_producer producer = mlt_factory_producer( profile, NULL, test_card );
 471
 472                         // Do we have a producer
 473                         if ( producer != NULL )
 474                         {
 475                                 // Test card should loop I guess...
 476                                 mlt_properties_set( MLT_PRODUCER_PROPERTIES( producer ), "eof", "loop" );
 477                                 //mlt_producer_set_speed( producer, 0 );
 478                                 //mlt_producer_set_in_and_out( producer, 0, 0 );
 479
 480                                 // Set the test card on the consumer
 481                                 mlt_properties_set_data( properties, "test_card_producer", producer, 0, ( mlt_destructor )mlt_producer_close, NULL );
 482                         }
 483                 }
 484         }
 485         else
 486         {
 487                 // Allow the hash table to speed things up
 488                 mlt_properties_set_data( properties, "test_card_producer", NULL, 0, NULL, NULL );
 489         }
 490
 491         // The profile could have changed between a stop and a restart.
 492         apply_profile_properties( self, mlt_service_profile( MLT_CONSUMER_SERVICE(self) ), properties );
 493
 494         // Set the frame duration in microseconds for the frame-dropping heuristic
 495         int frame_rate_num = mlt_properties_get_int( properties, "frame_rate_num" );
 496         int frame_rate_den = mlt_properties_get_int( properties, "frame_rate_den" );
 497         int frame_duration = 0;
 498
 499         if ( frame_rate_num && frame_rate_den )
 500         {
 501                 frame_duration = 1000000 / frame_rate_num * frame_rate_den;
 502         }
 503
 504         mlt_properties_set_int( properties, "frame_duration", frame_duration );
 505
 506         // Check and run an ante command
 507         if ( mlt_properties_get( properties, "ante" ) )
 508                 if ( system( mlt_properties_get( properties, "ante" ) ) == -1 )
 509                         mlt_log( MLT_CONSUMER_SERVICE( self ), MLT_LOG_ERROR, "system(%s) failed!\n", mlt_properties_get( properties, "ante" ) );
 510
 511         // Set the real_time preference
 512         priv->real_time = mlt_properties_get_int( properties, "real_time" );
 513
 514         // For worker threads implementation, buffer must be at least # threads
 515         if ( abs( priv->real_time ) > 1 && mlt_properties_get_int( properties, "buffer" ) <= abs( priv->real_time ) )
 516                 mlt_properties_set_int( properties, "_buffer", abs( priv->real_time ) + 1 );
 517
 518         // Get the image format to use for rendering threads
 519         const char* format = mlt_properties_get( properties, "mlt_image_format" );
 520         if ( format )
 521         {
 522                 if ( !strcmp( format, "rgb24" ) )
 523                         priv->format = mlt_image_rgb24;
 524                 else if ( !strcmp( format, "rgb24a" ) )
 525                         priv->format = mlt_image_rgb24a;
 526                 else if ( !strcmp( format, "yuv420p" ) )
 527                         priv->format = mlt_image_yuv420p;
 528                 else if ( !strcmp( format, "none" ) )
 529                         priv->format = mlt_image_none;
 530                 else if ( !strcmp( format, "glsl" ) )
 531                         priv->format = mlt_image_glsl_texture;
 532                 else
 533                         priv->format = mlt_image_yuv422;
 534         }
 535
 536         // Start the service
 537         if ( self->start != NULL )
 538                 return self->start( self );
 539
 540         return 0;
 541 }
 542
 543 /** An alternative method to feed frames into the consumer.
 544  *
 545  * Only valid if the consumer itself is not connected.
 546  *
 547  * \public \memberof mlt_consumer_s
 548  * \param self a consumer
 549  * \param frame a frame
 550  * \return true (ignore self for now)
 551  */
 552
 553 int mlt_consumer_put_frame( mlt_consumer self, mlt_frame frame )
 554 {
 555         int error = 1;
 556
 557         // Get the service assoicated to the consumer
 558         mlt_service service = MLT_CONSUMER_SERVICE( self );
 559
 560         if ( mlt_service_producer( service ) == NULL )
 561         {
 562                 struct timeval now;
 563                 struct timespec tm;
 564                 consumer_private *priv = self->local;
 565
 566                 pthread_mutex_lock( &priv->put_mutex );
 567                 while ( priv->put_active && priv->put != NULL )
 568                 {
 569                         gettimeofday( &now, NULL );
 570                         tm.tv_sec = now.tv_sec + 1;
 571                         tm.tv_nsec = now.tv_usec * 1000;
 572                         pthread_cond_timedwait( &priv->put_cond, &priv->put_mutex, &tm );
 573                 }
 574                 if ( priv->put_active && priv->put == NULL )
 575                         priv->put = frame;
 576                 else
 577                         mlt_frame_close( frame );
 578                 pthread_cond_broadcast( &priv->put_cond );
 579                 pthread_mutex_unlock( &priv->put_mutex );
 580         }
 581         else
 582         {
 583                 mlt_frame_close( frame );
 584         }
 585
 586         return error;
 587 }
 588
 589 /** Protected method for consumer to get frames from connected service
 590  *
 591  * \public \memberof mlt_consumer_s
 592  * \param self a consumer
 593  * \return a frame
 594  */
 595
 596 mlt_frame mlt_consumer_get_frame( mlt_consumer self )
 597 {
 598         // Frame to return
 599         mlt_frame frame = NULL;
 600
 601         // Get the service assoicated to the consumer
 602         mlt_service service = MLT_CONSUMER_SERVICE( self );
 603
 604         // Get the consumer properties
 605         mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
 606
 607         // Get the frame
 608         if ( mlt_service_producer( service ) == NULL && mlt_properties_get_int( properties, "put_mode" ) )
 609         {
 610                 struct timeval now;
 611                 struct timespec tm;
 612                 consumer_private *priv = self->local;
 613
 614                 pthread_mutex_lock( &priv->put_mutex );
 615                 while ( priv->put_active && priv->put == NULL )
 616                 {
 617                         gettimeofday( &now, NULL );
 618                         tm.tv_sec = now.tv_sec + 1;
 619                         tm.tv_nsec = now.tv_usec * 1000;
 620                         pthread_cond_timedwait( &priv->put_cond, &priv->put_mutex, &tm );
 621                 }
 622                 frame = priv->put;
 623                 priv->put = NULL;
 624                 pthread_cond_broadcast( &priv->put_cond );
 625                 pthread_mutex_unlock( &priv->put_mutex );
 626                 if ( frame != NULL )
 627                         mlt_service_apply_filters( service, frame, 0 );
 628         }
 629         else if ( mlt_service_producer( service ) != NULL )
 630         {
 631                 mlt_service_get_frame( service, &frame, 0 );
 632         }
 633         else
 634         {
 635                 frame = mlt_frame_init( service );
 636         }
 637
 638         if ( frame != NULL )
 639         {
 640                 // Get the frame properties
 641                 mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame );
 642
 643                 // Get the test card producer
 644                 mlt_producer test_card = mlt_properties_get_data( properties, "test_card_producer", NULL );
 645
 646                 // Attach the test frame producer to it.
 647                 if ( test_card != NULL )
 648                         mlt_properties_set_data( frame_properties, "test_card_producer", test_card, 0, NULL, NULL );
 649
 650                 // Pass along the interpolation and deinterlace options
 651                 // TODO: get rid of consumer_deinterlace and use profile.progressive
 652                 mlt_properties_set( frame_properties, "rescale.interp", mlt_properties_get( properties, "rescale" ) );
 653                 mlt_properties_set_int( frame_properties, "consumer_deinterlace", mlt_properties_get_int( properties, "progressive" ) | mlt_properties_get_int( properties, "deinterlace" ) );
 654                 mlt_properties_set( frame_properties, "deinterlace_method", mlt_properties_get( properties, "deinterlace_method" ) );
 655                 mlt_properties_set_int( frame_properties, "consumer_tff", mlt_properties_get_int( properties, "top_field_first" ) );
 656         }
 657
 658         // Return the frame
 659         return frame;
 660 }
 661
 662 /** Compute the time difference between now and a time value.
 663  *
 664  * \private \memberof mlt_consumer_s
 665  * \param time1 a time value to be compared against now
 666  * \return the difference in microseconds
 667  */
 668
 669 static inline long time_difference( struct timeval *time1 )
 670 {
 671         struct timeval time2;
 672         time2.tv_sec = time1->tv_sec;
 673         time2.tv_usec = time1->tv_usec;
 674         gettimeofday( time1, NULL );
 675         return time1->tv_sec * 1000000 + time1->tv_usec - time2.tv_sec * 1000000 - time2.tv_usec;
 676 }
 677
 678 /** The thread procedure for asynchronously pulling frames through the service
 679  * network connected to a consumer.
 680  *
 681  * \private \memberof mlt_consumer_s
 682  * \param arg a consumer
 683  */
 684
 685 static void *consumer_read_ahead_thread( void *arg )
 686 {
 687         // The argument is the consumer
 688         mlt_consumer self = arg;
 689         consumer_private *priv = self->local;
 690
 691         // Get the properties of the consumer
 692         mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
 693
 694         // Get the width and height
 695         int width = mlt_properties_get_int( properties, "width" );
 696         int height = mlt_properties_get_int( properties, "height" );
 697
 698         // See if video is turned off
 699         int video_off = mlt_properties_get_int( properties, "video_off" );
 700         int preview_off = mlt_properties_get_int( properties, "preview_off" );
 701         int preview_format = mlt_properties_get_int( properties, "preview_format" );
 702
 703         // Get the audio settings
 704         mlt_audio_format afmt = mlt_audio_s16;
 705         const char *format = mlt_properties_get( properties, "mlt_audio_format" );
 706         if ( format )
 707         {
 708                 if ( !strcmp( format, "none" ) )
 709                         afmt = mlt_audio_none;
 710                 else if ( !strcmp( format, "s32" ) )
 711                         afmt = mlt_audio_s32;
 712                 else if ( !strcmp( format, "s32le" ) )
 713                         afmt = mlt_audio_s32le;
 714                 else if ( !strcmp( format, "float" ) )
 715                         afmt = mlt_audio_float;
 716                 else if ( !strcmp( format, "f32le" ) )
 717                         afmt = mlt_audio_f32le;
 718                 else if ( !strcmp( format, "u8" ) )
 719                         afmt = mlt_audio_u8;
 720         }
 721         int counter = 0;
 722         double fps = mlt_properties_get_double( properties, "fps" );
 723         int channels = mlt_properties_get_int( properties, "channels" );
 724         int frequency = mlt_properties_get_int( properties, "frequency" );
 725         int samples = 0;
 726         void *audio = NULL;
 727
 728         // See if audio is turned off
 729         int audio_off = mlt_properties_get_int( properties, "audio_off" );
 730
 731         // Get the maximum size of the buffer
 732         int buffer = mlt_properties_get_int( properties, "buffer" ) + 1;
 733
 734         // General frame variable
 735         mlt_frame frame = NULL;
 736         uint8_t *image = NULL;
 737
 738         // Time structures
 739         struct timeval ante;
 740
 741         // Average time for get_frame and get_image
 742         int count = 0;
 743         int skipped = 0;
 744         int64_t time_process = 0;
 745         int skip_next = 0;
 746         mlt_position pos = 0;
 747         mlt_position start_pos = 0;
 748         mlt_position last_pos = 0;
 749         int frame_duration = mlt_properties_get_int( properties, "frame_duration" );
 750         int drop_max = mlt_properties_get_int( properties, "drop_max" );
 751
 752         if ( preview_off && preview_format != 0 )
 753                 priv->format = preview_format;
 754
 755         mlt_events_fire( properties, "consumer-thread-started", NULL );
 756
 757         // Get the first frame
 758         frame = mlt_consumer_get_frame( self );
 759
 760         if ( frame )
 761         {
 762                 // Get the image of the first frame
 763                 if ( !video_off )
 764                 {
 765                         mlt_events_fire( MLT_CONSUMER_PROPERTIES( self ), "consumer-frame-render", frame, NULL );
 766                         mlt_frame_get_image( frame, &image, &priv->format, &width, &height, 0 );
 767                 }
 768
 769                 if ( !audio_off )
 770                 {
 771                         samples = mlt_sample_calculator( fps, frequency, counter++ );
 772                         mlt_frame_get_audio( frame, &audio, &afmt, &frequency, &channels, &samples );
 773                 }
 774
 775                 // Mark as rendered
 776                 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "rendered", 1 );
 777                 last_pos = start_pos = pos = mlt_frame_get_position( frame );
 778         }
 779
 780         // Get the starting time (can ignore the times above)
 781         gettimeofday( &ante, NULL );
 782
 783         // Continue to read ahead
 784         while ( priv->ahead )
 785         {
 786                 // Put the current frame into the queue
 787                 pthread_mutex_lock( &priv->queue_mutex );
 788                 while( priv->ahead && mlt_deque_count( priv->queue ) >= buffer )
 789                         pthread_cond_wait( &priv->queue_cond, &priv->queue_mutex );
 790                 if ( priv->is_purge )
 791                 {
 792                         mlt_frame_close( frame );
 793                         priv->is_purge = 0;
 794                 }
 795                 else
 796                 {
 797                         mlt_deque_push_back( priv->queue, frame );
 798                 }
 799                 pthread_cond_broadcast( &priv->queue_cond );
 800                 pthread_mutex_unlock( &priv->queue_mutex );
 801
 802                 // Get the next frame
 803                 frame = mlt_consumer_get_frame( self );
 804
 805                 // If there's no frame, we're probably stopped...
 806                 if ( frame == NULL )
 807                         continue;
 808                 pos = mlt_frame_get_position( frame );
 809
 810                 // WebVfx uses this to setup a consumer-stopping event handler.
 811                 mlt_properties_set_data( MLT_FRAME_PROPERTIES( frame ), "consumer", self, 0, NULL, NULL );
 812
 813                 // Increment the counter used for averaging processing cost
 814                 count ++;
 815
 816                 // All non-normal playback frames should be shown
 817                 if ( mlt_properties_get_int( MLT_FRAME_PROPERTIES( frame ), "_speed" ) != 1 )
 818                 {
 819 #ifdef DEINTERLACE_ON_NOT_NORMAL_SPEED
 820                         mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "consumer_deinterlace", 1 );
 821 #endif
 822                         // Indicate seeking or trick-play
 823                         start_pos = pos;
 824                 }
 825
 826                 // If skip flag not set or frame-dropping disabled
 827                 if ( !skip_next || priv->real_time == -1 )
 828                 {
 829                         if ( !video_off )
 830                         {
 831                                 // Reset width/height - could have been changed by previous mlt_frame_get_image
 832                                 width = mlt_properties_get_int( properties, "width" );
 833                                 height = mlt_properties_get_int( properties, "height" );
 834
 835                                 // Get the image
 836                                 mlt_events_fire( MLT_CONSUMER_PROPERTIES( self ), "consumer-frame-render", frame, NULL );
 837                                 mlt_frame_get_image( frame, &image, &priv->format, &width, &height, 0 );
 838                         }
 839
 840                         // Indicate the rendered image is available.
 841                         mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "rendered", 1 );
 842
 843                         // Reset consecutively-skipped counter
 844                         skipped = 0;
 845                 }
 846                 else // Skip image processing
 847                 {
 848                         // Increment the number of consecutively-skipped frames
 849                         skipped++;
 850
 851                         // If too many (1 sec) consecutively-skipped frames
 852                         if ( skipped > drop_max )
 853                         {
 854                                 // Reset cost tracker
 855                                 time_process = 0;
 856                                 count = 1;
 857                                 mlt_log_verbose( self, "too many frames dropped - forcing next frame\n" );
 858                         }
 859                 }
 860
 861                 // Always process audio
 862                 if ( !audio_off )
 863                 {
 864                         samples = mlt_sample_calculator( fps, frequency, counter++ );
 865                         mlt_frame_get_audio( frame, &audio, &afmt, &frequency, &channels, &samples );
 866                 }
 867
 868                 // Get the time to process this frame
 869                 int64_t time_current = time_difference( &ante );
 870
 871                 // If the current time is not suddenly some large amount
 872                 if ( time_current < time_process / count * 20 || !time_process || count < 5 )
 873                 {
 874                         // Accumulate the cost for processing this frame
 875                         time_process += time_current;
 876                 }
 877                 else
 878                 {
 879                         mlt_log_debug( self, "current %"PRId64" threshold %"PRId64" count %d\n",
 880                                 time_current, (int64_t) (time_process / count * 20), count );
 881                         // Ignore the cost of this frame's time
 882                         count--;
 883                 }
 884
 885                 // Determine if we started, resumed, or seeked
 886                 if ( pos != last_pos + 1 )
 887                         start_pos = pos;
 888                 last_pos = pos;
 889
 890                 // Do not skip the first 20% of buffer at start, resume, or seek
 891                 if ( pos - start_pos <= buffer / 5 + 1 )
 892                 {
 893                         // Reset cost tracker
 894                         time_process = 0;
 895                         count = 1;
 896                 }
 897
 898                 // Reset skip flag
 899                 skip_next = 0;
 900
 901                 // Only consider skipping if the buffer level is low (or really small)
 902                 if ( mlt_deque_count( priv->queue ) <= buffer / 5 + 1 )
 903                 {
 904                         // Skip next frame if average cost exceeds frame duration.
 905                         if ( time_process / count > frame_duration )
 906                                 skip_next = 1;
 907                         if ( skip_next )
 908                                 mlt_log_debug( self, "avg usec %"PRId64" (%"PRId64"/%d) duration %d\n",
 909                                         time_process/count, time_process, count, frame_duration);
 910                 }
 911         }
 912
 913         // Remove the last frame
 914         mlt_frame_close( frame );
 915
 916         return NULL;
 917 }
 918
 919 /** Locate the first unprocessed frame in the queue.
 920  *
 921  * When playing with realtime behavior, we do not use the true head, but
 922  * rather an adjusted process_head. The process_head is adjusted based on
 923  * the rate of frame-dropping or recovery from frame-dropping. The idea is
 924  * that as the level of frame-dropping increases to move the process_head
 925  * closer to the tail because the frames are not completing processing prior
 926  * to their playout! Then, as frames are not dropped the process_head moves
 927  * back closer to the head of the queue so that worker threads can work
 928  * ahead of the playout point (queue head).
 929  *
 930  * \private \memberof mlt_consumer_s
 931  * \param self a consumer
 932  * \return an index into the queue
 933  */
 934
 935 static inline int first_unprocessed_frame( mlt_consumer self )
 936 {
 937         consumer_private *priv = self->local;
 938         int index = priv->real_time <= 0 ? 0 : priv->process_head;
 939         while ( index < mlt_deque_count( priv->queue ) && MLT_FRAME( mlt_deque_peek( priv->queue, index ) )->is_processing )
 940                 index++;
 941         return index;
 942 }
 943
 944 /** The worker thread procedure for parallel processing frames.
 945  *
 946  * \private \memberof mlt_consumer_s
 947  * \param arg a consumer
 948  */
 949
 950 static void *consumer_worker_thread( void *arg )
 951 {
 952         // The argument is the consumer
 953         mlt_consumer self = arg;
 954         consumer_private *priv = self->local;
 955
 956         // Get the properties of the consumer
 957         mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
 958
 959         // Get the width and height
 960         int width = mlt_properties_get_int( properties, "width" );
 961         int height = mlt_properties_get_int( properties, "height" );
 962         mlt_image_format format = priv->format;
 963
 964         // See if video is turned off
 965         int video_off = mlt_properties_get_int( properties, "video_off" );
 966         int preview_off = mlt_properties_get_int( properties, "preview_off" );
 967         int preview_format = mlt_properties_get_int( properties, "preview_format" );
 968
 969         // General frame variable
 970         mlt_frame frame = NULL;
 971         uint8_t *image = NULL;
 972
 973         if ( preview_off && preview_format != 0 )
 974                 format = preview_format;
 975
 976         mlt_events_fire( properties, "consumer-thread-started", NULL );
 977
 978         // Continue to read ahead
 979         while ( priv->ahead )
 980         {
 981                 // Get the next unprocessed frame from the work queue
 982                 pthread_mutex_lock( &priv->queue_mutex );
 983                 int index = first_unprocessed_frame( self );
 984                 while ( priv->ahead && index >= mlt_deque_count( priv->queue ) )
 985                 {
 986                         mlt_log_debug( MLT_CONSUMER_SERVICE(self), "waiting in worker index = %d queue count = %d\n",
 987                                 index, mlt_deque_count( priv->queue ) );
 988                         pthread_cond_wait( &priv->queue_cond, &priv->queue_mutex );
 989                         index = first_unprocessed_frame( self );
 990                 }
 991
 992                 // Mark the frame for processing
 993                 frame = mlt_deque_peek( priv->queue, index );
 994                 if ( frame )
 995                 {
 996                         mlt_log_debug( MLT_CONSUMER_SERVICE(self), "worker processing index = %d frame " MLT_POSITION_FMT " queue count = %d\n",
 997                                 index, mlt_frame_get_position(frame), mlt_deque_count( priv->queue ) );
 998                         frame->is_processing = 1;
 999                         mlt_properties_inc_ref( MLT_FRAME_PROPERTIES( frame ) );
1000                 }
1001                 pthread_mutex_unlock( &priv->queue_mutex );
1002
1003                 // If there's no frame, we're probably stopped...
1004                 if ( frame == NULL )
1005                         continue;
1006
1007                 // WebVfx uses this to setup a consumer-stopping event handler.
1008                 mlt_properties_set_data( MLT_FRAME_PROPERTIES( frame ), "consumer", self, 0, NULL, NULL );
1009
1010 #ifdef DEINTERLACE_ON_NOT_NORMAL_SPEED
1011                 // All non normal playback frames should be shown
1012                 if ( mlt_properties_get_int( MLT_FRAME_PROPERTIES( frame ), "_speed" ) != 1 )
1013                         mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "consumer_deinterlace", 1 );
1014 #endif
1015
1016                 // Get the image
1017                 if ( !video_off )
1018                 {
1019                         // Fetch width/height again
1020                         width = mlt_properties_get_int( properties, "width" );
1021                         height = mlt_properties_get_int( properties, "height" );
1022                         mlt_events_fire( MLT_CONSUMER_PROPERTIES( self ), "consumer-frame-render", frame, NULL );
1023                         mlt_frame_get_image( frame, &image, &format, &width, &height, 0 );
1024                 }
1025                 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "rendered", 1 );
1026                 mlt_frame_close( frame );
1027
1028                 // Tell a waiting thread (non-realtime main consumer thread) that we are done.
1029                 pthread_mutex_lock( &priv->done_mutex );
1030                 pthread_cond_broadcast( &priv->done_cond );
1031                 pthread_mutex_unlock( &priv->done_mutex );
1032         }
1033
1034         return NULL;
1035 }
1036
1037 /** Start the read/render thread.
1038  *
1039  * \private \memberof mlt_consumer_s
1040  * \param self a consumer
1041  */
1042
1043 static void consumer_read_ahead_start( mlt_consumer self )
1044 {
1045         consumer_private *priv = self->local;
1046
1047         // We're running now
1048         priv->ahead = 1;
1049
1050         // Create the frame queue
1051         priv->queue = mlt_deque_init( );
1052
1053         // Create the queue mutex
1054         pthread_mutex_init( &priv->queue_mutex, NULL );
1055
1056         // Create the condition
1057         pthread_cond_init( &priv->queue_cond, NULL );
1058
1059         // Create the read ahead
1060         if ( mlt_properties_get( MLT_CONSUMER_PROPERTIES( self ), "priority" ) )
1061         {
1062                 struct sched_param priority;
1063                 priority.sched_priority = mlt_properties_get_int( MLT_CONSUMER_PROPERTIES( self ), "priority" );
1064                 pthread_attr_t thread_attributes;
1065                 pthread_attr_init( &thread_attributes );
1066                 pthread_attr_setschedpolicy( &thread_attributes, SCHED_OTHER );
1067                 pthread_attr_setschedparam( &thread_attributes, &priority );
1068                 pthread_attr_setinheritsched( &thread_attributes, PTHREAD_EXPLICIT_SCHED );
1069                 pthread_attr_setscope( &thread_attributes, PTHREAD_SCOPE_SYSTEM );
1070                 if ( pthread_create( &priv->ahead_thread, &thread_attributes, consumer_read_ahead_thread, self ) < 0 )
1071                         pthread_create( &priv->ahead_thread, NULL, consumer_read_ahead_thread, self );
1072                 pthread_attr_destroy( &thread_attributes );
1073         }
1074         else
1075         {
1076                 pthread_create( &priv->ahead_thread, NULL, consumer_read_ahead_thread, self );
1077         }
1078         priv->started = 1;
1079 }
1080
1081 /** Start the worker threads.
1082  *
1083  * \private \memberof mlt_consumer_s
1084  * \param self a consumer
1085  */
1086
1087 static void consumer_work_start( mlt_consumer self )
1088 {
1089         consumer_private *priv = self->local;
1090         int n = abs( priv->real_time );
1091         pthread_t *thread = calloc( 1, sizeof( pthread_t ) * n );
1092
1093         // We're running now
1094         priv->ahead = 1;
1095         priv->threads = thread;
1096
1097         // These keep track of the accelleration of frame dropping or recovery.
1098         priv->consecutive_dropped = 0;
1099         priv->consecutive_rendered = 0;
1100
1101         // This is the position in the queue from which to look for a frame to process.
1102         // If we always start from the head, then we may likely not complete processing
1103         // before the frame is played out.
1104         priv->process_head = 0;
1105
1106         // Create the queues
1107         priv->queue = mlt_deque_init();
1108         priv->worker_threads = mlt_deque_init();
1109
1110         // Create the mutexes
1111         pthread_mutex_init( &priv->queue_mutex, NULL );
1112         pthread_mutex_init( &priv->done_mutex, NULL );
1113
1114         // Create the conditions
1115         pthread_cond_init( &priv->queue_cond, NULL );
1116         pthread_cond_init( &priv->done_cond, NULL );
1117
1118         // Create the read ahead
1119         if ( mlt_properties_get( MLT_CONSUMER_PROPERTIES( self ), "priority" ) )
1120         {
1121
1122                 struct sched_param priority;
1123                 pthread_attr_t thread_attributes;
1124
1125                 priority.sched_priority = mlt_properties_get_int( MLT_CONSUMER_PROPERTIES( self ), "priority" );
1126                 pthread_attr_init( &thread_attributes );
1127                 pthread_attr_setschedpolicy( &thread_attributes, SCHED_OTHER );
1128                 pthread_attr_setschedparam( &thread_attributes, &priority );
1129                 pthread_attr_setinheritsched( &thread_attributes, PTHREAD_EXPLICIT_SCHED );
1130                 pthread_attr_setscope( &thread_attributes, PTHREAD_SCOPE_SYSTEM );
1131
1132                 while ( n-- )
1133                 {
1134                         if ( pthread_create( thread, &thread_attributes, consumer_worker_thread, self ) < 0 )
1135                                 if ( pthread_create( thread, NULL, consumer_worker_thread, self ) == 0 )
1136                                         mlt_deque_push_back( priv->worker_threads, thread );
1137                         thread++;
1138                 }
1139                 pthread_attr_destroy( &thread_attributes );
1140         }
1141
1142         else
1143         {
1144                 while ( n-- )
1145                 {
1146                         if ( pthread_create( thread, NULL, consumer_worker_thread, self ) == 0 )
1147                                 mlt_deque_push_back( priv->worker_threads, thread );
1148                         thread++;
1149                 }
1150         }
1151         priv->started = 1;
1152 }
1153
1154 /** Stop the read/render thread.
1155  *
1156  * \private \memberof mlt_consumer_s
1157  * \param self a consumer
1158  */
1159
1160 static void consumer_read_ahead_stop( mlt_consumer self )
1161 {
1162         consumer_private *priv = self->local;
1163
1164         // Make sure we're running
1165 // TODO improve support for atomic ops in general (see libavutil/atomic.h)
1166 #ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4
1167         if ( __sync_val_compare_and_swap( &priv->started, 1, 0 ) )
1168         {
1169 #else
1170         if ( priv->started )
1171         {
1172                 priv->started = 0;
1173 #endif
1174                 // Inform thread to stop
1175                 priv->ahead = 0;
1176                 mlt_events_fire( MLT_CONSUMER_PROPERTIES(self), "consumer-stopping", NULL );
1177
1178                 // Broadcast to the condition in case it's waiting
1179                 pthread_mutex_lock( &priv->queue_mutex );
1180                 pthread_cond_broadcast( &priv->queue_cond );
1181                 pthread_mutex_unlock( &priv->queue_mutex );
1182
1183                 // Broadcast to the put condition in case it's waiting
1184                 pthread_mutex_lock( &priv->put_mutex );
1185                 pthread_cond_broadcast( &priv->put_cond );
1186                 pthread_mutex_unlock( &priv->put_mutex );
1187
1188                 // Join the thread
1189                 pthread_join( priv->ahead_thread, NULL );
1190
1191                 // Destroy the frame queue mutex
1192                 pthread_mutex_destroy( &priv->queue_mutex );
1193
1194                 // Destroy the condition
1195                 pthread_cond_destroy( &priv->queue_cond );
1196
1197                 // Wipe the queue
1198                 while ( mlt_deque_count( priv->queue ) )
1199                         mlt_frame_close( mlt_deque_pop_back( priv->queue ) );
1200
1201                 // Close the queue
1202                 mlt_deque_close( priv->queue );
1203
1204                 mlt_events_fire( MLT_CONSUMER_PROPERTIES(self), "consumer-thread-stopped", NULL );
1205         }
1206 }
1207
1208 /** Stop the worker threads.
1209  *
1210  * \private \memberof mlt_consumer_s
1211  * \param self a consumer
1212  */
1213
1214 static void consumer_work_stop( mlt_consumer self )
1215 {
1216         consumer_private *priv = self->local;
1217
1218         // Make sure we're running
1219 #ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4
1220         if ( __sync_val_compare_and_swap( &priv->started, 1, 0 ) )
1221         {
1222 #else
1223         if ( priv->started )
1224         {
1225                 priv->started = 0;
1226 #endif
1227                 // Inform thread to stop
1228                 priv->ahead = 0;
1229                 mlt_events_fire( MLT_CONSUMER_PROPERTIES(self), "consumer-stopping", NULL );
1230
1231                 // Broadcast to the queue condition in case it's waiting
1232                 pthread_mutex_lock( &priv->queue_mutex );
1233                 pthread_cond_broadcast( &priv->queue_cond );
1234                 pthread_mutex_unlock( &priv->queue_mutex );
1235
1236                 // Broadcast to the put condition in case it's waiting
1237                 pthread_mutex_lock( &priv->put_mutex );
1238                 pthread_cond_broadcast( &priv->put_cond );
1239                 pthread_mutex_unlock( &priv->put_mutex );
1240
1241                 // Broadcast to the done condition in case it's waiting
1242                 pthread_mutex_lock( &priv->done_mutex );
1243                 pthread_cond_broadcast( &priv->done_cond );
1244                 pthread_mutex_unlock( &priv->done_mutex );
1245
1246                 // Join the threads
1247                 pthread_t *thread;
1248                 while ( ( thread = mlt_deque_pop_back( priv->worker_threads ) ) )
1249                         pthread_join( *thread, NULL );
1250
1251                 // Deallocate the array of threads
1252                 if ( priv->threads )
1253                         free( priv->threads );
1254
1255                 // Destroy the mutexes
1256                 pthread_mutex_destroy( &priv->queue_mutex );
1257                 pthread_mutex_destroy( &priv->done_mutex );
1258
1259                 // Destroy the conditions
1260                 pthread_cond_destroy( &priv->queue_cond );
1261                 pthread_cond_destroy( &priv->done_cond );
1262
1263                 // Wipe the queues
1264                 while ( mlt_deque_count( priv->queue ) )
1265                         mlt_frame_close( mlt_deque_pop_back( priv->queue ) );
1266
1267                 // Close the queues
1268                 mlt_deque_close( priv->queue );
1269                 mlt_deque_close( priv->worker_threads );
1270
1271                 mlt_events_fire( MLT_CONSUMER_PROPERTIES(self), "consumer-thread-stopped", NULL );
1272         }
1273 }
1274
1275 /** Flush the read/render thread's buffer.
1276  *
1277  * \public \memberof mlt_consumer_s
1278  * \param self a consumer
1279  */
1280
1281 void mlt_consumer_purge( mlt_consumer self )
1282 {
1283         if ( self )
1284         {
1285                 consumer_private *priv = self->local;
1286
1287                 pthread_mutex_lock( &priv->put_mutex );
1288                 if ( priv->put ) {
1289                         mlt_frame_close( priv->put );
1290                         priv->put = NULL;
1291                 }
1292                 pthread_cond_broadcast( &priv->put_cond );
1293                 pthread_mutex_unlock( &priv->put_mutex );
1294
1295                 if ( self->purge )
1296                         self->purge( self );
1297
1298                 pthread_mutex_lock( &priv->queue_mutex );
1299                 while ( priv->started && mlt_deque_count( priv->queue ) )
1300                         mlt_frame_close( mlt_deque_pop_back( priv->queue ) );
1301                 pthread_mutex_unlock( &priv->queue_mutex );
1302
1303                 if ( priv->started && priv->real_time )
1304                 {
1305                         priv->is_purge = 1;
1306                         pthread_mutex_lock( &priv->queue_mutex );
1307                         pthread_cond_broadcast( &priv->queue_cond );
1308                         pthread_mutex_unlock( &priv->queue_mutex );
1309                         if ( abs( priv->real_time ) > 1 )
1310                         {
1311                                 pthread_mutex_lock( &priv->done_mutex );
1312                                 pthread_cond_broadcast( &priv->done_cond );
1313                                 pthread_mutex_unlock( &priv->done_mutex );
1314                         }
1315                 }
1316
1317                 pthread_mutex_lock( &priv->put_mutex );
1318                 if ( priv->put ) {
1319                         mlt_frame_close( priv->put );
1320                         priv->put = NULL;
1321                 }
1322                 pthread_cond_broadcast( &priv->put_cond );
1323                 pthread_mutex_unlock( &priv->put_mutex );
1324         }
1325 }
1326
1327 /** Use multiple worker threads and a work queue.
1328  */
1329
1330 static mlt_frame worker_get_frame( mlt_consumer self, mlt_properties properties )
1331 {
1332         // Frame to return
1333         mlt_frame frame = NULL;
1334         consumer_private *priv = self->local;
1335         double fps = mlt_properties_get_double( properties, "fps" );
1336         int threads = abs( priv->real_time );
1337         int buffer = mlt_properties_get_int( properties, "_buffer" );
1338         buffer = buffer > 0 ? buffer : mlt_properties_get_int( properties, "buffer" );
1339         // This is a heuristic to determine a suitable minimum buffer size for the number of threads.
1340         int headroom = 2 + threads * threads;
1341         buffer = buffer < headroom ? headroom : buffer;
1342
1343         // Start worker threads if not already started.
1344         if ( ! priv->ahead )
1345         {
1346                 int prefill = mlt_properties_get_int( properties, "prefill" );
1347                 prefill = prefill > 0 && prefill < buffer ? prefill : buffer;
1348
1349                 consumer_work_start( self );
1350
1351                 // Fill the work queue.
1352                 int i = buffer;
1353                 while ( priv->ahead && i-- )
1354                 {
1355                         frame = mlt_consumer_get_frame( self );
1356                         if ( frame )
1357                         {
1358                                 pthread_mutex_lock( &priv->queue_mutex );
1359                                 mlt_deque_push_back( priv->queue, frame );
1360                                 pthread_cond_signal( &priv->queue_cond );
1361                                 pthread_mutex_unlock( &priv->queue_mutex );
1362                         }
1363                 }
1364
1365                 // Wait for prefill
1366                 while ( priv->ahead && first_unprocessed_frame( self ) < prefill )
1367                 {
1368                         pthread_mutex_lock( &priv->done_mutex );
1369                         pthread_cond_wait( &priv->done_cond, &priv->done_mutex );
1370                         pthread_mutex_unlock( &priv->done_mutex );
1371                 }
1372                 priv->process_head = threads;
1373         }
1374
1375 //      mlt_log_verbose( MLT_CONSUMER_SERVICE(self), "size %d done count %d work count %d process_head %d\n",
1376 //              threads, first_unprocessed_frame( self ), mlt_deque_count( priv->queue ), priv->process_head );
1377
1378         // Feed the work queue
1379         while ( priv->ahead && mlt_deque_count( priv->queue ) < buffer )
1380         {
1381                 frame = mlt_consumer_get_frame( self );
1382                 if ( frame )
1383                 {
1384                         pthread_mutex_lock( &priv->queue_mutex );
1385                         mlt_deque_push_back( priv->queue, frame );
1386                         pthread_cond_signal( &priv->queue_cond );
1387                         pthread_mutex_unlock( &priv->queue_mutex );
1388                 }
1389         }
1390
1391         // Wait if not realtime.
1392         while ( priv->ahead && priv->real_time < 0 && !priv->is_purge &&
1393                 !( mlt_properties_get_int( MLT_FRAME_PROPERTIES( MLT_FRAME( mlt_deque_peek_front( priv->queue ) ) ), "rendered" ) ) )
1394         {
1395                 pthread_mutex_lock( &priv->done_mutex );
1396                 pthread_cond_wait( &priv->done_cond, &priv->done_mutex );
1397                 pthread_mutex_unlock( &priv->done_mutex );
1398         }
1399
1400         // Get the frame from the queue.
1401         pthread_mutex_lock( &priv->queue_mutex );
1402         frame = mlt_deque_pop_front( priv->queue );
1403         pthread_mutex_unlock( &priv->queue_mutex );
1404         if ( ! frame ) {
1405                 priv->is_purge = 0;
1406                 return frame;
1407         }
1408
1409         // Adapt the worker process head to the runtime conditions.
1410         if ( priv->real_time > 0 )
1411         {
1412                 if ( mlt_properties_get_int( MLT_FRAME_PROPERTIES( frame ), "rendered" ) )
1413                 {
1414                         priv->consecutive_dropped = 0;
1415                         if ( priv->process_head > threads && priv->consecutive_rendered >= priv->process_head )
1416                                 priv->process_head--;
1417                         else
1418                                 priv->consecutive_rendered++;
1419                 }
1420                 else
1421                 {
1422                         priv->consecutive_rendered = 0;
1423                         if ( priv->process_head < buffer - threads && priv->consecutive_dropped > threads )
1424                                 priv->process_head++;
1425                         else
1426                                 priv->consecutive_dropped++;
1427                 }
1428 //              mlt_log_verbose( MLT_CONSUMER_SERVICE(self), "dropped %d rendered %d process_head %d\n",
1429 //                      priv->consecutive_dropped, priv->consecutive_rendered, priv->process_head );
1430
1431                 // Check for too many consecutively dropped frames
1432                 if ( priv->consecutive_dropped > mlt_properties_get_int( properties, "drop_max" ) )
1433                 {
1434                         int orig_buffer = mlt_properties_get_int( properties, "buffer" );
1435                         int prefill = mlt_properties_get_int( properties, "prefill" );
1436                         mlt_log_verbose( self, "too many frames dropped - " );
1437
1438                         // If using a default low-latency buffer level (SDL) and below the limit
1439                         if ( ( orig_buffer == 1 || prefill == 1 ) && buffer < (threads + 1) * 10 )
1440                         {
1441                                 // Auto-scale the buffer to compensate
1442                                 mlt_log_verbose( self, "increasing buffer to %d\n", buffer + threads );
1443                                 mlt_properties_set_int( properties, "_buffer", buffer + threads );
1444                                 priv->consecutive_dropped = fps / 2;
1445                         }
1446                         else
1447                         {
1448                                 // Tell the consumer to render it
1449                                 mlt_log_verbose( self, "forcing next frame\n" );
1450                                 mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "rendered", 1 );
1451                                 priv->consecutive_dropped = 0;
1452                         }
1453                 }
1454         }
1455         if ( priv->is_purge ) {
1456                 priv->is_purge = 0;
1457                 mlt_frame_close( frame );
1458                 frame = NULL;
1459         }
1460         return frame;
1461 }
1462
1463 /** Get the next frame from the producer connected to a consumer.
1464  *
1465  * Typically, one uses this instead of \p mlt_consumer_get_frame to make
1466  * the asynchronous/real-time behavior configurable at runtime.
1467  * You should close the frame returned from this when you are done with it.
1468  *
1469  * \public \memberof mlt_consumer_s
1470  * \param self a consumer
1471  * \return a frame
1472  */
1473
1474 mlt_frame mlt_consumer_rt_frame( mlt_consumer self )
1475 {
1476         // Frame to return
1477         mlt_frame frame = NULL;
1478
1479         // Get the properties
1480         mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
1481         consumer_private *priv = self->local;
1482
1483         // Check if the user has requested real time or not
1484         if ( priv->real_time > 1 || priv->real_time < -1 )
1485         {
1486                 // see above
1487                 return worker_get_frame( self, properties );
1488         }
1489         else if ( priv->real_time == 1 || priv->real_time == -1 )
1490         {
1491                 int size = 1;
1492
1493                 // Is the read ahead running?
1494                 if ( priv->ahead == 0 )
1495                 {
1496                         int buffer = mlt_properties_get_int( properties, "buffer" );
1497                         int prefill = mlt_properties_get_int( properties, "prefill" );
1498                         consumer_read_ahead_start( self );
1499                         if ( buffer > 1 )
1500                                 size = prefill > 0 && prefill < buffer ? prefill : buffer;
1501                 }
1502
1503                 // Get frame from queue
1504                 pthread_mutex_lock( &priv->queue_mutex );
1505                 while( priv->ahead && mlt_deque_count( priv->queue ) < size )
1506                         pthread_cond_wait( &priv->queue_cond, &priv->queue_mutex );
1507                 frame = mlt_deque_pop_front( priv->queue );
1508                 pthread_cond_broadcast( &priv->queue_cond );
1509                 pthread_mutex_unlock( &priv->queue_mutex );
1510         }
1511         else // real_time == 0
1512         {
1513                 if ( !priv->ahead )
1514                 {
1515                         priv->ahead = 1;
1516                         mlt_events_fire( properties, "consumer-thread-started", NULL );
1517                 }
1518                 // Get the frame in non real time
1519                 frame = mlt_consumer_get_frame( self );
1520
1521                 // This isn't true, but from the consumers perspective it is
1522                 if ( frame != NULL )
1523                 {
1524                         mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "rendered", 1 );
1525
1526                         // WebVfx uses this to setup a consumer-stopping event handler.
1527                         mlt_properties_set_data( MLT_FRAME_PROPERTIES( frame ), "consumer", self, 0, NULL, NULL );
1528                 }
1529         }
1530
1531         return frame;
1532 }
1533
1534 /** Callback for the implementation to indicate a stopped condition.
1535  *
1536  * \public \memberof mlt_consumer_s
1537  * \param self a consumer
1538  */
1539
1540 void mlt_consumer_stopped( mlt_consumer self )
1541 {
1542         mlt_properties_set_int( MLT_CONSUMER_PROPERTIES( self ), "running", 0 );
1543         mlt_events_fire( MLT_CONSUMER_PROPERTIES( self ), "consumer-stopped", NULL );
1544         mlt_event_unblock( ( ( consumer_private* ) self->local )->event_listener );
1545 }
1546
1547 /** Stop the consumer.
1548  *
1549  * \public \memberof mlt_consumer_s
1550  * \param self a consumer
1551  * \return true if there was an error
1552  */
1553
1554 int mlt_consumer_stop( mlt_consumer self )
1555 {
1556         // Get the properies
1557         mlt_properties properties = MLT_CONSUMER_PROPERTIES( self );
1558         consumer_private *priv = self->local;
1559
1560         // Just in case...
1561         mlt_log( MLT_CONSUMER_SERVICE( self ), MLT_LOG_DEBUG, "stopping put waiting\n" );
1562         pthread_mutex_lock( &priv->put_mutex );
1563         priv->put_active = 0;
1564         pthread_cond_broadcast( &priv->put_cond );
1565         pthread_mutex_unlock( &priv->put_mutex );
1566
1567         // Stop the consumer
1568         mlt_log( MLT_CONSUMER_SERVICE( self ), MLT_LOG_DEBUG, "stopping consumer\n" );
1569
1570         // Cancel the read ahead threads
1571         priv->ahead = 0;
1572         if ( priv->started )
1573         {
1574                 // Unblock the consumer calling mlt_consumer_rt_frame
1575                 pthread_mutex_lock( &priv->queue_mutex );
1576                 pthread_cond_broadcast( &priv->queue_cond );
1577                 pthread_mutex_unlock( &priv->queue_mutex );
1578         }
1579
1580         // Invoke the child callback
1581         if ( self->stop != NULL )
1582                 self->stop( self );
1583
1584         // Check if the user has requested real time or not and stop if necessary
1585         mlt_log( MLT_CONSUMER_SERVICE( self ), MLT_LOG_DEBUG, "stopping read_ahead\n" );
1586         if ( abs( priv->real_time ) == 1 )
1587                 consumer_read_ahead_stop( self );
1588         else if ( abs( priv->real_time ) > 1 )
1589                 consumer_work_stop( self );
1590
1591         // Kill the test card
1592         mlt_properties_set_data( properties, "test_card_producer", NULL, 0, NULL, NULL );
1593
1594         // Check and run a post command
1595         if ( mlt_properties_get( properties, "post" ) )
1596                 if (system( mlt_properties_get( properties, "post" ) ) == -1 )
1597                         mlt_log( MLT_CONSUMER_SERVICE( self ), MLT_LOG_ERROR, "system(%s) failed!\n", mlt_properties_get( properties, "post" ) );
1598
1599         mlt_log( MLT_CONSUMER_SERVICE( self ), MLT_LOG_DEBUG, "stopped\n" );
1600
1601         return 0;
1602 }
1603
1604 /** Determine if the consumer is stopped.
1605  *
1606  * \public \memberof mlt_consumer_s
1607  * \param self a consumer
1608  * \return true if the consumer is stopped
1609  */
1610
1611 int mlt_consumer_is_stopped( mlt_consumer self )
1612 {
1613         // Check if the consumer is stopped
1614         if ( self && self->is_stopped )
1615                 return self->is_stopped( self );
1616
1617         return 0;
1618 }
1619
1620 /** Close and destroy the consumer.
1621  *
1622  * \public \memberof mlt_consumer_s
1623  * \param self a consumer
1624  */
1625
1626 void mlt_consumer_close( mlt_consumer self )
1627 {
1628         if ( self != NULL && mlt_properties_dec_ref( MLT_CONSUMER_PROPERTIES( self ) ) <= 0 )
1629         {
1630                 // Get the childs close function
1631                 void ( *consumer_close )( ) = self->close;
1632
1633                 if ( consumer_close )
1634                 {
1635                         // Just in case...
1636                         //mlt_consumer_stop( self );
1637
1638                         self->close = NULL;
1639                         consumer_close( self );
1640                 }
1641                 else
1642                 {
1643                         consumer_private *priv = self->local;
1644
1645                         // Make sure it only gets called once
1646                         self->parent.close = NULL;
1647
1648                         // Destroy the push mutex and condition
1649                         pthread_mutex_destroy( &priv->put_mutex );
1650                         pthread_cond_destroy( &priv->put_cond );
1651
1652                         mlt_service_close( &self->parent );
1653                         free( priv );
1654                 }
1655         }
1656 }
1657
1658 /** Get the position of the last frame shown.
1659  *
1660  * \public \memberof mlt_consumer_s
1661  * \param consumer a consumer
1662  * \return the position
1663  */
1664
1665 mlt_position mlt_consumer_position( mlt_consumer consumer )
1666 {
1667         return ( ( consumer_private* ) consumer->local )->position;
1668 }
1669