git.sesse.net Git - mlt/blob - src/modules/avformat/producer_avformat.c

   1 /*
   2  * producer_avformat.c -- avformat producer
   3  * Copyright (C) 2003-2004 Ushodaya Enterprises Limited
   4  * Author: Charles Yates <charles.yates@pandora.be>
   5  * Much code borrowed from ffmpeg.c: Copyright (c) 2000-2003 Fabrice Bellard
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2.1 of the License, or (at your option) any later version.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with this library; if not, write to the Free Software
  19  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20  */
  21
  22 // MLT Header files
  23 #include <framework/mlt_producer.h>
  24 #include <framework/mlt_frame.h>
  25 #include <framework/mlt_profile.h>
  26 #include <framework/mlt_log.h>
  27
  28 // ffmpeg Header files
  29 #include <avformat.h>
  30 #include <opt.h>
  31 #ifdef SWSCALE
  32 #  include <swscale.h>
  33 #endif
  34 #if (LIBAVCODEC_VERSION_INT >= ((51<<16)+(71<<8)+0))
  35 #  include "audioconvert.h"
  36 #endif
  37
  38 // System header files
  39 #include <stdlib.h>
  40 #include <string.h>
  41 #include <pthread.h>
  42
  43 #if LIBAVUTIL_VERSION_INT < (50<<16)
  44 #define PIX_FMT_RGB32 PIX_FMT_RGBA32
  45 #define PIX_FMT_YUYV422 PIX_FMT_YUV422
  46 #endif
  47
  48 #define SEEK_ON_OPEN
  49 #define POSITION_INITIAL (-2)
  50 #define POSITION_INVALID (-1)
  51
  52 void avformat_lock( );
  53 void avformat_unlock( );
  54
  55 // Forward references.
  56 static int producer_open( mlt_producer this, mlt_profile profile, char *file );
  57 static int producer_get_frame( mlt_producer this, mlt_frame_ptr frame, int index );
  58
  59 /** Constructor for libavformat.
  60 */
  61
  62 mlt_producer producer_avformat_init( mlt_profile profile, char *file )
  63 {
  64         int error = 0;
  65
  66         // Report information about available demuxers and codecs as YAML Tiny
  67         if ( file && strstr( file, "f-list" ) )
  68         {
  69                 fprintf( stderr, "---\nformats:\n" );
  70                 AVInputFormat *format = NULL;
  71                 while ( ( format = av_iformat_next( format ) ) )
  72                         fprintf( stderr, "  - %s\n", format->name );
  73                 fprintf( stderr, "...\n" );
  74                 error = 1;
  75         }
  76         if ( file && strstr( file, "acodec-list" ) )
  77         {
  78                 fprintf( stderr, "---\naudio_codecs:\n" );
  79                 AVCodec *codec = NULL;
  80                 while ( ( codec = av_codec_next( codec ) ) )
  81                         if ( codec->decode && codec->type == CODEC_TYPE_AUDIO )
  82                                 fprintf( stderr, "  - %s\n", codec->name );
  83                 fprintf( stderr, "...\n" );
  84                 error = 1;
  85         }
  86         if ( file && strstr( file, "vcodec-list" ) )
  87         {
  88                 fprintf( stderr, "---\nvideo_codecs:\n" );
  89                 AVCodec *codec = NULL;
  90                 while ( ( codec = av_codec_next( codec ) ) )
  91                         if ( codec->decode && codec->type == CODEC_TYPE_VIDEO )
  92                                 fprintf( stderr, "  - %s\n", codec->name );
  93                 fprintf( stderr, "...\n" );
  94                 error = 1;
  95         }
  96         if ( error )
  97                 return NULL;
  98
  99         mlt_producer this = NULL;
 100
 101         // Check that we have a non-NULL argument
 102         if ( file != NULL )
 103         {
 104                 // Construct the producer
 105                 this = calloc( 1, sizeof( struct mlt_producer_s ) );
 106
 107                 // Initialise it
 108                 if ( mlt_producer_init( this, NULL ) == 0 )
 109                 {
 110                         // Get the properties
 111                         mlt_properties properties = MLT_PRODUCER_PROPERTIES( this );
 112
 113                         // Set the resource property (required for all producers)
 114                         mlt_properties_set( properties, "resource", file );
 115
 116                         // Register our get_frame implementation
 117                         this->get_frame = producer_get_frame;
 118
 119                         // Open the file
 120                         if ( producer_open( this, profile, file ) != 0 )
 121                         {
 122                                 // Clean up
 123                                 mlt_producer_close( this );
 124                                 this = NULL;
 125                         }
 126                         else
 127                         {
 128                                 // Close the file to release resources for large playlists - reopen later as needed
 129                                 mlt_properties_set_data( properties, "dummy_context", NULL, 0, NULL, NULL );
 130                                 mlt_properties_set_data( properties, "audio_context", NULL, 0, NULL, NULL );
 131                                 mlt_properties_set_data( properties, "video_context", NULL, 0, NULL, NULL );
 132
 133                                 // Default the user-selectable indices from the auto-detected indices
 134                                 mlt_properties_set_int( properties, "audio_index",  mlt_properties_get_int( properties, "_audio_index" ) );
 135                                 mlt_properties_set_int( properties, "video_index",  mlt_properties_get_int( properties, "_video_index" ) );
 136                         }
 137                 }
 138         }
 139
 140         return this;
 141 }
 142
 143 /** Find the default streams.
 144 */
 145
 146 static mlt_properties find_default_streams( mlt_properties meta_media, AVFormatContext *context, int *audio_index, int *video_index )
 147 {
 148         int i;
 149         char key[200];
 150
 151         mlt_properties_set_int( meta_media, "meta.media.nb_streams", context->nb_streams );
 152
 153         // Allow for multiple audio and video streams in the file and select first of each (if available)
 154         for( i = 0; i < context->nb_streams; i++ )
 155         {
 156                 // Get the codec context
 157                 AVStream *stream = context->streams[ i ];
 158                 if ( ! stream ) continue;
 159                 AVCodecContext *codec_context = stream->codec;
 160                 if ( ! codec_context ) continue;
 161                 AVCodec *codec = avcodec_find_decoder( codec_context->codec_id );
 162                 if ( ! codec ) continue;
 163
 164                 snprintf( key, sizeof(key), "meta.media.%d.stream.type", i );
 165
 166                 // Determine the type and obtain the first index of each type
 167                 switch( codec_context->codec_type )
 168                 {
 169                         case CODEC_TYPE_VIDEO:
 170                                 if ( *video_index < 0 )
 171                                         *video_index = i;
 172                                 mlt_properties_set( meta_media, key, "video" );
 173                                 snprintf( key, sizeof(key), "meta.media.%d.stream.frame_rate", i );
 174                                 mlt_properties_set_double( meta_media, key, av_q2d( context->streams[ i ]->r_frame_rate ) );
 175 #if LIBAVFORMAT_VERSION_INT >= ((52<<16)+(21<<8)+0)
 176                                 snprintf( key, sizeof(key), "meta.media.%d.stream.sample_aspect_ratio", i );
 177                                 mlt_properties_set_double( meta_media, key, av_q2d( context->streams[ i ]->sample_aspect_ratio ) );
 178 #endif
 179                                 snprintf( key, sizeof(key), "meta.media.%d.codec.pix_fmt", i );
 180                                 mlt_properties_set( meta_media, key, avcodec_get_pix_fmt_name( codec_context->pix_fmt ) );
 181                                 snprintf( key, sizeof(key), "meta.media.%d.codec.sample_aspect_ratio", i );
 182                                 mlt_properties_set_double( meta_media, key, av_q2d( codec_context->sample_aspect_ratio ) );
 183                                 break;
 184                         case CODEC_TYPE_AUDIO:
 185                                 if ( *audio_index < 0 )
 186                                         *audio_index = i;
 187                                 mlt_properties_set( meta_media, key, "audio" );
 188 #if (LIBAVCODEC_VERSION_INT >= ((51<<16)+(71<<8)+0))
 189                                 snprintf( key, sizeof(key), "meta.media.%d.codec.sample_fmt", i );
 190                                 mlt_properties_set( meta_media, key, avcodec_get_sample_fmt_name( codec_context->sample_fmt ) );
 191 #endif
 192                                 snprintf( key, sizeof(key), "meta.media.%d.codec.sample_rate", i );
 193                                 mlt_properties_set_int( meta_media, key, codec_context->sample_rate );
 194                                 snprintf( key, sizeof(key), "meta.media.%d.codec.channels", i );
 195                                 mlt_properties_set_int( meta_media, key, codec_context->channels );
 196                                 break;
 197                         default:
 198                                 break;
 199                 }
 200 //              snprintf( key, sizeof(key), "meta.media.%d.stream.time_base", i );
 201 //              mlt_properties_set_double( meta_media, key, av_q2d( context->streams[ i ]->time_base ) );
 202                 snprintf( key, sizeof(key), "meta.media.%d.codec.name", i );
 203                 mlt_properties_set( meta_media, key, codec->name );
 204 #if (LIBAVCODEC_VERSION_INT >= ((51<<16)+(55<<8)+0))
 205                 snprintf( key, sizeof(key), "meta.media.%d.codec.long_name", i );
 206                 mlt_properties_set( meta_media, key, codec->long_name );
 207 #endif
 208                 snprintf( key, sizeof(key), "meta.media.%d.codec.bit_rate", i );
 209                 mlt_properties_set_int( meta_media, key, codec_context->bit_rate );
 210 //              snprintf( key, sizeof(key), "meta.media.%d.codec.time_base", i );
 211 //              mlt_properties_set_double( meta_media, key, av_q2d( codec_context->time_base ) );
 212                 snprintf( key, sizeof(key), "meta.media.%d.codec.profile", i );
 213                 mlt_properties_set_int( meta_media, key, codec_context->profile );
 214                 snprintf( key, sizeof(key), "meta.media.%d.codec.level", i );
 215                 mlt_properties_set_int( meta_media, key, codec_context->level );
 216         }
 217
 218         return meta_media;
 219 }
 220
 221 /** Producer file destructor.
 222 */
 223
 224 static void producer_file_close( void *context )
 225 {
 226         if ( context != NULL )
 227         {
 228                 // Lock the mutex now
 229                 avformat_lock( );
 230
 231                 // Close the file
 232                 av_close_input_file( context );
 233
 234                 // Unlock the mutex now
 235                 avformat_unlock( );
 236         }
 237 }
 238
 239 /** Producer file destructor.
 240 */
 241
 242 static void producer_codec_close( void *codec )
 243 {
 244         if ( codec != NULL )
 245         {
 246                 // Lock the mutex now
 247                 avformat_lock( );
 248
 249                 // Close the file
 250                 avcodec_close( codec );
 251
 252                 // Unlock the mutex now
 253                 avformat_unlock( );
 254         }
 255 }
 256
 257 static inline int dv_is_pal( AVPacket *pkt )
 258 {
 259         return pkt->data[3] & 0x80;
 260 }
 261
 262 static int dv_is_wide( AVPacket *pkt )
 263 {
 264         int i = 80 /* block size */ *3 /* VAUX starts at block 3 */ +3 /* skip block header */;
 265
 266         for ( ; i < pkt->size; i += 5 /* packet size */ )
 267         {
 268                 if ( pkt->data[ i ] == 0x61 )
 269                 {
 270                         uint8_t x = pkt->data[ i + 2 ] & 0x7;
 271                         return ( x == 2 ) || ( x == 7 );
 272                 }
 273         }
 274         return 0;
 275 }
 276
 277 static double get_aspect_ratio( AVStream *stream, AVCodecContext *codec_context, AVPacket *pkt )
 278 {
 279         double aspect_ratio = 1.0;
 280
 281         if ( codec_context->codec_id == CODEC_ID_DVVIDEO )
 282         {
 283                 if ( pkt )
 284                 {
 285                         if ( dv_is_pal( pkt ) )
 286                         {
 287                                 aspect_ratio = dv_is_wide( pkt )
 288                                         ? 64.0/45.0 // 16:9 PAL
 289                                         : 16.0/15.0; // 4:3 PAL
 290                         }
 291                         else
 292                         {
 293                                 aspect_ratio = dv_is_wide( pkt )
 294                                         ? 32.0/27.0 // 16:9 NTSC
 295                                         : 8.0/9.0; // 4:3 NTSC
 296                         }
 297                 }
 298                 else
 299                 {
 300                         AVRational ar =
 301 #if LIBAVFORMAT_VERSION_INT >= ((52<<16)+(21<<8)+0)
 302                                 stream->sample_aspect_ratio;
 303 #else
 304                                 codec_context->sample_aspect_ratio;
 305 #endif
 306                         // Override FFmpeg's notion of DV aspect ratios, which are
 307                         // based upon a width of 704. Since we do not have a normaliser
 308                         // that crops (nor is cropping 720 wide ITU-R 601 video always desirable)
 309                         // we just coerce the values to facilitate a passive behaviour through
 310                         // the rescale normaliser when using equivalent producers and consumers.
 311                         // = display_aspect / (width * height)
 312                         if ( ar.num == 10 && ar.den == 11 )
 313                                 aspect_ratio = 8.0/9.0; // 4:3 NTSC
 314                         else if ( ar.num == 59 && ar.den == 54 )
 315                                 aspect_ratio = 16.0/15.0; // 4:3 PAL
 316                         else if ( ar.num == 40 && ar.den == 33 )
 317                                 aspect_ratio = 32.0/27.0; // 16:9 NTSC
 318                         else if ( ar.num == 118 && ar.den == 81 )
 319                                 aspect_ratio = 64.0/45.0; // 16:9 PAL
 320                 }
 321         }
 322         else
 323         {
 324                 AVRational codec_sar = codec_context->sample_aspect_ratio;
 325                 AVRational stream_sar =
 326 #if LIBAVFORMAT_VERSION_INT >= ((52<<16)+(21<<8)+0)
 327                         stream->sample_aspect_ratio;
 328 #else
 329                         { 0, 1 };
 330 #endif
 331                 if ( codec_sar.num > 0 )
 332                         aspect_ratio = av_q2d( codec_sar );
 333                 else if ( stream_sar.num > 0 )
 334                         aspect_ratio = av_q2d( stream_sar );
 335         }
 336         return aspect_ratio;
 337 }
 338
 339 /** Open the file.
 340 */
 341
 342 static int producer_open( mlt_producer this, mlt_profile profile, char *file )
 343 {
 344         // Return an error code (0 == no error)
 345         int error = 0;
 346
 347         // Context for avformat
 348         AVFormatContext *context = NULL;
 349
 350         // Get the properties
 351         mlt_properties properties = MLT_PRODUCER_PROPERTIES( this );
 352
 353         // We will treat everything with the producer fps
 354         double fps = mlt_profile_fps( profile );
 355
 356         // Lock the mutex now
 357         avformat_lock( );
 358
 359         // If "MRL", then create AVInputFormat
 360         AVInputFormat *format = NULL;
 361         AVFormatParameters *params = NULL;
 362         char *standard = NULL;
 363         char *mrl = strchr( file, ':' );
 364
 365         // AV option (0 = both, 1 = video, 2 = audio)
 366         int av = 0;
 367
 368         // Only if there is not a protocol specification that avformat can handle
 369         if ( mrl && !url_exist( file ) )
 370         {
 371                 // 'file' becomes format abbreviation
 372                 mrl[0] = 0;
 373
 374                 // Lookup the format
 375                 format = av_find_input_format( file );
 376
 377                 // Eat the format designator
 378                 file = ++mrl;
 379
 380                 if ( format )
 381                 {
 382                         // Allocate params
 383                         params = calloc( sizeof( AVFormatParameters ), 1 );
 384
 385                         // These are required by video4linux (defaults)
 386                         params->width = 640;
 387                         params->height = 480;
 388                         params->time_base= (AVRational){1,25};
 389                         // params->device = file;
 390                         params->channels = 2;
 391                         params->sample_rate = 48000;
 392                 }
 393
 394                 // XXX: this does not work anymore since avdevice
 395                 // TODO: make producer_avddevice?
 396                 // Parse out params
 397                 mrl = strchr( file, '?' );
 398                 while ( mrl )
 399                 {
 400                         mrl[0] = 0;
 401                         char *name = strdup( ++mrl );
 402                         char *value = strchr( name, ':' );
 403                         if ( value )
 404                         {
 405                                 value[0] = 0;
 406                                 value++;
 407                                 char *t = strchr( value, '&' );
 408                                 if ( t )
 409                                         t[0] = 0;
 410                                 if ( !strcmp( name, "frame_rate" ) )
 411                                         params->time_base.den = atoi( value );
 412                                 else if ( !strcmp( name, "frame_rate_base" ) )
 413                                         params->time_base.num = atoi( value );
 414                                 else if ( !strcmp( name, "sample_rate" ) )
 415                                         params->sample_rate = atoi( value );
 416                                 else if ( !strcmp( name, "channels" ) )
 417                                         params->channels = atoi( value );
 418                                 else if ( !strcmp( name, "width" ) )
 419                                         params->width = atoi( value );
 420                                 else if ( !strcmp( name, "height" ) )
 421                                         params->height = atoi( value );
 422                                 else if ( !strcmp( name, "standard" ) )
 423                                 {
 424                                         standard = strdup( value );
 425                                         params->standard = standard;
 426                                 }
 427                                 else if ( !strcmp( name, "av" ) )
 428                                         av = atoi( value );
 429                         }
 430                         free( name );
 431                         mrl = strchr( mrl, '&' );
 432                 }
 433         }
 434
 435         // Now attempt to open the file
 436         error = av_open_input_file( &context, file, format, 0, params ) < 0;
 437
 438         // Cleanup AVFormatParameters
 439         free( standard );
 440         free( params );
 441
 442         // If successful, then try to get additional info
 443         if ( error == 0 )
 444         {
 445                 // Get the stream info
 446                 error = av_find_stream_info( context ) < 0;
 447
 448                 // Continue if no error
 449                 if ( error == 0 )
 450                 {
 451                         // We will default to the first audio and video streams found
 452                         int audio_index = -1;
 453                         int video_index = -1;
 454                         int av_bypass = 0;
 455
 456                         // Now set properties where we can (use default unknowns if required)
 457                         if ( context->duration != AV_NOPTS_VALUE )
 458                         {
 459                                 // This isn't going to be accurate for all formats
 460                                 mlt_position frames = ( mlt_position )( ( ( double )context->duration / ( double )AV_TIME_BASE ) * fps + 0.5 );
 461                                 mlt_properties_set_position( properties, "out", frames - 1 );
 462                                 mlt_properties_set_position( properties, "length", frames );
 463                         }
 464
 465                         // Find default audio and video streams
 466                         find_default_streams( properties, context, &audio_index, &video_index );
 467
 468                         if ( context->start_time != AV_NOPTS_VALUE )
 469                                 mlt_properties_set_double( properties, "_start_time", context->start_time );
 470
 471                         // Check if we're seekable (something funny about mpeg here :-/)
 472                         if ( strcmp( file, "pipe:" ) && strncmp( file, "http://", 6 )  && strncmp( file, "udp:", 4 )  && strncmp( file, "tcp:", 4 ) && strncmp( file, "rtsp:", 5 )  && strncmp( file, "rtp:", 4 ) )
 473                         {
 474                                 mlt_properties_set_int( properties, "seekable", av_seek_frame( context, -1, mlt_properties_get_double( properties, "_start_time" ), AVSEEK_FLAG_BACKWARD ) >= 0 );
 475                                 mlt_properties_set_data( properties, "dummy_context", context, 0, producer_file_close, NULL );
 476                                 av_open_input_file( &context, file, NULL, 0, NULL );
 477                                 av_find_stream_info( context );
 478                         }
 479                         else
 480                                 av_bypass = 1;
 481
 482                         // Store selected audio and video indexes on properties
 483                         mlt_properties_set_int( properties, "_audio_index", audio_index );
 484                         mlt_properties_set_int( properties, "_video_index", video_index );
 485                         mlt_properties_set_int( properties, "_pts_dts_delta", -1 );
 486                         mlt_properties_set_int( properties, "_first_pts", -1 );
 487                         mlt_properties_set_int( properties, "_last_position", POSITION_INITIAL );
 488
 489                         // Fetch the width, height and aspect ratio
 490                         if ( video_index != -1 )
 491                         {
 492                                 AVCodecContext *codec_context = context->streams[ video_index ]->codec;
 493                                 mlt_properties_set_int( properties, "width", codec_context->width );
 494                                 mlt_properties_set_int( properties, "height", codec_context->height );
 495
 496                                 if ( codec_context->codec_id == CODEC_ID_DVVIDEO )
 497                                 {
 498                                         // Fetch the first frame of DV so we can read it directly
 499                                         AVPacket pkt;
 500                                         int ret = 0;
 501                                         while ( ret >= 0 )
 502                                         {
 503                                                 ret = av_read_frame( context, &pkt );
 504                                                 if ( ret >= 0 && pkt.stream_index == video_index && pkt.size > 0 )
 505                                                 {
 506                                                         mlt_properties_set_double( properties, "aspect_ratio",
 507                                                                 get_aspect_ratio( context->streams[ video_index ], codec_context, &pkt ) );
 508                                                         break;
 509                                                 }
 510                                         }
 511                                 }
 512                                 else
 513                                 {
 514                                         mlt_properties_set_double( properties, "aspect_ratio",
 515                                                 get_aspect_ratio( context->streams[ video_index ], codec_context, NULL ) );
 516                                 }
 517
 518 #ifdef SEEK_ON_OPEN
 519                                 int use_new_seek = codec_context->codec_id == CODEC_ID_H264 && !strcmp( context->iformat->name, "mpegts" );
 520                                 if ( mlt_properties_get( properties, "new_seek" ) )
 521                                         use_new_seek = mlt_properties_get_int( properties, "new_seek" );
 522                                 if ( use_new_seek && mlt_properties_get_int( properties, "seekable" ) )
 523                                 {
 524                                         // find first key frame
 525                                         int ret = 0;
 526                                         int toscan = 100;
 527                                         AVPacket pkt;
 528                                         while ( ret >= 0 && toscan-- > 0 )
 529                                         {
 530                                                 ret = av_read_frame( context, &pkt );
 531                                                 if ( pkt.flags & PKT_FLAG_KEY &&
 532                                                         pkt.stream_index == video_index )
 533                                                 {
 534                                                         mlt_log_verbose( MLT_PRODUCER_SERVICE(this), "first_pts %lld dts %lld pts_dts_delta %d\n", pkt.pts, pkt.dts, (int)(pkt.pts - pkt.dts) );
 535                                                         mlt_properties_set_int( properties, "_pts_dts_delta", pkt.pts - pkt.dts );
 536                                                         mlt_properties_set_int( properties, "_first_pts", pkt.pts );
 537                                                         toscan = 0;
 538                                                 }
 539                                                 av_free_packet( &pkt );
 540                                         }
 541                                         av_seek_frame( context, -1, 0, AVSEEK_FLAG_BACKWARD );
 542                                 }
 543 #endif
 544                         }
 545
 546                         // Read Metadata
 547                         if (context->title != NULL)
 548                                 mlt_properties_set(properties, "meta.attr.title.markup", context->title );
 549                         if (context->author != NULL)
 550                                 mlt_properties_set(properties, "meta.attr.author.markup", context->author );
 551                         if (context->copyright != NULL)
 552                                 mlt_properties_set(properties, "meta.attr.copyright.markup", context->copyright );
 553                         if (context->comment != NULL)
 554                                 mlt_properties_set(properties, "meta.attr.comment.markup", context->comment );
 555                         if (context->album != NULL)
 556                                 mlt_properties_set(properties, "meta.attr.album.markup", context->album );
 557                         if (context->year != 0)
 558                                 mlt_properties_set_int(properties, "meta.attr.year.markup", context->year );
 559                         if (context->track != 0)
 560                                 mlt_properties_set_int(properties, "meta.attr.track.markup", context->track );
 561
 562                         // We're going to cheat here - for a/v files, we will have two contexts (reasoning will be clear later)
 563                         if ( av == 0 && audio_index != -1 && video_index != -1 )
 564                         {
 565                                 // We'll use the open one as our video_context
 566                                 mlt_properties_set_data( properties, "video_context", context, 0, producer_file_close, NULL );
 567
 568                                 // And open again for our audio context
 569                                 av_open_input_file( &context, file, NULL, 0, NULL );
 570                                 av_find_stream_info( context );
 571
 572                                 // Audio context
 573                                 mlt_properties_set_data( properties, "audio_context", context, 0, producer_file_close, NULL );
 574                         }
 575                         else if ( av != 2 && video_index != -1 )
 576                         {
 577                                 // We only have a video context
 578                                 mlt_properties_set_data( properties, "video_context", context, 0, producer_file_close, NULL );
 579                         }
 580                         else if ( audio_index != -1 )
 581                         {
 582                                 // We only have an audio context
 583                                 mlt_properties_set_data( properties, "audio_context", context, 0, producer_file_close, NULL );
 584                         }
 585                         else
 586                         {
 587                                 // Something has gone wrong
 588                                 error = -1;
 589                         }
 590
 591                         mlt_properties_set_int( properties, "av_bypass", av_bypass );
 592                 }
 593         }
 594
 595         // Unlock the mutex now
 596         avformat_unlock( );
 597
 598         return error;
 599 }
 600
 601 /** Convert a frame position to a time code.
 602 */
 603
 604 static double producer_time_of_frame( mlt_producer this, mlt_position position )
 605 {
 606         return ( double )position / mlt_producer_get_fps( this );
 607 }
 608
 609 static inline void convert_image( AVFrame *frame, uint8_t *buffer, int pix_fmt, mlt_image_format *format, int width, int height )
 610 {
 611 #ifdef SWSCALE
 612         if ( pix_fmt == PIX_FMT_RGB32 )
 613         {
 614                 *format = mlt_image_rgb24a;
 615                 struct SwsContext *context = sws_getContext( width, height, pix_fmt,
 616                         width, height, PIX_FMT_RGBA, SWS_FAST_BILINEAR, NULL, NULL, NULL);
 617                 AVPicture output;
 618                 avpicture_fill( &output, buffer, PIX_FMT_RGBA, width, height );
 619                 sws_scale( context, frame->data, frame->linesize, 0, height,
 620                         output.data, output.linesize);
 621                 sws_freeContext( context );
 622         }
 623         else if ( *format == mlt_image_yuv420p )
 624         {
 625                 struct SwsContext *context = sws_getContext( width, height, pix_fmt,
 626                         width, height, PIX_FMT_YUV420P, SWS_FAST_BILINEAR, NULL, NULL, NULL);
 627                 AVPicture output;
 628                 output.data[0] = buffer;
 629                 output.data[1] = buffer + width * height;
 630                 output.data[2] = buffer + ( 3 * width * height ) / 2;
 631                 output.linesize[0] = width;
 632                 output.linesize[1] = width >> 1;
 633                 output.linesize[2] = width >> 1;
 634                 sws_scale( context, frame->data, frame->linesize, 0, height,
 635                         output.data, output.linesize);
 636                 sws_freeContext( context );
 637         }
 638         else if ( *format == mlt_image_rgb24 )
 639         {
 640                 struct SwsContext *context = sws_getContext( width, height, pix_fmt,
 641                         width, height, PIX_FMT_RGB24, SWS_FAST_BILINEAR, NULL, NULL, NULL);
 642                 AVPicture output;
 643                 avpicture_fill( &output, buffer, PIX_FMT_RGB24, width, height );
 644                 sws_scale( context, frame->data, frame->linesize, 0, height,
 645                         output.data, output.linesize);
 646                 sws_freeContext( context );
 647         }
 648         else if ( *format == mlt_image_rgb24a || *format == mlt_image_opengl )
 649         {
 650                 struct SwsContext *context = sws_getContext( width, height, pix_fmt,
 651                         width, height, PIX_FMT_RGBA, SWS_FAST_BILINEAR, NULL, NULL, NULL);
 652                 AVPicture output;
 653                 avpicture_fill( &output, buffer, PIX_FMT_RGBA, width, height );
 654                 sws_scale( context, frame->data, frame->linesize, 0, height,
 655                         output.data, output.linesize);
 656                 sws_freeContext( context );
 657         }
 658         else
 659         {
 660                 struct SwsContext *context = sws_getContext( width, height, pix_fmt,
 661                         width, height, PIX_FMT_YUYV422, SWS_FAST_BILINEAR, NULL, NULL, NULL);
 662                 AVPicture output;
 663                 avpicture_fill( &output, buffer, PIX_FMT_YUYV422, width, height );
 664                 sws_scale( context, frame->data, frame->linesize, 0, height,
 665                         output.data, output.linesize);
 666                 sws_freeContext( context );
 667         }
 668 #else
 669         if ( *format == mlt_image_yuv420p )
 670         {
 671                 AVPicture pict;
 672                 pict.data[0] = buffer;
 673                 pict.data[1] = buffer + width * height;
 674                 pict.data[2] = buffer + ( 3 * width * height ) / 2;
 675                 pict.linesize[0] = width;
 676                 pict.linesize[1] = width >> 1;
 677                 pict.linesize[2] = width >> 1;
 678                 img_convert( &pict, PIX_FMT_YUV420P, (AVPicture *)frame, pix_fmt, width, height );
 679         }
 680         else if ( *format == mlt_image_rgb24 )
 681         {
 682                 AVPicture output;
 683                 avpicture_fill( &output, buffer, PIX_FMT_RGB24, width, height );
 684                 img_convert( &output, PIX_FMT_RGB24, (AVPicture *)frame, pix_fmt, width, height );
 685         }
 686         else if ( format == mlt_image_rgb24a || format == mlt_image_opengl )
 687         {
 688                 AVPicture output;
 689                 avpicture_fill( &output, buffer, PIX_FMT_RGB32, width, height );
 690                 img_convert( &output, PIX_FMT_RGB32, (AVPicture *)frame, pix_fmt, width, height );
 691         }
 692         else
 693         {
 694                 AVPicture output;
 695                 avpicture_fill( &output, buffer, PIX_FMT_YUYV422, width, height );
 696                 img_convert( &output, PIX_FMT_YUYV422, (AVPicture *)frame, pix_fmt, width, height );
 697         }
 698 #endif
 699 }
 700
 701 /** Allocate the image buffer and set it on the frame.
 702 */
 703
 704 static int allocate_buffer( mlt_properties frame_properties, AVCodecContext *codec_context, uint8_t **buffer, mlt_image_format *format, int *width, int *height )
 705 {
 706         int size = 0;
 707
 708         if ( codec_context->width == 0 || codec_context->height == 0 )
 709                 return size;
 710
 711         *width = codec_context->width;
 712         *height = codec_context->height;
 713         mlt_properties_set_int( frame_properties, "width", *width );
 714         mlt_properties_set_int( frame_properties, "height", *height );
 715
 716         if ( codec_context->pix_fmt == PIX_FMT_RGB32 )
 717                 size = *width * ( *height + 1 ) * 4;
 718         else switch ( *format )
 719         {
 720                 case mlt_image_yuv420p:
 721                         size = *width * 3 * ( *height + 1 ) / 2;
 722                         break;
 723                 case mlt_image_rgb24:
 724                         size = *width * ( *height + 1 ) * 3;
 725                         break;
 726                 case mlt_image_rgb24a:
 727                 case mlt_image_opengl:
 728                         size = *width * ( *height + 1 ) * 4;
 729                         break;
 730                 default:
 731                         *format = mlt_image_yuv422;
 732                         size = *width * ( *height + 1 ) * 2;
 733                         break;
 734         }
 735
 736         // Construct the output image
 737         *buffer = mlt_pool_alloc( size );
 738         if ( *buffer )
 739                 mlt_properties_set_data( frame_properties, "image", *buffer, size, (mlt_destructor)mlt_pool_release, NULL );
 740         else
 741                 size = 0;
 742
 743         return size;
 744 }
 745
 746 /** Get an image from a frame.
 747 */
 748
 749 static int producer_get_image( mlt_frame frame, uint8_t **buffer, mlt_image_format *format, int *width, int *height, int writable )
 750 {
 751         // Get the properties from the frame
 752         mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame );
 753
 754         // Obtain the frame number of this frame
 755         mlt_position position = mlt_properties_get_position( frame_properties, "avformat_position" );
 756
 757         // Get the producer
 758         mlt_producer this = mlt_properties_get_data( frame_properties, "avformat_producer", NULL );
 759
 760         // Get the producer properties
 761         mlt_properties properties = MLT_PRODUCER_PROPERTIES( this );
 762
 763         // Fetch the video_context
 764         AVFormatContext *context = mlt_properties_get_data( properties, "video_context", NULL );
 765
 766         // Get the video_index
 767         int index = mlt_properties_get_int( properties, "video_index" );
 768
 769         // Obtain the expected frame numer
 770         mlt_position expected = mlt_properties_get_position( properties, "_video_expected" );
 771
 772         // Get the video stream
 773         AVStream *stream = context->streams[ index ];
 774
 775         // Get codec context
 776         AVCodecContext *codec_context = stream->codec;
 777
 778         // Packet
 779         AVPacket pkt;
 780
 781         // Get the conversion frame
 782         AVFrame *av_frame = mlt_properties_get_data( properties, "av_frame", NULL );
 783
 784         // Special case pause handling flag
 785         int paused = 0;
 786
 787         // Special case ffwd handling
 788         int ignore = 0;
 789
 790         // We may want to use the source fps if available
 791         double source_fps = mlt_properties_get_double( properties, "source_fps" );
 792         double fps = mlt_producer_get_fps( this );
 793
 794         // Get pts info
 795         //int pts_dts_delta = mlt_properties_get_int( properties, "_pts_dts_delta" );
 796         int first_pts = mlt_properties_get_int( properties, "_first_pts" );
 797
 798         // This is the physical frame position in the source
 799         int req_position = ( int )( position / fps * source_fps + 0.5 );
 800
 801         // Get the seekable status
 802         int seekable = mlt_properties_get_int( properties, "seekable" );
 803
 804         // Hopefully provide better support for streams...
 805         int av_bypass = mlt_properties_get_int( properties, "av_bypass" );
 806
 807         // Determines if we have to decode all frames in a sequence
 808         int must_decode = 1;
 809
 810         // Temporary hack to improve intra frame only
 811         must_decode = strcmp( codec_context->codec->name, "dnxhd" ) &&
 812                                   strcmp( codec_context->codec->name, "dvvideo" ) &&
 813                                   strcmp( codec_context->codec->name, "huffyuv" ) &&
 814                                   strcmp( codec_context->codec->name, "mjpeg" ) &&
 815                                   strcmp( codec_context->codec->name, "rawvideo" );
 816
 817         int last_position = mlt_properties_get_int( properties, "_last_position" );
 818
 819         // Turn on usage of new seek API and PTS for seeking
 820         int use_new_seek = codec_context->codec_id == CODEC_ID_H264 && !strcmp( context->iformat->name, "mpegts" );
 821         if ( mlt_properties_get( properties, "new_seek" ) )
 822                 use_new_seek = mlt_properties_get_int( properties, "new_seek" );
 823
 824         // Seek if necessary
 825         if ( position != expected || last_position < 0 )
 826         {
 827                 if ( av_frame != NULL && position + 1 == expected )
 828                 {
 829                         // We're paused - use last image
 830                         paused = 1;
 831                 }
 832                 else if ( !seekable && position > expected && ( position - expected ) < 250 )
 833                 {
 834                         // Fast forward - seeking is inefficient for small distances - just ignore following frames
 835                         ignore = ( int )( ( position - expected ) / fps * source_fps );
 836                         codec_context->skip_loop_filter = AVDISCARD_NONREF;
 837                 }
 838                 else if ( seekable && ( position < expected || position - expected >= 12 || last_position <-0 ) )
 839                 {
 840 #ifndef SEEK_ON_OPEN
 841                         if ( use_new_seek && last_position == POSITION_INITIAL )
 842                         {
 843                                 // find first key frame
 844                                 int ret = 0;
 845                                 int toscan = 100;
 846
 847                                 while ( ret >= 0 && toscan-- > 0 )
 848                                 {
 849                                         ret = av_read_frame( context, &pkt );
 850                                         if ( pkt.flags & PKT_FLAG_KEY &&
 851                                                 pkt.stream_index == index )
 852                                         {
 853                                                 mlt_log_verbose( MLT_PRODUCER_SERVICE(this), "first_pts %lld dts %lld pts_dts_delta %d\n", pkt.pts, pkt.dts, (int)(pkt.pts - pkt.dts) );
 854                                                 mlt_properties_set_int( properties, "_pts_dts_delta", pkt.pts - pkt.dts );
 855                                                 mlt_properties_set_int( properties, "_first_pts", pkt.pts );
 856                                                 toscan = 0;
 857                                         }
 858                                         av_free_packet( &pkt );
 859                                 }
 860                                 av_seek_frame( context, -1, 0, AVSEEK_FLAG_BACKWARD );
 861                         }
 862 #endif
 863                         // Calculate the timestamp for the requested frame
 864                         int64_t timestamp;
 865                         if ( use_new_seek )
 866                         {
 867                                 timestamp = ( req_position - 0.1 / source_fps ) /
 868                                         ( av_q2d( stream->time_base ) * source_fps );
 869                                 mlt_log_verbose( MLT_PRODUCER_SERVICE(this), "pos %d pts %lld ", req_position, timestamp );
 870                                 if ( first_pts > 0 )
 871                                         timestamp += first_pts;
 872                                 else if ( context->start_time != AV_NOPTS_VALUE )
 873                                         timestamp += context->start_time;
 874                         }
 875                         else
 876                         {
 877                                 timestamp = ( int64_t )( ( double )req_position / source_fps * AV_TIME_BASE + 0.5 );
 878                                 if ( ( uint64_t )context->start_time != AV_NOPTS_VALUE )
 879                                         timestamp += context->start_time;
 880                                 if ( must_decode )
 881                                         timestamp -= AV_TIME_BASE;
 882                         }
 883
 884                         if ( timestamp < 0 )
 885                                 timestamp = 0;
 886
 887                         // Set to the timestamp
 888                         mlt_log_debug( MLT_PRODUCER_SERVICE( this ), "seeking timestamp %lld position %d expected %d last_pos %d\n",
 889                                 timestamp, position, expected, last_position );
 890
 891                         if ( use_new_seek )
 892                         {
 893                                 codec_context->skip_loop_filter = AVDISCARD_NONREF;
 894                                 av_seek_frame( context, index, timestamp, AVSEEK_FLAG_BACKWARD );
 895                         }
 896                         else
 897                         {
 898                                 av_seek_frame( context, -1, timestamp, AVSEEK_FLAG_BACKWARD );
 899                         }
 900
 901                         // Remove the cached info relating to the previous position
 902                         mlt_properties_set_int( properties, "_current_position", -1 );
 903                         mlt_properties_set_int( properties, "_last_position", POSITION_INVALID );
 904                         mlt_properties_set_data( properties, "av_frame", NULL, 0, NULL, NULL );
 905                         av_frame = NULL;
 906
 907                         if ( use_new_seek )
 908                         {
 909                                 // flush any pictures still in decode buffer
 910                                 avcodec_flush_buffers( codec_context );
 911                         }
 912                 }
 913         }
 914
 915         // Duplicate the last image if necessary (see comment on rawvideo below)
 916         int current_position = mlt_properties_get_int( properties, "_current_position" );
 917         int got_picture = mlt_properties_get_int( properties, "_got_picture" );
 918         if ( av_frame != NULL && got_picture && ( paused || current_position == req_position || ( !use_new_seek && current_position > req_position ) ) && av_bypass == 0 )
 919         {
 920                 // Duplicate it
 921                 if ( allocate_buffer( frame_properties, codec_context, buffer, format, width, height ) )
 922                         convert_image( av_frame, *buffer, codec_context->pix_fmt, format, *width, *height );
 923                 else
 924                         mlt_frame_get_image( frame, buffer, format, width, height, writable );
 925         }
 926         else
 927         {
 928                 int ret = 0;
 929                 int int_position = 0;
 930                 int decode_errors = 0;
 931                 got_picture = 0;
 932
 933                 av_init_packet( &pkt );
 934
 935                 // Construct an AVFrame for YUV422 conversion
 936                 if ( av_frame == NULL )
 937                         av_frame = avcodec_alloc_frame( );
 938
 939                 while( ret >= 0 && !got_picture )
 940                 {
 941                         // Read a packet
 942                         ret = av_read_frame( context, &pkt );
 943
 944                         // We only deal with video from the selected video_index
 945                         if ( ret >= 0 && pkt.stream_index == index && pkt.size > 0 )
 946                         {
 947                                 // Determine time code of the packet
 948                                 if ( use_new_seek )
 949                                 {
 950                                         int64_t pts = pkt.pts;
 951                                         if ( first_pts > 0 )
 952                                                 pts -= first_pts;
 953                                         else if ( context->start_time != AV_NOPTS_VALUE )
 954                                                 pts -= context->start_time;
 955                                         int_position = ( int )( av_q2d( stream->time_base ) * pts * source_fps + 0.1 );
 956                                 }
 957                                 else
 958                                 {
 959                                         if ( pkt.dts != AV_NOPTS_VALUE )
 960                                         {
 961                                                 int_position = ( int )( av_q2d( stream->time_base ) * pkt.dts * source_fps + 0.5 );
 962                                                 if ( context->start_time != AV_NOPTS_VALUE )
 963                                                         int_position -= ( int )( context->start_time * source_fps / AV_TIME_BASE + 0.5 );
 964                                                 last_position = mlt_properties_get_int( properties, "_last_position" );
 965                                                 if ( int_position == last_position )
 966                                                         int_position = last_position + 1;
 967                                         }
 968                                         else
 969                                         {
 970                                                 int_position = req_position;
 971                                         }
 972                                         mlt_log_debug( MLT_PRODUCER_SERVICE(this), "pkt.dts %llu req_pos %d cur_pos %d pkt_pos %d",
 973                                                 pkt.dts, req_position, current_position, int_position );
 974                                         // Make a dumb assumption on streams that contain wild timestamps
 975                                         if ( abs( req_position - int_position ) > 999 )
 976                                         {
 977                                                 int_position = req_position;
 978                                                 mlt_log_debug( MLT_PRODUCER_SERVICE(this), " WILD TIMESTAMP!" );
 979                                         }
 980                                 }
 981                                 mlt_properties_set_int( properties, "_last_position", int_position );
 982
 983                                 // Decode the image
 984                                 if ( must_decode || int_position >= req_position )
 985                                 {
 986                                         codec_context->reordered_opaque = pkt.pts;
 987                                         if ( int_position >= req_position )
 988                                                 codec_context->skip_loop_filter = AVDISCARD_NONE;
 989                                         ret = avcodec_decode_video( codec_context, av_frame, &got_picture, pkt.data, pkt.size );
 990                                         // Note: decode may fail at the beginning of MPEGfile (B-frames referencing before first I-frame), so allow a few errors.
 991                                         if ( ret < 0 )
 992                                         {
 993                                                 if ( ++decode_errors <= 10 )
 994                                                         ret = 0;
 995                                         }
 996                                         else
 997                                         {
 998                                                 decode_errors = 0;
 999                                         }
1000                                 }
1001
1002                                 if ( got_picture )
1003                                 {
1004                                         if ( use_new_seek )
1005                                         {
1006                                                 // Determine time code of the packet
1007                                                 int64_t pts = av_frame->reordered_opaque;
1008                                                 if ( first_pts > 0 )
1009                                                         pts -= first_pts;
1010                                                 else if ( context->start_time != AV_NOPTS_VALUE )
1011                                                         pts -= context->start_time;
1012                                                 int_position = ( int )( av_q2d( stream->time_base) * pts * source_fps + 0.1 );
1013                                                 mlt_log_verbose( MLT_PRODUCER_SERVICE(this), "got frame %d, key %d\n", int_position, av_frame->key_frame );
1014                                         }
1015                                         // Handle ignore
1016                                         if ( int_position < req_position )
1017                                         {
1018                                                 ignore = 0;
1019                                                 got_picture = 0;
1020                                         }
1021                                         else if ( int_position >= req_position )
1022                                         {
1023                                                 ignore = 0;
1024                                                 codec_context->skip_loop_filter = AVDISCARD_NONE;
1025                                         }
1026                                         else if ( ignore -- )
1027                                         {
1028                                                 got_picture = 0;
1029                                         }
1030                                 }
1031                                 mlt_log_debug( MLT_PRODUCER_SERVICE(this), " got_pic %d key %d\n", got_picture, pkt.flags & PKT_FLAG_KEY );
1032                                 av_free_packet( &pkt );
1033                         }
1034                         else if ( ret >= 0 )
1035                         {
1036                                 av_free_packet( &pkt );
1037                         }
1038
1039                         // Now handle the picture if we have one
1040                         if ( got_picture )
1041                         {
1042                                 if ( allocate_buffer( frame_properties, codec_context, buffer, format, width, height ) )
1043                                 {
1044                                         convert_image( av_frame, *buffer, codec_context->pix_fmt, format, *width, *height );
1045                                         if ( !mlt_properties_get( properties, "force_progressive" ) )
1046                                                 mlt_properties_set_int( frame_properties, "progressive", !av_frame->interlaced_frame );
1047                                         mlt_properties_set_int( properties, "top_field_first", av_frame->top_field_first );
1048                                         mlt_properties_set_int( properties, "_current_position", int_position );
1049                                         mlt_properties_set_int( properties, "_got_picture", 1 );
1050                                         mlt_properties_set_data( properties, "av_frame", av_frame, 0, av_free, NULL );
1051                                 }
1052                                 else
1053                                 {
1054                                         got_picture = 0;
1055                                 }
1056                         }
1057                 }
1058                 if ( !got_picture )
1059                         mlt_frame_get_image( frame, buffer, format, width, height, writable );
1060         }
1061
1062         // Very untidy - for rawvideo, the packet contains the frame, hence the free packet
1063         // above will break the pause behaviour - so we wipe the frame now
1064         if ( !strcmp( codec_context->codec->name, "rawvideo" ) )
1065                 mlt_properties_set_data( properties, "av_frame", NULL, 0, NULL, NULL );
1066
1067         // Set the field order property for this frame
1068         mlt_properties_set_int( frame_properties, "top_field_first", mlt_properties_get_int( properties, "top_field_first" ) );
1069
1070         // Regardless of speed, we expect to get the next frame (cos we ain't too bright)
1071         mlt_properties_set_position( properties, "_video_expected", position + 1 );
1072
1073         return 0;
1074 }
1075
1076 /** Process properties as AVOptions and apply to AV context obj
1077 */
1078
1079 static void apply_properties( void *obj, mlt_properties properties, int flags )
1080 {
1081         int i;
1082         int count = mlt_properties_count( properties );
1083         for ( i = 0; i < count; i++ )
1084         {
1085                 const char *opt_name = mlt_properties_get_name( properties, i );
1086                 const AVOption *opt = av_find_opt( obj, opt_name, NULL, flags, flags );
1087                 if ( opt != NULL )
1088 #if LIBAVCODEC_VERSION_INT >= ((52<<16)+(7<<8)+0)
1089                         av_set_string3( obj, opt_name, mlt_properties_get( properties, opt_name), 0, NULL );
1090 #elif LIBAVCODEC_VERSION_INT >= ((51<<16)+(59<<8)+0)
1091                         av_set_string2( obj, opt_name, mlt_properties_get( properties, opt_name), 0 );
1092 #else
1093                         av_set_string( obj, opt_name, mlt_properties_get( properties, opt_name) );
1094 #endif
1095         }
1096 }
1097
1098 /** Set up video handling.
1099 */
1100
1101 static void producer_set_up_video( mlt_producer this, mlt_frame frame )
1102 {
1103         // Get the properties
1104         mlt_properties properties = MLT_PRODUCER_PROPERTIES( this );
1105
1106         // Fetch the video_context
1107         AVFormatContext *context = mlt_properties_get_data( properties, "video_context", NULL );
1108
1109         // Get the video_index
1110         int index = mlt_properties_get_int( properties, "video_index" );
1111
1112         // Reopen the file if necessary
1113         if ( !context && index > -1 )
1114         {
1115                 mlt_events_block( properties, this );
1116                 producer_open( this, mlt_service_profile( MLT_PRODUCER_SERVICE(this) ),
1117                         mlt_properties_get( properties, "resource" ) );
1118                 context = mlt_properties_get_data( properties, "video_context", NULL );
1119                 mlt_properties_set_data( properties, "dummy_context", NULL, 0, NULL, NULL );
1120                 mlt_events_unblock( properties, this );
1121
1122                 // Process properties as AVOptions
1123                 apply_properties( context, properties, AV_OPT_FLAG_DECODING_PARAM );
1124         }
1125
1126         // Exception handling for video_index
1127         if ( context && index >= (int) context->nb_streams )
1128         {
1129                 // Get the last video stream
1130                 for ( index = context->nb_streams - 1; index >= 0 && context->streams[ index ]->codec->codec_type != CODEC_TYPE_VIDEO; --index );
1131                 mlt_properties_set_int( properties, "video_index", index );
1132         }
1133         if ( context && index > -1 && context->streams[ index ]->codec->codec_type != CODEC_TYPE_VIDEO )
1134         {
1135                 // Invalidate the video stream
1136                 index = -1;
1137                 mlt_properties_set_int( properties, "video_index", index );
1138         }
1139
1140         // Get the frame properties
1141         mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame );
1142
1143         if ( context && index > -1 )
1144         {
1145                 // Get the video stream
1146                 AVStream *stream = context->streams[ index ];
1147
1148                 // Get codec context
1149                 AVCodecContext *codec_context = stream->codec;
1150
1151                 // Get the codec
1152                 AVCodec *codec = mlt_properties_get_data( properties, "video_codec", NULL );
1153
1154                 // Update the video properties if the index changed
1155                 if ( index != mlt_properties_get_int( properties, "_video_index" ) )
1156                 {
1157                         // Reset the video properties if the index changed
1158                         mlt_properties_set_int( properties, "_video_index", index );
1159                         mlt_properties_set_data( properties, "video_codec", NULL, 0, NULL, NULL );
1160                         mlt_properties_set_int( properties, "width", codec_context->width );
1161                         mlt_properties_set_int( properties, "height", codec_context->height );
1162                         // TODO: get the first usable AVPacket and reset the stream position
1163                         mlt_properties_set_double( properties, "aspect_ratio",
1164                                 get_aspect_ratio( context->streams[ index ], codec_context, NULL ) );
1165                         codec = NULL;
1166                 }
1167
1168                 // Initialise the codec if necessary
1169                 if ( codec == NULL )
1170                 {
1171                         // Initialise multi-threading
1172                         int thread_count = mlt_properties_get_int( properties, "threads" );
1173                         if ( thread_count == 0 && getenv( "MLT_AVFORMAT_THREADS" ) )
1174                                 thread_count = atoi( getenv( "MLT_AVFORMAT_THREADS" ) );
1175                         if ( thread_count > 1 )
1176                         {
1177                                 avcodec_thread_init( codec_context, thread_count );
1178                                 codec_context->thread_count = thread_count;
1179                         }
1180
1181                         // Find the codec
1182                         codec = avcodec_find_decoder( codec_context->codec_id );
1183
1184                         // If we don't have a codec and we can't initialise it, we can't do much more...
1185                         avformat_lock( );
1186                         if ( codec != NULL && avcodec_open( codec_context, codec ) >= 0 )
1187                         {
1188                                 // Now store the codec with its destructor
1189                                 mlt_properties_set_data( properties, "video_codec", codec_context, 0, producer_codec_close, NULL );
1190                         }
1191                         else
1192                         {
1193                                 // Remember that we can't use this later
1194                                 mlt_properties_set_int( properties, "video_index", -1 );
1195                                 index = -1;
1196                         }
1197                         avformat_unlock( );
1198
1199                         // Process properties as AVOptions
1200                         apply_properties( codec_context, properties, AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM );
1201                 }
1202
1203                 // No codec, no show...
1204                 if ( codec && index > -1 )
1205                 {
1206                         double source_fps = 0;
1207                         double force_aspect_ratio = mlt_properties_get_double( properties, "force_aspect_ratio" );
1208                         double aspect_ratio = ( force_aspect_ratio > 0.0 ) ?
1209                                 force_aspect_ratio : mlt_properties_get_double( properties, "aspect_ratio" );
1210
1211                         // Determine the fps
1212                         source_fps = ( double )codec_context->time_base.den / ( codec_context->time_base.num == 0 ? 1 : codec_context->time_base.num );
1213
1214                         // If the muxer reports a frame rate different than the codec
1215                         double muxer_fps = av_q2d( context->streams[ index ]->r_frame_rate );
1216                         if ( source_fps != muxer_fps )
1217                                 // Choose the lesser - the wrong tends to be off by some multiple of 10
1218                                 source_fps = muxer_fps < source_fps ? muxer_fps : source_fps;
1219
1220                         // We'll use fps if it's available
1221                         if ( source_fps > 0 )
1222                                 mlt_properties_set_double( properties, "source_fps", source_fps );
1223                         else
1224                                 mlt_properties_set_double( properties, "source_fps", mlt_producer_get_fps( this ) );
1225                         mlt_properties_set_double( properties, "aspect_ratio", aspect_ratio );
1226
1227                         // Set the width and height
1228                         mlt_properties_set_int( frame_properties, "width", codec_context->width );
1229                         mlt_properties_set_int( frame_properties, "height", codec_context->height );
1230                         mlt_properties_set_int( frame_properties, "real_width", codec_context->width );
1231                         mlt_properties_set_int( frame_properties, "real_height", codec_context->height );
1232                         mlt_properties_set_double( frame_properties, "aspect_ratio", aspect_ratio );
1233                         if ( mlt_properties_get( properties, "force_progressive" ) )
1234                                 mlt_properties_set_int( frame_properties, "progressive", mlt_properties_get_int( properties, "force_progressive" ) );
1235
1236                         mlt_frame_push_get_image( frame, producer_get_image );
1237                         mlt_properties_set_data( frame_properties, "avformat_producer", this, 0, NULL, NULL );
1238                 }
1239                 else
1240                 {
1241                         mlt_properties_set_int( frame_properties, "test_image", 1 );
1242                 }
1243         }
1244         else
1245         {
1246                 mlt_properties_set_int( frame_properties, "test_image", 1 );
1247         }
1248 }
1249
1250 /** Get the audio from a frame.
1251 */
1252
1253 static int producer_get_audio( mlt_frame frame, void **buffer, mlt_audio_format *format, int *frequency, int *channels, int *samples )
1254 {
1255         // Get the properties from the frame
1256         mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame );
1257
1258         // Obtain the frame number of this frame
1259         mlt_position position = mlt_properties_get_position( frame_properties, "avformat_position" );
1260
1261         // Get the producer
1262         mlt_producer this = mlt_properties_get_data( frame_properties, "avformat_producer", NULL );
1263
1264         // Get the producer properties
1265         mlt_properties properties = MLT_PRODUCER_PROPERTIES( this );
1266
1267         // Fetch the audio_context
1268         AVFormatContext *context = mlt_properties_get_data( properties, "audio_context", NULL );
1269
1270         // Get the audio_index
1271         int index = mlt_properties_get_int( properties, "audio_index" );
1272
1273         // Get the seekable status
1274         int seekable = mlt_properties_get_int( properties, "seekable" );
1275
1276         // Obtain the expected frame numer
1277         mlt_position expected = mlt_properties_get_position( properties, "_audio_expected" );
1278
1279         // Obtain the resample context if it exists (not always needed)
1280         ReSampleContext *resample = mlt_properties_get_data( properties, "audio_resample", NULL );
1281
1282         // Obtain the audio buffers
1283         int16_t *audio_buffer = mlt_properties_get_data( properties, "audio_buffer", NULL );
1284         int16_t *decode_buffer = mlt_properties_get_data( properties, "decode_buffer", NULL );
1285
1286         // Get amount of audio used
1287         int audio_used =  mlt_properties_get_int( properties, "_audio_used" );
1288
1289         // Calculate the real time code
1290         double real_timecode = producer_time_of_frame( this, position );
1291
1292         // Get the audio stream
1293         AVStream *stream = context->streams[ index ];
1294
1295         // Get codec context
1296         AVCodecContext *codec_context = stream->codec;
1297
1298         // Packet
1299         AVPacket pkt;
1300
1301         // Number of frames to ignore (for ffwd)
1302         int ignore = 0;
1303
1304         // Flag for paused (silence)
1305         int paused = 0;
1306
1307         // Check for resample and create if necessary
1308         if ( resample == NULL && codec_context->channels <= 2 )
1309         {
1310                 // Create the resampler
1311 #if (LIBAVCODEC_VERSION_INT >= ((52<<16)+(15<<8)+0))
1312                 resample = av_audio_resample_init( *channels, codec_context->channels, *frequency, codec_context->sample_rate,
1313                         SAMPLE_FMT_S16, codec_context->sample_fmt, 16, 10, 0, 0.8 );
1314 #else
1315                 resample = audio_resample_init( *channels, codec_context->channels, *frequency, codec_context->sample_rate );
1316 #endif
1317
1318                 // And store it on properties
1319                 mlt_properties_set_data( properties, "audio_resample", resample, 0, ( mlt_destructor )audio_resample_close, NULL );
1320         }
1321         else if ( resample == NULL )
1322         {
1323                 // TODO: uncomment and remove following line when full multi-channel support is ready
1324                 // *channels = codec_context->channels;
1325                 codec_context->request_channels = *channels;
1326
1327                 *frequency = codec_context->sample_rate;
1328         }
1329
1330         // Check for audio buffer and create if necessary
1331         if ( audio_buffer == NULL )
1332         {
1333                 // Allocate the audio buffer
1334                 audio_buffer = mlt_pool_alloc( AVCODEC_MAX_AUDIO_FRAME_SIZE * sizeof( int16_t ) );
1335
1336                 // And store it on properties for reuse
1337                 mlt_properties_set_data( properties, "audio_buffer", audio_buffer, 0, ( mlt_destructor )mlt_pool_release, NULL );
1338         }
1339
1340         // Check for decoder buffer and create if necessary
1341         if ( decode_buffer == NULL )
1342         {
1343                 // Allocate the audio buffer
1344                 decode_buffer = av_malloc( AVCODEC_MAX_AUDIO_FRAME_SIZE * sizeof( int16_t ) );
1345
1346                 // And store it on properties for reuse
1347                 mlt_properties_set_data( properties, "decode_buffer", decode_buffer, 0, ( mlt_destructor )av_free, NULL );
1348         }
1349
1350         // Seek if necessary
1351         if ( position != expected )
1352         {
1353                 if ( position + 1 == expected )
1354                 {
1355                         // We're paused - silence required
1356                         paused = 1;
1357                 }
1358                 else if ( !seekable && position > expected && ( position - expected ) < 250 )
1359                 {
1360                         // Fast forward - seeking is inefficient for small distances - just ignore following frames
1361                         ignore = position - expected;
1362                 }
1363                 else if ( position < expected || position - expected >= 12 )
1364                 {
1365                         int64_t timestamp = ( int64_t )( real_timecode * AV_TIME_BASE + 0.5 );
1366                         if ( context->start_time != AV_NOPTS_VALUE )
1367                                 timestamp += context->start_time;
1368                         if ( timestamp < 0 )
1369                                 timestamp = 0;
1370
1371                         // Set to the real timecode
1372                         if ( av_seek_frame( context, -1, timestamp, AVSEEK_FLAG_BACKWARD ) != 0 )
1373                                 paused = 1;
1374
1375                         // Clear the usage in the audio buffer
1376                         audio_used = 0;
1377                 }
1378         }
1379
1380         // Get the audio if required
1381         if ( !paused )
1382         {
1383                 int ret = 0;
1384                 int got_audio = 0;
1385
1386                 av_init_packet( &pkt );
1387
1388                 while( ret >= 0 && !got_audio )
1389                 {
1390                         // Check if the buffer already contains the samples required
1391                         if ( audio_used >= *samples && ignore == 0 )
1392                         {
1393                                 got_audio = 1;
1394                                 break;
1395                         }
1396
1397                         // Read a packet
1398                         ret = av_read_frame( context, &pkt );
1399
1400                         int len = pkt.size;
1401                         uint8_t *ptr = pkt.data;
1402
1403                         // We only deal with audio from the selected audio_index
1404                         while ( ptr != NULL && ret >= 0 && pkt.stream_index == index && len > 0 )
1405                         {
1406                                 int data_size = sizeof( int16_t ) * AVCODEC_MAX_AUDIO_FRAME_SIZE;
1407
1408                                 // Decode the audio
1409 #if (LIBAVCODEC_VERSION_INT >= ((51<<16)+(29<<8)+0))
1410                                 ret = avcodec_decode_audio2( codec_context, decode_buffer, &data_size, ptr, len );
1411 #else
1412                                 ret = avcodec_decode_audio( codec_context, decode_buffer, &data_size, ptr, len );
1413 #endif
1414                                 if ( ret < 0 )
1415                                 {
1416                                         ret = 0;
1417                                         break;
1418                                 }
1419
1420                                 len -= ret;
1421                                 ptr += ret;
1422
1423                                 if ( data_size > 0 && ( audio_used * *channels + data_size < AVCODEC_MAX_AUDIO_FRAME_SIZE ) )
1424                                 {
1425                                         if ( resample )
1426                                         {
1427                                                 int16_t *source = decode_buffer;
1428                                                 int16_t *dest = &audio_buffer[ audio_used * *channels ];
1429                                                 int convert_samples = data_size / av_get_bits_per_sample_format( codec_context->sample_fmt ) * 8 / codec_context->channels;
1430
1431                                                 audio_used += audio_resample( resample, dest, source, convert_samples );
1432                                         }
1433                                         else
1434                                         {
1435                                                 memcpy( &audio_buffer[ audio_used * *channels ], decode_buffer, data_size );
1436                                                 audio_used += data_size / *channels / av_get_bits_per_sample_format( codec_context->sample_fmt ) * 8;
1437                                         }
1438
1439                                         // Handle ignore
1440                                         while ( ignore && audio_used > *samples )
1441                                         {
1442                                                 ignore --;
1443                                                 audio_used -= *samples;
1444                                                 memmove( audio_buffer, &audio_buffer[ *samples * *channels ], audio_used * sizeof( int16_t ) );
1445                                         }
1446                                 }
1447
1448                                 // If we're behind, ignore this packet
1449                                 if ( pkt.pts >= 0 )
1450                                 {
1451                                         double current_pts = av_q2d( stream->time_base ) * pkt.pts;
1452                                         double source_fps = mlt_properties_get_double( properties, "source_fps" );
1453                                         int req_position = ( int )( real_timecode * source_fps + 0.5 );
1454                                         int int_position = ( int )( current_pts * source_fps + 0.5 );
1455
1456                                         if ( context->start_time != AV_NOPTS_VALUE )
1457                                                 int_position -= ( int )( context->start_time * source_fps / AV_TIME_BASE + 0.5 );
1458                                         if ( seekable && !ignore && int_position < req_position )
1459                                                 ignore = 1;
1460                                 }
1461                         }
1462
1463                         // We're finished with this packet regardless
1464                         av_free_packet( &pkt );
1465                 }
1466
1467                 int size = *samples * *channels * sizeof( int16_t );
1468                 *format = mlt_audio_s16;
1469                 *buffer = mlt_pool_alloc( size );
1470                 mlt_frame_set_audio( frame, *buffer, *format, size, mlt_pool_release );
1471
1472                 // Now handle the audio if we have enough
1473                 if ( audio_used >= *samples )
1474                 {
1475                         memcpy( *buffer, audio_buffer, *samples * *channels * sizeof( int16_t ) );
1476                         audio_used -= *samples;
1477                         memmove( audio_buffer, &audio_buffer[ *samples * *channels ], audio_used * *channels * sizeof( int16_t ) );
1478                 }
1479                 else
1480                 {
1481                         memset( *buffer, 0, *samples * *channels * sizeof( int16_t ) );
1482                 }
1483
1484                 // Store the number of audio samples still available
1485                 mlt_properties_set_int( properties, "_audio_used", audio_used );
1486         }
1487         else
1488         {
1489                 // Get silence and don't touch the context
1490                 mlt_frame_get_audio( frame, buffer, format, frequency, channels, samples );
1491         }
1492
1493         // Regardless of speed (other than paused), we expect to get the next frame
1494         if ( !paused )
1495                 mlt_properties_set_position( properties, "_audio_expected", position + 1 );
1496
1497         return 0;
1498 }
1499
1500 /** Set up audio handling.
1501 */
1502
1503 static void producer_set_up_audio( mlt_producer this, mlt_frame frame )
1504 {
1505         // Get the properties
1506         mlt_properties properties = MLT_PRODUCER_PROPERTIES( this );
1507
1508         // Fetch the audio_context
1509         AVFormatContext *context = mlt_properties_get_data( properties, "audio_context", NULL );
1510
1511         // Get the audio_index
1512         int index = mlt_properties_get_int( properties, "audio_index" );
1513
1514         // Reopen the file if necessary
1515         if ( !context && index > -1 )
1516         {
1517                 mlt_events_block( properties, this );
1518                 producer_open( this, mlt_service_profile( MLT_PRODUCER_SERVICE(this) ),
1519                         mlt_properties_get( properties, "resource" ) );
1520                 context = mlt_properties_get_data( properties, "audio_context", NULL );
1521                 mlt_properties_set_data( properties, "dummy_context", NULL, 0, NULL, NULL );
1522                 mlt_events_unblock( properties, this );
1523         }
1524
1525         // Exception handling for audio_index
1526         if ( context && index >= (int) context->nb_streams )
1527         {
1528                 for ( index = context->nb_streams - 1; index >= 0 && context->streams[ index ]->codec->codec_type != CODEC_TYPE_AUDIO; --index );
1529                 mlt_properties_set_int( properties, "audio_index", index );
1530         }
1531         if ( context && index > -1 && context->streams[ index ]->codec->codec_type != CODEC_TYPE_AUDIO )
1532         {
1533                 index = -1;
1534                 mlt_properties_set_int( properties, "audio_index", index );
1535         }
1536
1537         // Update the audio properties if the index changed
1538         if ( index > -1 && index != mlt_properties_get_int( properties, "_audio_index" ) )
1539         {
1540                 mlt_properties_set_int( properties, "_audio_index", index );
1541                 mlt_properties_set_data( properties, "audio_codec", NULL, 0, NULL, NULL );
1542         }
1543
1544         // Deal with audio context
1545         if ( context != NULL && index > -1 )
1546         {
1547                 // Get the frame properties
1548                 mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame );
1549
1550                 // Get the audio stream
1551                 AVStream *stream = context->streams[ index ];
1552
1553                 // Get codec context
1554                 AVCodecContext *codec_context = stream->codec;
1555
1556                 // Get the codec
1557                 AVCodec *codec = mlt_properties_get_data( properties, "audio_codec", NULL );
1558
1559                 // Initialise the codec if necessary
1560                 if ( codec == NULL )
1561                 {
1562                         // Find the codec
1563                         codec = avcodec_find_decoder( codec_context->codec_id );
1564
1565                         // If we don't have a codec and we can't initialise it, we can't do much more...
1566                         avformat_lock( );
1567                         if ( codec != NULL && avcodec_open( codec_context, codec ) >= 0 )
1568                         {
1569                                 // Now store the codec with its destructor
1570                                 mlt_properties_set_data( properties, "audio_codec", codec_context, 0, producer_codec_close, NULL );
1571
1572                         }
1573                         else
1574                         {
1575                                 // Remember that we can't use this later
1576                                 mlt_properties_set_int( properties, "audio_index", -1 );
1577                                 index = -1;
1578                         }
1579                         avformat_unlock( );
1580
1581                         // Process properties as AVOptions
1582                         apply_properties( codec_context, properties, AV_OPT_FLAG_AUDIO_PARAM | AV_OPT_FLAG_DECODING_PARAM );
1583                 }
1584
1585                 // No codec, no show...
1586                 if ( codec && index > -1 )
1587                 {
1588                         mlt_frame_push_audio( frame, producer_get_audio );
1589                         mlt_properties_set_data( frame_properties, "avformat_producer", this, 0, NULL, NULL );
1590                         mlt_properties_set_int( frame_properties, "frequency", codec_context->sample_rate );
1591                         mlt_properties_set_int( frame_properties, "channels", codec_context->channels );
1592                 }
1593         }
1594 }
1595
1596 /** Our get frame implementation.
1597 */
1598
1599 static int producer_get_frame( mlt_producer this, mlt_frame_ptr frame, int index )
1600 {
1601         // Create an empty frame
1602         *frame = mlt_frame_init( MLT_PRODUCER_SERVICE( this ) );
1603
1604         // Update timecode on the frame we're creating
1605         mlt_frame_set_position( *frame, mlt_producer_position( this ) );
1606
1607         // Set the position of this producer
1608         mlt_properties_set_position( MLT_FRAME_PROPERTIES( *frame ), "avformat_position", mlt_producer_frame( this ) );
1609
1610         // Set up the video
1611         producer_set_up_video( this, *frame );
1612
1613         // Set up the audio
1614         producer_set_up_audio( this, *frame );
1615
1616         // Set the aspect_ratio
1617         mlt_properties_set_double( MLT_FRAME_PROPERTIES( *frame ), "aspect_ratio", mlt_properties_get_double( MLT_PRODUCER_PROPERTIES( this ), "aspect_ratio" ) );
1618
1619         // Calculate the next timecode
1620         mlt_producer_prepare_next( this );
1621
1622         return 0;
1623 }