git.sesse.net Git - mlt/blob - src/modules/avformat/producer_avformat.c

   1 /*
   2  * producer_avformat.c -- avformat producer
   3  * Copyright (C) 2003-2009 Ushodaya Enterprises Limited
   4  * Author: Charles Yates <charles.yates@pandora.be>
   5  * Author: Dan Dennedy <dan@dennedy.org>
   6  * Much code borrowed from ffmpeg.c: Copyright (c) 2000-2003 Fabrice Bellard
   7  *
   8  * This library is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License as published by the Free Software Foundation; either
  11  * version 2.1 of the License, or (at your option) any later version.
  12  *
  13  * This library is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public
  19  * License along with this library; if not, write to the Free Software
  20  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  21  */
  22
  23 // MLT Header files
  24 #include <framework/mlt_producer.h>
  25 #include <framework/mlt_frame.h>
  26 #include <framework/mlt_profile.h>
  27 #include <framework/mlt_log.h>
  28 #include <framework/mlt_deque.h>
  29 #include <framework/mlt_factory.h>
  30 #include <framework/mlt_cache.h>
  31
  32 // ffmpeg Header files
  33 #include <avformat.h>
  34 #include <opt.h>
  35 #ifdef SWSCALE
  36 #  include <swscale.h>
  37 #endif
  38 #if (LIBAVCODEC_VERSION_INT >= ((51<<16)+(71<<8)+0))
  39 #  include "audioconvert.h"
  40 #endif
  41 #ifdef VDPAU
  42 #include <vdpau.h>
  43 #endif
  44
  45 // System header files
  46 #include <stdlib.h>
  47 #include <string.h>
  48 #include <pthread.h>
  49 #include <limits.h>
  50
  51 #if LIBAVUTIL_VERSION_INT < (50<<16)
  52 #define PIX_FMT_RGB32 PIX_FMT_RGBA32
  53 #define PIX_FMT_YUYV422 PIX_FMT_YUV422
  54 #endif
  55
  56 #define POSITION_INITIAL (-2)
  57 #define POSITION_INVALID (-1)
  58
  59 #define MAX_AUDIO_STREAMS (10)
  60 #define MAX_VDPAU_SURFACES (10)
  61
  62 void avformat_lock( );
  63 void avformat_unlock( );
  64
  65 struct producer_avformat_s
  66 {
  67         mlt_producer parent;
  68         AVFormatContext *dummy_context;
  69         AVFormatContext *audio_format;
  70         AVFormatContext *video_format;
  71         AVCodecContext *audio_codec[ MAX_AUDIO_STREAMS ];
  72         AVCodecContext *video_codec;
  73         AVFrame *av_frame;
  74         ReSampleContext *audio_resample[ MAX_AUDIO_STREAMS ];
  75         mlt_position audio_expected;
  76         mlt_position video_expected;
  77         int audio_index;
  78         int video_index;
  79         double start_time;
  80         int first_pts;
  81         int last_position;
  82         int seekable;
  83         int current_position;
  84         int got_picture;
  85         int top_field_first;
  86         int16_t *audio_buffer[ MAX_AUDIO_STREAMS ];
  87         size_t audio_buffer_size[ MAX_AUDIO_STREAMS ];
  88         int16_t *decode_buffer[ MAX_AUDIO_STREAMS ];
  89         int audio_used[ MAX_AUDIO_STREAMS ];
  90         int audio_streams;
  91         int audio_max_stream;
  92         int total_channels;
  93         int max_channel;
  94         int max_frequency;
  95         unsigned int invalid_pts_counter;
  96         double resample_factor;
  97         mlt_cache image_cache;
  98 #ifdef VDPAU
  99         struct
 100         {
 101                 // from FFmpeg
 102                 struct vdpau_render_state render_states[MAX_VDPAU_SURFACES];
 103
 104                 // internal
 105                 mlt_deque deque;
 106                 int b_age;
 107                 int ip_age[2];
 108                 int is_decoded;
 109                 uint8_t *buffer;
 110         } *vdpau;
 111 #endif
 112 #ifdef SWSCALE
 113         struct SwsContext *swscale;
 114 #else
 115         void *swscale;
 116 #endif
 117 };
 118 typedef struct producer_avformat_s *producer_avformat;
 119
 120 // Forward references.
 121 static int producer_open( producer_avformat this, mlt_profile profile, char *file );
 122 static int producer_get_frame( mlt_producer this, mlt_frame_ptr frame, int index );
 123 static void producer_avformat_close( producer_avformat );
 124 static void producer_close( mlt_producer parent );
 125
 126 #ifdef VDPAU
 127 #include "vdpau.c"
 128 #endif
 129
 130 /** Constructor for libavformat.
 131 */
 132
 133 mlt_producer producer_avformat_init( mlt_profile profile, const char *service, char *file )
 134 {
 135         int skip = 0;
 136
 137         // Report information about available demuxers and codecs as YAML Tiny
 138         if ( file && strstr( file, "f-list" ) )
 139         {
 140                 fprintf( stderr, "---\nformats:\n" );
 141                 AVInputFormat *format = NULL;
 142                 while ( ( format = av_iformat_next( format ) ) )
 143                         fprintf( stderr, "  - %s\n", format->name );
 144                 fprintf( stderr, "...\n" );
 145                 skip = 1;
 146         }
 147         if ( file && strstr( file, "acodec-list" ) )
 148         {
 149                 fprintf( stderr, "---\naudio_codecs:\n" );
 150                 AVCodec *codec = NULL;
 151                 while ( ( codec = av_codec_next( codec ) ) )
 152                         if ( codec->decode && codec->type == CODEC_TYPE_AUDIO )
 153                                 fprintf( stderr, "  - %s\n", codec->name );
 154                 fprintf( stderr, "...\n" );
 155                 skip = 1;
 156         }
 157         if ( file && strstr( file, "vcodec-list" ) )
 158         {
 159                 fprintf( stderr, "---\nvideo_codecs:\n" );
 160                 AVCodec *codec = NULL;
 161                 while ( ( codec = av_codec_next( codec ) ) )
 162                         if ( codec->decode && codec->type == CODEC_TYPE_VIDEO )
 163                                 fprintf( stderr, "  - %s\n", codec->name );
 164                 fprintf( stderr, "...\n" );
 165                 skip = 1;
 166         }
 167
 168         // Check that we have a non-NULL argument
 169         if ( !skip && file )
 170         {
 171                 // Construct the producer
 172                 mlt_producer producer = calloc( 1, sizeof( struct mlt_producer_s ) );
 173                 producer_avformat this = calloc( 1, sizeof( struct producer_avformat_s ) );
 174
 175                 // Initialise it
 176                 if ( mlt_producer_init( producer, this ) == 0 )
 177                 {
 178                         this->parent = producer;
 179
 180                         // Get the properties
 181                         mlt_properties properties = MLT_PRODUCER_PROPERTIES( producer );
 182
 183                         // Set the resource property (required for all producers)
 184                         mlt_properties_set( properties, "resource", file );
 185
 186                         // Register transport implementation with the producer
 187                         producer->close = (mlt_destructor) producer_close;
 188
 189                         // Register our get_frame implementation
 190                         producer->get_frame = producer_get_frame;
 191
 192                         if ( strcmp( service, "avformat-novalidate" ) )
 193                         {
 194                                 // Open the file
 195                                 if ( producer_open( this, profile, file ) != 0 )
 196                                 {
 197                                         // Clean up
 198                                         mlt_producer_close( producer );
 199                                         producer = NULL;
 200                                 }
 201                                 else
 202                                 {
 203                                         // Close the file to release resources for large playlists - reopen later as needed
 204                                         avformat_lock();
 205                                         if ( this->dummy_context )
 206                                                 av_close_input_file( this->dummy_context );
 207                                         this->dummy_context = NULL;
 208                                         if ( this->audio_format )
 209                                                 av_close_input_file( this->audio_format );
 210                                         this->audio_format = NULL;
 211                                         if ( this->video_format )
 212                                                 av_close_input_file( this->video_format );
 213                                         this->video_format = NULL;
 214                                         avformat_unlock();
 215
 216                                         // Default the user-selectable indices from the auto-detected indices
 217                                         mlt_properties_set_int( properties, "audio_index",  this->audio_index );
 218                                         mlt_properties_set_int( properties, "video_index",  this->video_index );
 219
 220 #ifdef VDPAU
 221                                         mlt_service_cache_set_size( MLT_PRODUCER_SERVICE(producer), "producer_avformat", 5 );
 222 #endif
 223                                         mlt_service_cache_put( MLT_PRODUCER_SERVICE(producer), "producer_avformat", this, 0, (mlt_destructor) producer_avformat_close );
 224                                 }
 225                         }
 226                         else
 227                         {
 228 #ifdef VDPAU
 229                                 mlt_service_cache_set_size( MLT_PRODUCER_SERVICE(producer), "producer_avformat", 5 );
 230 #endif
 231                                 mlt_service_cache_put( MLT_PRODUCER_SERVICE(producer), "producer_avformat", this, 0, (mlt_destructor) producer_avformat_close );
 232                         }
 233                         return producer;
 234                 }
 235         }
 236         return NULL;
 237 }
 238
 239 /** Find the default streams.
 240 */
 241
 242 static mlt_properties find_default_streams( mlt_properties meta_media, AVFormatContext *context, int *audio_index, int *video_index )
 243 {
 244         int i;
 245         char key[200];
 246
 247         mlt_properties_set_int( meta_media, "meta.media.nb_streams", context->nb_streams );
 248
 249         // Allow for multiple audio and video streams in the file and select first of each (if available)
 250         for( i = 0; i < context->nb_streams; i++ )
 251         {
 252                 // Get the codec context
 253                 AVStream *stream = context->streams[ i ];
 254                 if ( ! stream ) continue;
 255                 AVCodecContext *codec_context = stream->codec;
 256                 if ( ! codec_context ) continue;
 257                 AVCodec *codec = avcodec_find_decoder( codec_context->codec_id );
 258                 if ( ! codec ) continue;
 259
 260                 snprintf( key, sizeof(key), "meta.media.%d.stream.type", i );
 261
 262                 // Determine the type and obtain the first index of each type
 263                 switch( codec_context->codec_type )
 264                 {
 265                         case CODEC_TYPE_VIDEO:
 266                                 if ( *video_index < 0 )
 267                                         *video_index = i;
 268                                 mlt_properties_set( meta_media, key, "video" );
 269                                 snprintf( key, sizeof(key), "meta.media.%d.stream.frame_rate", i );
 270                                 mlt_properties_set_double( meta_media, key, av_q2d( context->streams[ i ]->r_frame_rate ) );
 271 #if LIBAVFORMAT_VERSION_INT >= ((52<<16)+(21<<8)+0)
 272                                 snprintf( key, sizeof(key), "meta.media.%d.stream.sample_aspect_ratio", i );
 273                                 mlt_properties_set_double( meta_media, key, av_q2d( context->streams[ i ]->sample_aspect_ratio ) );
 274 #endif
 275                                 snprintf( key, sizeof(key), "meta.media.%d.codec.frame_rate", i );
 276                                 mlt_properties_set_double( meta_media, key, (double) codec_context->time_base.den /
 277                                                                                    ( codec_context->time_base.num == 0 ? 1 : codec_context->time_base.num ) );
 278                                 snprintf( key, sizeof(key), "meta.media.%d.codec.pix_fmt", i );
 279                                 mlt_properties_set( meta_media, key, avcodec_get_pix_fmt_name( codec_context->pix_fmt ) );
 280                                 snprintf( key, sizeof(key), "meta.media.%d.codec.sample_aspect_ratio", i );
 281                                 mlt_properties_set_double( meta_media, key, av_q2d( codec_context->sample_aspect_ratio ) );
 282                                 break;
 283                         case CODEC_TYPE_AUDIO:
 284                                 if ( *audio_index < 0 )
 285                                         *audio_index = i;
 286                                 mlt_properties_set( meta_media, key, "audio" );
 287 #if (LIBAVCODEC_VERSION_INT >= ((51<<16)+(71<<8)+0))
 288                                 snprintf( key, sizeof(key), "meta.media.%d.codec.sample_fmt", i );
 289                                 mlt_properties_set( meta_media, key, avcodec_get_sample_fmt_name( codec_context->sample_fmt ) );
 290 #endif
 291                                 snprintf( key, sizeof(key), "meta.media.%d.codec.sample_rate", i );
 292                                 mlt_properties_set_int( meta_media, key, codec_context->sample_rate );
 293                                 snprintf( key, sizeof(key), "meta.media.%d.codec.channels", i );
 294                                 mlt_properties_set_int( meta_media, key, codec_context->channels );
 295                                 break;
 296                         default:
 297                                 break;
 298                 }
 299 //              snprintf( key, sizeof(key), "meta.media.%d.stream.time_base", i );
 300 //              mlt_properties_set_double( meta_media, key, av_q2d( context->streams[ i ]->time_base ) );
 301                 snprintf( key, sizeof(key), "meta.media.%d.codec.name", i );
 302                 mlt_properties_set( meta_media, key, codec->name );
 303 #if (LIBAVCODEC_VERSION_INT >= ((51<<16)+(55<<8)+0))
 304                 snprintf( key, sizeof(key), "meta.media.%d.codec.long_name", i );
 305                 mlt_properties_set( meta_media, key, codec->long_name );
 306 #endif
 307                 snprintf( key, sizeof(key), "meta.media.%d.codec.bit_rate", i );
 308                 mlt_properties_set_int( meta_media, key, codec_context->bit_rate );
 309 //              snprintf( key, sizeof(key), "meta.media.%d.codec.time_base", i );
 310 //              mlt_properties_set_double( meta_media, key, av_q2d( codec_context->time_base ) );
 311 //              snprintf( key, sizeof(key), "meta.media.%d.codec.profile", i );
 312 //              mlt_properties_set_int( meta_media, key, codec_context->profile );
 313 //              snprintf( key, sizeof(key), "meta.media.%d.codec.level", i );
 314 //              mlt_properties_set_int( meta_media, key, codec_context->level );
 315         }
 316
 317         return meta_media;
 318 }
 319
 320 static inline int dv_is_pal( AVPacket *pkt )
 321 {
 322         return pkt->data[3] & 0x80;
 323 }
 324
 325 static int dv_is_wide( AVPacket *pkt )
 326 {
 327         int i = 80 /* block size */ *3 /* VAUX starts at block 3 */ +3 /* skip block header */;
 328
 329         for ( ; i < pkt->size; i += 5 /* packet size */ )
 330         {
 331                 if ( pkt->data[ i ] == 0x61 )
 332                 {
 333                         uint8_t x = pkt->data[ i + 2 ] & 0x7;
 334                         return ( x == 2 ) || ( x == 7 );
 335                 }
 336         }
 337         return 0;
 338 }
 339
 340 static double get_aspect_ratio( AVStream *stream, AVCodecContext *codec_context, AVPacket *pkt )
 341 {
 342         double aspect_ratio = 1.0;
 343
 344         if ( codec_context->codec_id == CODEC_ID_DVVIDEO )
 345         {
 346                 if ( pkt )
 347                 {
 348                         if ( dv_is_pal( pkt ) )
 349                         {
 350                                 aspect_ratio = dv_is_wide( pkt )
 351                                         ? 64.0/45.0 // 16:9 PAL
 352                                         : 16.0/15.0; // 4:3 PAL
 353                         }
 354                         else
 355                         {
 356                                 aspect_ratio = dv_is_wide( pkt )
 357                                         ? 32.0/27.0 // 16:9 NTSC
 358                                         : 8.0/9.0; // 4:3 NTSC
 359                         }
 360                 }
 361                 else
 362                 {
 363                         AVRational ar =
 364 #if LIBAVFORMAT_VERSION_INT >= ((52<<16)+(21<<8)+0)
 365                                 stream->sample_aspect_ratio;
 366 #else
 367                                 codec_context->sample_aspect_ratio;
 368 #endif
 369                         // Override FFmpeg's notion of DV aspect ratios, which are
 370                         // based upon a width of 704. Since we do not have a normaliser
 371                         // that crops (nor is cropping 720 wide ITU-R 601 video always desirable)
 372                         // we just coerce the values to facilitate a passive behaviour through
 373                         // the rescale normaliser when using equivalent producers and consumers.
 374                         // = display_aspect / (width * height)
 375                         if ( ar.num == 10 && ar.den == 11 )
 376                                 aspect_ratio = 8.0/9.0; // 4:3 NTSC
 377                         else if ( ar.num == 59 && ar.den == 54 )
 378                                 aspect_ratio = 16.0/15.0; // 4:3 PAL
 379                         else if ( ar.num == 40 && ar.den == 33 )
 380                                 aspect_ratio = 32.0/27.0; // 16:9 NTSC
 381                         else if ( ar.num == 118 && ar.den == 81 )
 382                                 aspect_ratio = 64.0/45.0; // 16:9 PAL
 383                 }
 384         }
 385         else
 386         {
 387                 AVRational codec_sar = codec_context->sample_aspect_ratio;
 388                 AVRational stream_sar =
 389 #if LIBAVFORMAT_VERSION_INT >= ((52<<16)+(21<<8)+0)
 390                         stream->sample_aspect_ratio;
 391 #else
 392                         { 0, 1 };
 393 #endif
 394                 if ( codec_sar.num > 0 )
 395                         aspect_ratio = av_q2d( codec_sar );
 396                 else if ( stream_sar.num > 0 )
 397                         aspect_ratio = av_q2d( stream_sar );
 398         }
 399         return aspect_ratio;
 400 }
 401
 402 /** Open the file.
 403 */
 404
 405 static int producer_open( producer_avformat this, mlt_profile profile, char *file )
 406 {
 407         // Return an error code (0 == no error)
 408         int error = 0;
 409
 410         // Context for avformat
 411         AVFormatContext *context = NULL;
 412
 413         // Get the properties
 414         mlt_properties properties = MLT_PRODUCER_PROPERTIES( this->parent );
 415
 416         // We will treat everything with the producer fps
 417         double fps = mlt_profile_fps( profile );
 418
 419         // Lock the mutex now
 420         avformat_lock( );
 421
 422         // If "MRL", then create AVInputFormat
 423         AVInputFormat *format = NULL;
 424         AVFormatParameters *params = NULL;
 425         char *standard = NULL;
 426         char *mrl = strchr( file, ':' );
 427
 428         // AV option (0 = both, 1 = video, 2 = audio)
 429         int av = 0;
 430
 431         // Only if there is not a protocol specification that avformat can handle
 432         if ( mrl && !url_exist( file ) )
 433         {
 434                 // 'file' becomes format abbreviation
 435                 mrl[0] = 0;
 436
 437                 // Lookup the format
 438                 format = av_find_input_format( file );
 439
 440                 // Eat the format designator
 441                 file = ++mrl;
 442
 443                 if ( format )
 444                 {
 445                         // Allocate params
 446                         params = calloc( sizeof( AVFormatParameters ), 1 );
 447
 448                         // These are required by video4linux (defaults)
 449                         params->width = 640;
 450                         params->height = 480;
 451                         params->time_base= (AVRational){1,25};
 452                         // params->device = file;
 453                         params->channels = 2;
 454                         params->sample_rate = 48000;
 455                 }
 456
 457                 // Parse out params
 458                 mrl = strchr( file, '?' );
 459                 while ( mrl )
 460                 {
 461                         mrl[0] = 0;
 462                         char *name = strdup( ++mrl );
 463                         char *value = strchr( name, ':' );
 464                         if ( value )
 465                         {
 466                                 value[0] = 0;
 467                                 value++;
 468                                 char *t = strchr( value, '&' );
 469                                 if ( t )
 470                                         t[0] = 0;
 471                                 if ( !strcmp( name, "frame_rate" ) )
 472                                         params->time_base.den = atoi( value );
 473                                 else if ( !strcmp( name, "frame_rate_base" ) )
 474                                         params->time_base.num = atoi( value );
 475                                 else if ( !strcmp( name, "sample_rate" ) )
 476                                         params->sample_rate = atoi( value );
 477                                 else if ( !strcmp( name, "channels" ) )
 478                                         params->channels = atoi( value );
 479                                 else if ( !strcmp( name, "width" ) )
 480                                         params->width = atoi( value );
 481                                 else if ( !strcmp( name, "height" ) )
 482                                         params->height = atoi( value );
 483                                 else if ( !strcmp( name, "standard" ) )
 484                                 {
 485                                         standard = strdup( value );
 486                                         params->standard = standard;
 487                                 }
 488                                 else if ( !strcmp( name, "av" ) )
 489                                         av = atoi( value );
 490                         }
 491                         free( name );
 492                         mrl = strchr( mrl, '&' );
 493                 }
 494         }
 495
 496         // Now attempt to open the file
 497         error = av_open_input_file( &context, file, format, 0, params ) < 0;
 498
 499         // Cleanup AVFormatParameters
 500         free( standard );
 501         free( params );
 502
 503         // If successful, then try to get additional info
 504         if ( !error )
 505         {
 506                 // Get the stream info
 507                 error = av_find_stream_info( context ) < 0;
 508
 509                 // Continue if no error
 510                 if ( !error )
 511                 {
 512                         // We will default to the first audio and video streams found
 513                         int audio_index = -1;
 514                         int video_index = -1;
 515
 516                         // Now set properties where we can (use default unknowns if required)
 517                         if ( context->duration != AV_NOPTS_VALUE )
 518                         {
 519                                 // This isn't going to be accurate for all formats
 520                                 mlt_position frames = ( mlt_position )( ( ( double )context->duration / ( double )AV_TIME_BASE ) * fps + 0.5 );
 521                                 mlt_properties_set_position( properties, "out", frames - 1 );
 522                                 mlt_properties_set_position( properties, "length", frames );
 523                         }
 524
 525                         // Find default audio and video streams
 526                         find_default_streams( properties, context, &audio_index, &video_index );
 527
 528                         if ( context->start_time != AV_NOPTS_VALUE )
 529                                 this->start_time = context->start_time;
 530
 531                         // Check if we're seekable (something funny about mpeg here :-/)
 532                         if ( strncmp( file, "pipe:", 5 ) &&
 533                                  strncmp( file, "/dev/", 5 ) &&
 534                                  strncmp( file, "http:", 5 ) &&
 535                                  strncmp( file, "udp:", 4 )  &&
 536                                  strncmp( file, "tcp:", 4 )  &&
 537                                  strncmp( file, "rtsp:", 5 ) &&
 538                                  strncmp( file, "rtp:", 4 ) )
 539                         {
 540                                 this->seekable = av_seek_frame( context, -1, this->start_time, AVSEEK_FLAG_BACKWARD ) >= 0;
 541                                 mlt_properties_set_int( properties, "seekable", this->seekable );
 542                                 this->dummy_context = context;
 543                                 av_open_input_file( &context, file, NULL, 0, NULL );
 544                                 av_find_stream_info( context );
 545                         }
 546
 547                         // Store selected audio and video indexes on properties
 548                         this->audio_index = audio_index;
 549                         this->video_index = video_index;
 550                         this->first_pts = -1;
 551                         this->last_position = POSITION_INITIAL;
 552
 553                         // Fetch the width, height and aspect ratio
 554                         if ( video_index != -1 )
 555                         {
 556                                 AVCodecContext *codec_context = context->streams[ video_index ]->codec;
 557                                 mlt_properties_set_int( properties, "width", codec_context->width );
 558                                 mlt_properties_set_int( properties, "height", codec_context->height );
 559
 560                                 if ( codec_context->codec_id == CODEC_ID_DVVIDEO )
 561                                 {
 562                                         // Fetch the first frame of DV so we can read it directly
 563                                         AVPacket pkt;
 564                                         int ret = 0;
 565                                         while ( ret >= 0 )
 566                                         {
 567                                                 ret = av_read_frame( context, &pkt );
 568                                                 if ( ret >= 0 && pkt.stream_index == video_index && pkt.size > 0 )
 569                                                 {
 570                                                         mlt_properties_set_double( properties, "aspect_ratio",
 571                                                                 get_aspect_ratio( context->streams[ video_index ], codec_context, &pkt ) );
 572                                                         break;
 573                                                 }
 574                                         }
 575                                 }
 576                                 else
 577                                 {
 578                                         mlt_properties_set_double( properties, "aspect_ratio",
 579                                                 get_aspect_ratio( context->streams[ video_index ], codec_context, NULL ) );
 580                                 }
 581                         }
 582
 583                         // Read Metadata
 584                         if ( context->title )
 585                                 mlt_properties_set(properties, "meta.attr.title.markup", context->title );
 586                         if ( context->author )
 587                                 mlt_properties_set(properties, "meta.attr.author.markup", context->author );
 588                         if ( context->copyright )
 589                                 mlt_properties_set(properties, "meta.attr.copyright.markup", context->copyright );
 590                         if ( context->comment )
 591                                 mlt_properties_set(properties, "meta.attr.comment.markup", context->comment );
 592                         if ( context->album )
 593                                 mlt_properties_set(properties, "meta.attr.album.markup", context->album );
 594                         if ( context->year )
 595                                 mlt_properties_set_int(properties, "meta.attr.year.markup", context->year );
 596                         if ( context->track )
 597                                 mlt_properties_set_int(properties, "meta.attr.track.markup", context->track );
 598
 599                         // We're going to cheat here - for a/v files, we will have two contexts (reasoning will be clear later)
 600                         if ( av == 0 && audio_index != -1 && video_index != -1 )
 601                         {
 602                                 // We'll use the open one as our video_format
 603                                 this->video_format = context;
 604
 605                                 // And open again for our audio context
 606                                 av_open_input_file( &context, file, NULL, 0, NULL );
 607                                 av_find_stream_info( context );
 608
 609                                 // Audio context
 610                                 this->audio_format = context;
 611                         }
 612                         else if ( av != 2 && video_index != -1 )
 613                         {
 614                                 // We only have a video context
 615                                 this->video_format = context;
 616                         }
 617                         else if ( audio_index != -1 )
 618                         {
 619                                 // We only have an audio context
 620                                 this->audio_format = context;
 621                         }
 622                         else
 623                         {
 624                                 // Something has gone wrong
 625                                 error = -1;
 626                         }
 627                 }
 628         }
 629
 630         // Unlock the mutex now
 631         avformat_unlock( );
 632
 633         return error;
 634 }
 635
 636 /** Convert a frame position to a time code.
 637 */
 638
 639 static double producer_time_of_frame( mlt_producer this, mlt_position position )
 640 {
 641         return ( double )position / mlt_producer_get_fps( this );
 642 }
 643
 644                 // Collect information about all audio streams
 645
 646 static void get_audio_streams_info( producer_avformat this )
 647 {
 648         // Fetch the audio format context
 649         AVFormatContext *context = this->audio_format;
 650         int i;
 651
 652         for ( i = 0;
 653                   i < context->nb_streams;
 654                   i++ )
 655         {
 656                 if ( context->streams[i]->codec->codec_type == CODEC_TYPE_AUDIO )
 657                 {
 658                         AVCodecContext *codec_context = context->streams[i]->codec;
 659                         AVCodec *codec = avcodec_find_decoder( codec_context->codec_id );
 660
 661                         // If we don't have a codec and we can't initialise it, we can't do much more...
 662                         avformat_lock( );
 663                         if ( codec && avcodec_open( codec_context, codec ) >= 0 )
 664                         {
 665                                 this->audio_streams++;
 666                                 this->audio_max_stream = i;
 667                                 this->total_channels += codec_context->channels;
 668                                 if ( codec_context->channels > this->max_channel )
 669                                         this->max_channel = codec_context->channels;
 670                                 if ( codec_context->sample_rate > this->max_frequency )
 671                                         this->max_frequency = codec_context->sample_rate;
 672                                 avcodec_close( codec_context );
 673                         }
 674                         avformat_unlock( );
 675                 }
 676         }
 677         mlt_log_verbose( NULL, "[producer avformat] audio: total_streams %d max_stream %d total_channels %d max_channels %d\n",
 678                 this->audio_streams, this->audio_max_stream, this->total_channels, this->max_channel );
 679
 680         // Other audio-specific initializations
 681         this->resample_factor = 1.0;
 682 }
 683
 684 static inline void convert_image( void **ctx, AVFrame *frame, uint8_t *buffer, int pix_fmt, mlt_image_format *format, int width, int height )
 685 {
 686 #ifdef SWSCALE
 687         struct SwsContext **context = ctx;
 688         int flags = SWS_BILINEAR | SWS_ACCURATE_RND;
 689
 690 #ifdef USE_MMX
 691         flags |= SWS_CPU_CAPS_MMX;
 692 #endif
 693 #ifdef USE_SSE
 694         flags |= SWS_CPU_CAPS_MMX2;
 695 #endif
 696
 697         if ( pix_fmt == PIX_FMT_RGB32 )
 698         {
 699                 *format = mlt_image_rgb24a;
 700                 *context = sws_getCachedContext( *context, width, height, pix_fmt,
 701                         width, height, PIX_FMT_RGBA, flags, NULL, NULL, NULL);
 702                 AVPicture output;
 703                 avpicture_fill( &output, buffer, PIX_FMT_RGBA, width, height );
 704                 sws_scale( *context, frame->data, frame->linesize, 0, height,
 705                         output.data, output.linesize);
 706         }
 707         else if ( *format == mlt_image_yuv420p )
 708         {
 709                 *context = sws_getCachedContext( *context, width, height, pix_fmt,
 710                         width, height, PIX_FMT_YUV420P, flags, NULL, NULL, NULL);
 711                 AVPicture output;
 712                 output.data[0] = buffer;
 713                 output.data[1] = buffer + width * height;
 714                 output.data[2] = buffer + ( 5 * width * height ) / 4;
 715                 output.linesize[0] = width;
 716                 output.linesize[1] = width >> 1;
 717                 output.linesize[2] = width >> 1;
 718                 sws_scale( *context, frame->data, frame->linesize, 0, height,
 719                         output.data, output.linesize);
 720         }
 721         else if ( *format == mlt_image_rgb24 )
 722         {
 723                 *context = sws_getCachedContext( *context, width, height, pix_fmt,
 724                         width, height, PIX_FMT_RGB24, flags | SWS_FULL_CHR_H_INT, NULL, NULL, NULL);
 725                 AVPicture output;
 726                 avpicture_fill( &output, buffer, PIX_FMT_RGB24, width, height );
 727                 sws_scale( *context, frame->data, frame->linesize, 0, height,
 728                         output.data, output.linesize);
 729         }
 730         else if ( *format == mlt_image_rgb24a || *format == mlt_image_opengl )
 731         {
 732                 *context = sws_getCachedContext( *context, width, height, pix_fmt,
 733                         width, height, PIX_FMT_RGBA, flags | SWS_FULL_CHR_H_INT, NULL, NULL, NULL);
 734                 AVPicture output;
 735                 avpicture_fill( &output, buffer, PIX_FMT_RGBA, width, height );
 736                 sws_scale( *context, frame->data, frame->linesize, 0, height,
 737                         output.data, output.linesize);
 738         }
 739         else
 740         {
 741                 *context = sws_getCachedContext( *context, width, height, pix_fmt,
 742                         width, height, PIX_FMT_YUYV422, flags | SWS_FULL_CHR_H_INP, NULL, NULL, NULL);
 743                 AVPicture output;
 744                 avpicture_fill( &output, buffer, PIX_FMT_YUYV422, width, height );
 745                 sws_scale( *context, frame->data, frame->linesize, 0, height,
 746                         output.data, output.linesize);
 747         }
 748 #else
 749         if ( *format == mlt_image_yuv420p )
 750         {
 751                 AVPicture pict;
 752                 pict.data[0] = buffer;
 753                 pict.data[1] = buffer + width * height;
 754                 pict.data[2] = buffer + ( 5 * width * height ) / 4;
 755                 pict.linesize[0] = width;
 756                 pict.linesize[1] = width >> 1;
 757                 pict.linesize[2] = width >> 1;
 758                 img_convert( &pict, PIX_FMT_YUV420P, (AVPicture *)frame, pix_fmt, width, height );
 759         }
 760         else if ( *format == mlt_image_rgb24 )
 761         {
 762                 AVPicture output;
 763                 avpicture_fill( &output, buffer, PIX_FMT_RGB24, width, height );
 764                 img_convert( &output, PIX_FMT_RGB24, (AVPicture *)frame, pix_fmt, width, height );
 765         }
 766         else if ( *format == mlt_image_rgb24a || *format == mlt_image_opengl )
 767         {
 768                 AVPicture output;
 769                 avpicture_fill( &output, buffer, PIX_FMT_RGB32, width, height );
 770                 img_convert( &output, PIX_FMT_RGB32, (AVPicture *)frame, pix_fmt, width, height );
 771         }
 772         else
 773         {
 774                 AVPicture output;
 775                 avpicture_fill( &output, buffer, PIX_FMT_YUYV422, width, height );
 776                 img_convert( &output, PIX_FMT_YUYV422, (AVPicture *)frame, pix_fmt, width, height );
 777         }
 778 #endif
 779 }
 780
 781 /** Allocate the image buffer and set it on the frame.
 782 */
 783
 784 static int allocate_buffer( mlt_properties frame_properties, AVCodecContext *codec_context, uint8_t **buffer, mlt_image_format *format, int *width, int *height )
 785 {
 786         int size = 0;
 787
 788         if ( codec_context->width == 0 || codec_context->height == 0 )
 789                 return size;
 790
 791         *width = codec_context->width;
 792         *height = codec_context->height;
 793
 794         if ( codec_context->pix_fmt == PIX_FMT_RGB32 )
 795                 size = *width * ( *height + 1 ) * 4;
 796         else switch ( *format )
 797         {
 798                 case mlt_image_yuv420p:
 799                         size = *width * 3 * ( *height + 1 ) / 2;
 800                         break;
 801                 case mlt_image_rgb24:
 802                         size = *width * ( *height + 1 ) * 3;
 803                         break;
 804                 case mlt_image_rgb24a:
 805                 case mlt_image_opengl:
 806                         size = *width * ( *height + 1 ) * 4;
 807                         break;
 808                 default:
 809                         *format = mlt_image_yuv422;
 810                         size = *width * ( *height + 1 ) * 2;
 811                         break;
 812         }
 813
 814         // Construct the output image
 815         *buffer = mlt_pool_alloc( size );
 816         if ( *buffer )
 817                 mlt_properties_set_data( frame_properties, "image", *buffer, size, mlt_pool_release, NULL );
 818         else
 819                 size = 0;
 820
 821         return size;
 822 }
 823
 824 /** Get an image from a frame.
 825 */
 826
 827 static int producer_get_image( mlt_frame frame, uint8_t **buffer, mlt_image_format *format, int *width, int *height, int writable )
 828 {
 829         // Get the producer
 830         producer_avformat this = mlt_frame_pop_service( frame );
 831         mlt_producer producer = this->parent;
 832
 833         // Get the properties from the frame
 834         mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame );
 835
 836         // Obtain the frame number of this frame
 837         mlt_position position = mlt_properties_get_position( frame_properties, "avformat_position" );
 838
 839         // Get the producer properties
 840         mlt_properties properties = MLT_PRODUCER_PROPERTIES( producer );
 841
 842         // Fetch the video format context
 843         AVFormatContext *context = this->video_format;
 844
 845         // Get the video stream
 846         AVStream *stream = context->streams[ this->video_index ];
 847
 848         // Get codec context
 849         AVCodecContext *codec_context = stream->codec;
 850
 851         // Get the image cache
 852         if ( ! this->image_cache && ! mlt_properties_get_int( properties, "noimagecache" ) )
 853                 this->image_cache = mlt_cache_init();
 854         if ( this->image_cache )
 855         {
 856                 mlt_cache_item item = mlt_cache_get( this->image_cache, (void*) position );
 857                 *buffer = mlt_cache_item_data( item, (int*) format );
 858                 if ( *buffer )
 859                 {
 860                         // Set the resolution
 861                         *width = codec_context->width;
 862                         *height = codec_context->height;
 863
 864                         // Workaround 1088 encodings missing cropping info.
 865                         if ( *height == 1088 && mlt_profile_dar( mlt_service_profile( MLT_PRODUCER_SERVICE( producer ) ) ) == 16.0/9.0 )
 866                                 *height = 1080;
 867
 868                         // Cache hit
 869                         int size;
 870                         switch ( *format )
 871                         {
 872                                 case mlt_image_yuv420p:
 873                                         size = *width * 3 * ( *height + 1 ) / 2;
 874                                         break;
 875                                 case mlt_image_rgb24:
 876                                         size = *width * ( *height + 1 ) * 3;
 877                                         break;
 878                                 case mlt_image_rgb24a:
 879                                 case mlt_image_opengl:
 880                                         size = *width * ( *height + 1 ) * 4;
 881                                         break;
 882                                 default:
 883                                         *format = mlt_image_yuv422;
 884                                         size = *width * ( *height + 1 ) * 2;
 885                                         break;
 886                         }
 887                         mlt_properties_set_data( frame_properties, "avformat.image_cache", item, 0, ( mlt_destructor )mlt_cache_item_close, NULL );
 888                         mlt_properties_set_data( frame_properties, "image", *buffer, size, NULL, NULL );
 889                         this->top_field_first = mlt_properties_get_int( frame_properties, "top_field_first" );
 890                         this->got_picture = 1;
 891
 892                         goto exit_get_image;
 893                 }
 894         }
 895         // Cache miss
 896         int image_size = 0;
 897
 898         avformat_lock();
 899
 900         // Packet
 901         AVPacket pkt;
 902
 903         // Special case pause handling flag
 904         int paused = 0;
 905
 906         // Special case ffwd handling
 907         int ignore = 0;
 908
 909         // We may want to use the source fps if available
 910         double source_fps = mlt_properties_get_double( properties, "source_fps" );
 911         double fps = mlt_producer_get_fps( producer );
 912
 913         // This is the physical frame position in the source
 914         int req_position = ( int )( position / fps * source_fps + 0.5 );
 915
 916         // Determines if we have to decode all frames in a sequence
 917         // Temporary hack to improve intra frame only
 918         int must_decode = strcmp( codec_context->codec->name, "dnxhd" ) &&
 919                                   strcmp( codec_context->codec->name, "dvvideo" ) &&
 920                                   strcmp( codec_context->codec->name, "huffyuv" ) &&
 921                                   strcmp( codec_context->codec->name, "mjpeg" ) &&
 922                                   strcmp( codec_context->codec->name, "rawvideo" );
 923
 924         int last_position = this->last_position;
 925
 926         // Turn on usage of new seek API and PTS for seeking
 927         int use_new_seek = codec_context->codec_id == CODEC_ID_H264 && !strcmp( context->iformat->name, "mpegts" );
 928         if ( mlt_properties_get( properties, "new_seek" ) )
 929                 use_new_seek = mlt_properties_get_int( properties, "new_seek" );
 930
 931         // Seek if necessary
 932         if ( position != this->video_expected || last_position < 0 )
 933         {
 934                 if ( this->av_frame && position + 1 == this->video_expected )
 935                 {
 936                         // We're paused - use last image
 937                         paused = 1;
 938                 }
 939                 else if ( !this->seekable && position > this->video_expected && ( position - this->video_expected ) < 250 )
 940                 {
 941                         // Fast forward - seeking is inefficient for small distances - just ignore following frames
 942                         ignore = ( int )( ( position - this->video_expected ) / fps * source_fps );
 943                         codec_context->skip_loop_filter = AVDISCARD_NONREF;
 944                 }
 945                 else if ( this->seekable && ( position < this->video_expected || position - this->video_expected >= 12 || last_position < 0 ) )
 946                 {
 947                         if ( use_new_seek && last_position == POSITION_INITIAL )
 948                         {
 949                                 // find first key frame
 950                                 int ret = 0;
 951                                 int toscan = 100;
 952
 953                                 while ( ret >= 0 && toscan-- > 0 )
 954                                 {
 955                                         ret = av_read_frame( context, &pkt );
 956                                         if ( ret >= 0 && ( pkt.flags & PKT_FLAG_KEY ) && pkt.stream_index == this->video_index )
 957                                         {
 958                                                 mlt_log_verbose( MLT_PRODUCER_SERVICE(producer), "first_pts %lld dts %lld pts_dts_delta %d\n", pkt.pts, pkt.dts, (int)(pkt.pts - pkt.dts) );
 959                                                 this->first_pts = pkt.pts;
 960                                                 toscan = 0;
 961                                         }
 962                                         av_free_packet( &pkt );
 963                                 }
 964                                 // Rewind
 965                                 av_seek_frame( context, -1, 0, AVSEEK_FLAG_BACKWARD );
 966                         }
 967
 968                         // Calculate the timestamp for the requested frame
 969                         int64_t timestamp;
 970                         if ( use_new_seek )
 971                         {
 972                                 timestamp = ( req_position - 0.1 / source_fps ) /
 973                                         ( av_q2d( stream->time_base ) * source_fps );
 974                                 mlt_log_verbose( MLT_PRODUCER_SERVICE(producer), "pos %d pts %lld ", req_position, timestamp );
 975                                 if ( this->first_pts > 0 )
 976                                         timestamp += this->first_pts;
 977                                 else if ( context->start_time != AV_NOPTS_VALUE )
 978                                         timestamp += context->start_time;
 979                         }
 980                         else
 981                         {
 982                                 timestamp = ( int64_t )( ( double )req_position / source_fps * AV_TIME_BASE + 0.5 );
 983                                 if ( context->start_time != AV_NOPTS_VALUE )
 984                                         timestamp += context->start_time;
 985                         }
 986                         if ( must_decode )
 987                                 timestamp -= AV_TIME_BASE;
 988                         if ( timestamp < 0 )
 989                                 timestamp = 0;
 990                         mlt_log_debug( MLT_PRODUCER_SERVICE(producer), "seeking timestamp %lld position %d expected %d last_pos %d\n",
 991                                 timestamp, position, this->video_expected, last_position );
 992
 993                         // Seek to the timestamp
 994                         if ( use_new_seek )
 995                         {
 996                                 codec_context->skip_loop_filter = AVDISCARD_NONREF;
 997                                 av_seek_frame( context, this->video_index, timestamp, AVSEEK_FLAG_BACKWARD );
 998                         }
 999                         else
1000                         {
1001                                 av_seek_frame( context, -1, timestamp, AVSEEK_FLAG_BACKWARD );
1002                         }
1003
1004                         // Remove the cached info relating to the previous position
1005                         this->current_position = POSITION_INVALID;
1006                         this->last_position = POSITION_INVALID;
1007                         av_freep( &this->av_frame );
1008
1009                         if ( use_new_seek )
1010                         {
1011                                 // flush any pictures still in decode buffer
1012                                 avcodec_flush_buffers( codec_context );
1013                         }
1014                 }
1015         }
1016
1017         // Duplicate the last image if necessary (see comment on rawvideo below)
1018         if ( this->av_frame && this->av_frame->linesize[0] && this->got_picture && this->seekable
1019                  && ( paused
1020                           || this->current_position == req_position
1021                           || ( !use_new_seek && this->current_position > req_position ) ) )
1022         {
1023                 // Duplicate it
1024                 if ( ( image_size = allocate_buffer( frame_properties, codec_context, buffer, format, width, height ) ) )
1025                 {
1026                         // Workaround 1088 encodings missing cropping info.
1027                         if ( *height == 1088 && mlt_profile_dar( mlt_service_profile( MLT_PRODUCER_SERVICE( producer ) ) ) == 16.0/9.0 )
1028                                 *height = 1080;
1029 #ifdef VDPAU
1030                         if ( this->vdpau && this->vdpau->buffer )
1031                         {
1032                                 AVPicture picture;
1033                                 picture.data[0] = this->vdpau->buffer;
1034                                 picture.data[2] = this->vdpau->buffer + codec_context->width * codec_context->height;
1035                                 picture.data[1] = this->vdpau->buffer + codec_context->width * codec_context->height * 5 / 4;
1036                                 picture.linesize[0] = codec_context->width;
1037                                 picture.linesize[1] = codec_context->width / 2;
1038                                 picture.linesize[2] = codec_context->width / 2;
1039                                 convert_image( (void**) &this->swscale, (AVFrame*) &picture, *buffer, PIX_FMT_YUV420P, format, *width, *height );
1040                         }
1041                         else
1042 #endif
1043                         convert_image( (void**) &this->swscale, this->av_frame, *buffer, codec_context->pix_fmt, format, *width, *height );
1044                 }
1045                 else
1046                         mlt_frame_get_image( frame, buffer, format, width, height, writable );
1047         }
1048         else
1049         {
1050                 int ret = 0;
1051                 int int_position = 0;
1052                 int decode_errors = 0;
1053                 int got_picture = 0;
1054
1055                 av_init_packet( &pkt );
1056
1057                 // Construct an AVFrame for YUV422 conversion
1058                 if ( !this->av_frame )
1059                         this->av_frame = avcodec_alloc_frame( );
1060
1061                 while( ret >= 0 && !got_picture )
1062                 {
1063                         // Read a packet
1064                         ret = av_read_frame( context, &pkt );
1065
1066                         // We only deal with video from the selected video_index
1067                         if ( ret >= 0 && pkt.stream_index == this->video_index && pkt.size > 0 )
1068                         {
1069                                 // Determine time code of the packet
1070                                 if ( use_new_seek )
1071                                 {
1072                                         int64_t pts = pkt.pts;
1073                                         if ( this->first_pts > 0 )
1074                                                 pts -= this->first_pts;
1075                                         else if ( context->start_time != AV_NOPTS_VALUE )
1076                                                 pts -= context->start_time;
1077                                         int_position = ( int )( av_q2d( stream->time_base ) * pts * source_fps + 0.1 );
1078                                         if ( pkt.pts == AV_NOPTS_VALUE )
1079                                         {
1080                                                 this->invalid_pts_counter++;
1081                                                 if ( this->invalid_pts_counter > 20 )
1082                                                 {
1083                                                         mlt_log_panic( MLT_PRODUCER_SERVICE(producer), "\ainvalid PTS; DISABLING NEW_SEEK!\n" );
1084                                                         mlt_properties_set_int( properties, "new_seek", 0 );
1085                                                         int_position = req_position;
1086                                                         use_new_seek = 0;
1087                                                 }
1088                                         }
1089                                         else
1090                                         {
1091                                                 this->invalid_pts_counter = 0;
1092                                         }
1093                                         mlt_log_debug( MLT_PRODUCER_SERVICE(producer), "pkt.pts %llu req_pos %d cur_pos %d pkt_pos %d\n",
1094                                                 pkt.pts, req_position, this->current_position, int_position );
1095                                 }
1096                                 else
1097                                 {
1098                                         if ( pkt.dts != AV_NOPTS_VALUE )
1099                                         {
1100                                                 int_position = ( int )( av_q2d( stream->time_base ) * pkt.dts * source_fps + 0.5 );
1101                                                 if ( context->start_time != AV_NOPTS_VALUE )
1102                                                         int_position -= ( int )( context->start_time * source_fps / AV_TIME_BASE + 0.5 );
1103                                                 last_position = this->last_position;
1104                                                 if ( int_position == last_position )
1105                                                         int_position = last_position + 1;
1106                                         }
1107                                         else
1108                                         {
1109                                                 int_position = req_position;
1110                                         }
1111                                         mlt_log_debug( MLT_PRODUCER_SERVICE(producer), "pkt.dts %llu req_pos %d cur_pos %d pkt_pos %d\n",
1112                                                 pkt.dts, req_position, this->current_position, int_position );
1113                                         // Make a dumb assumption on streams that contain wild timestamps
1114                                         if ( abs( req_position - int_position ) > 999 )
1115                                         {
1116                                                 int_position = req_position;
1117                                                 mlt_log_debug( MLT_PRODUCER_SERVICE(producer), " WILD TIMESTAMP!" );
1118                                         }
1119                                 }
1120                                 this->last_position = int_position;
1121
1122                                 // Decode the image
1123                                 if ( must_decode || int_position >= req_position )
1124                                 {
1125 #ifdef VDPAU
1126                                         if ( g_vdpau && this->vdpau )
1127                                         {
1128                                                 if ( g_vdpau->producer != this )
1129                                                 {
1130                                                         vdpau_decoder_close();
1131                                                         vdpau_decoder_init( this );
1132                                                 }
1133                                                 if ( this->vdpau )
1134                                                         this->vdpau->is_decoded = 0;
1135                                         }
1136 #endif
1137                                         codec_context->reordered_opaque = pkt.pts;
1138                                         if ( int_position >= req_position )
1139                                                 codec_context->skip_loop_filter = AVDISCARD_NONE;
1140 #if (LIBAVCODEC_VERSION_INT >= ((52<<16)+(26<<8)+0))
1141                                         ret = avcodec_decode_video2( codec_context, this->av_frame, &got_picture, &pkt );
1142 #else
1143                                         ret = avcodec_decode_video( codec_context, this->av_frame, &got_picture, pkt.data, pkt.size );
1144 #endif
1145                                         // Note: decode may fail at the beginning of MPEGfile (B-frames referencing before first I-frame), so allow a few errors.
1146                                         if ( ret < 0 )
1147                                         {
1148                                                 if ( ++decode_errors <= 10 )
1149                                                         ret = 0;
1150                                         }
1151                                         else
1152                                         {
1153                                                 decode_errors = 0;
1154                                         }
1155                                 }
1156
1157                                 if ( got_picture )
1158                                 {
1159                                         if ( use_new_seek )
1160                                         {
1161                                                 // Determine time code of the packet
1162                                                 int64_t pts = this->av_frame->reordered_opaque;
1163                                                 if ( this->first_pts > 0 )
1164                                                         pts -= this->first_pts;
1165                                                 else if ( context->start_time != AV_NOPTS_VALUE )
1166                                                         pts -= context->start_time;
1167                                                 int_position = ( int )( av_q2d( stream->time_base) * pts * source_fps + 0.1 );
1168                                                 mlt_log_debug( MLT_PRODUCER_SERVICE(producer), "got frame %d, key %d\n", int_position, this->av_frame->key_frame );
1169                                         }
1170                                         // Handle ignore
1171                                         if ( int_position < req_position )
1172                                         {
1173                                                 ignore = 0;
1174                                                 got_picture = 0;
1175                                         }
1176                                         else if ( int_position >= req_position )
1177                                         {
1178                                                 ignore = 0;
1179                                                 codec_context->skip_loop_filter = AVDISCARD_NONE;
1180                                         }
1181                                         else if ( ignore -- )
1182                                         {
1183                                                 got_picture = 0;
1184                                         }
1185                                 }
1186                                 mlt_log_debug( MLT_PRODUCER_SERVICE(producer), " got_pic %d key %d\n", got_picture, pkt.flags & PKT_FLAG_KEY );
1187                                 av_free_packet( &pkt );
1188                         }
1189                         else if ( ret >= 0 )
1190                         {
1191                                 av_free_packet( &pkt );
1192                         }
1193
1194                         // Now handle the picture if we have one
1195                         if ( got_picture )
1196                         {
1197                                 if ( ( image_size = allocate_buffer( frame_properties, codec_context, buffer, format, width, height ) ) )
1198                                 {
1199                                         // Workaround 1088 encodings missing cropping info.
1200                                         if ( *height == 1088 && mlt_profile_dar( mlt_service_profile( MLT_PRODUCER_SERVICE( producer ) ) ) == 16.0/9.0 )
1201                                                 *height = 1080;
1202 #ifdef VDPAU
1203                                         if ( this->vdpau )
1204                                         {
1205                                                 if ( this->vdpau->is_decoded )
1206                                                 {
1207                                                         struct vdpau_render_state *render = (struct vdpau_render_state*) this->av_frame->data[0];
1208                                                         void *planes[3];
1209                                                         uint32_t pitches[3];
1210                                                         VdpYCbCrFormat dest_format = VDP_YCBCR_FORMAT_YV12;
1211                                                         AVPicture picture;
1212
1213                                                         if ( !this->vdpau->buffer )
1214                                                                 this->vdpau->buffer = mlt_pool_alloc( codec_context->width * codec_context->height * 3 / 2 );
1215                                                         picture.data[0] = planes[0] = this->vdpau->buffer;
1216                                                         picture.data[2] = planes[1] = this->vdpau->buffer + codec_context->width * codec_context->height;
1217                                                         picture.data[1] = planes[2] = this->vdpau->buffer + codec_context->width * codec_context->height * 5 / 4;
1218                                                         picture.linesize[0] = pitches[0] = codec_context->width;
1219                                                         picture.linesize[1] = pitches[1] = codec_context->width / 2;
1220                                                         picture.linesize[2] = pitches[2] = codec_context->width / 2;
1221
1222                                                         VdpStatus status = vdp_surface_get_bits( render->surface, dest_format, planes, pitches );
1223                                                         if ( status == VDP_STATUS_OK )
1224                                                         {
1225                                                                 convert_image( (void**) &this->swscale, (AVFrame*) &picture, *buffer, PIX_FMT_YUV420P, format, *width, *height );
1226                                                         }
1227                                                         else
1228                                                         {
1229                                                                 mlt_log_error( MLT_PRODUCER_SERVICE(producer), "VDPAU Error: %s\n", vdp_get_error_string( status ) );
1230                                                                 this->vdpau->is_decoded = 0;
1231                                                         }
1232                                                 }
1233                                                 else
1234                                                 {
1235                                                         mlt_log_error( MLT_PRODUCER_SERVICE(producer), "VDPAU error in VdpDecoderRender\n" );
1236                                                         got_picture = 0;
1237                                                 }
1238                                         }
1239                                         else
1240 #endif
1241                                         convert_image( (void**) &this->swscale, this->av_frame, *buffer, codec_context->pix_fmt, format, *width, *height );
1242                                         this->top_field_first |= this->av_frame->top_field_first;
1243                                         this->current_position = int_position;
1244                                         this->got_picture = 1;
1245                                 }
1246                                 else
1247                                 {
1248                                         got_picture = 0;
1249                                 }
1250                         }
1251                 }
1252         }
1253
1254         avformat_unlock();
1255
1256         if ( this->got_picture && image_size > 0 && this->image_cache )
1257         {
1258                 // Copy buffer to image cache
1259                 uint8_t *image = mlt_pool_alloc( image_size );
1260                 memcpy( image, *buffer, image_size );
1261                 mlt_cache_put( this->image_cache, (void*) position, image, *format, mlt_pool_release );
1262         }
1263
1264         // Regardless of speed, we expect to get the next frame (cos we ain't too bright)
1265         this->video_expected = position + 1;
1266
1267 exit_get_image:
1268         // Set the progressive flag
1269         if ( mlt_properties_get( properties, "force_progressive" ) )
1270                 mlt_properties_set_int( frame_properties, "progressive", !!mlt_properties_get_int( properties, "force_progressive" ) );
1271         else if ( this->av_frame )
1272                 mlt_properties_set_int( frame_properties, "progressive", !this->av_frame->interlaced_frame );
1273
1274         // Set the field order property for this frame
1275         mlt_properties_set_int( frame_properties, "top_field_first", this->top_field_first );
1276
1277         return !this->got_picture;
1278 }
1279
1280 /** Process properties as AVOptions and apply to AV context obj
1281 */
1282
1283 static void apply_properties( void *obj, mlt_properties properties, int flags )
1284 {
1285         int i;
1286         int count = mlt_properties_count( properties );
1287         for ( i = 0; i < count; i++ )
1288         {
1289                 const char *opt_name = mlt_properties_get_name( properties, i );
1290                 const AVOption *opt = av_find_opt( obj, opt_name, NULL, flags, flags );
1291                 if ( opt_name && mlt_properties_get( properties, opt_name ) )
1292                 {
1293                         if ( opt )
1294 #if LIBAVCODEC_VERSION_INT >= ((52<<16)+(7<<8)+0)
1295                                 av_set_string3( obj, opt_name, mlt_properties_get( properties, opt_name), 0, NULL );
1296 #elif LIBAVCODEC_VERSION_INT >= ((51<<16)+(59<<8)+0)
1297                                 av_set_string2( obj, opt_name, mlt_properties_get( properties, opt_name), 0 );
1298 #else
1299                                 av_set_string( obj, opt_name, mlt_properties_get( properties, opt_name) );
1300 #endif
1301                 }
1302         }
1303 }
1304
1305 /** Initialize the video codec context.
1306  */
1307
1308 static int video_codec_init( producer_avformat this, int index, mlt_properties properties )
1309 {
1310         // Initialise the codec if necessary
1311         if ( !this->video_codec )
1312         {
1313                 // Get the video stream
1314                 AVStream *stream = this->video_format->streams[ index ];
1315
1316                 // Get codec context
1317                 AVCodecContext *codec_context = stream->codec;
1318
1319                 // Find the codec
1320                 AVCodec *codec = avcodec_find_decoder( codec_context->codec_id );
1321 #ifdef VDPAU
1322                 if ( codec_context->codec_id == CODEC_ID_H264 )
1323                 {
1324                         if ( ( codec = avcodec_find_decoder_by_name( "h264_vdpau" ) ) )
1325                         {
1326                                 if ( vdpau_init( this ) )
1327                                 {
1328                                         this->video_codec = codec_context;
1329                                         if ( !vdpau_decoder_init( this ) )
1330                                                 vdpau_decoder_close();
1331                                 }
1332                         }
1333                         if ( !this->vdpau )
1334                                 codec = avcodec_find_decoder( codec_context->codec_id );
1335                 }
1336 #endif
1337
1338                 // Initialise multi-threading
1339                 int thread_count = mlt_properties_get_int( properties, "threads" );
1340                 if ( thread_count == 0 && getenv( "MLT_AVFORMAT_THREADS" ) )
1341                         thread_count = atoi( getenv( "MLT_AVFORMAT_THREADS" ) );
1342                 if ( thread_count > 1 )
1343                 {
1344                         avcodec_thread_init( codec_context, thread_count );
1345                         codec_context->thread_count = thread_count;
1346                 }
1347
1348                 // If we don't have a codec and we can't initialise it, we can't do much more...
1349                 avformat_lock( );
1350                 if ( codec && avcodec_open( codec_context, codec ) >= 0 )
1351                 {
1352                         // Now store the codec with its destructor
1353                         this->video_codec = codec_context;
1354                 }
1355                 else
1356                 {
1357                         // Remember that we can't use this later
1358                         this->video_index = -1;
1359                 }
1360                 avformat_unlock( );
1361
1362                 // Process properties as AVOptions
1363                 apply_properties( codec_context, properties, AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM );
1364
1365                 // Reset some image properties
1366                 mlt_properties_set_int( properties, "width", this->video_codec->width );
1367                 mlt_properties_set_int( properties, "height", this->video_codec->height );
1368                 // For DV, we'll just use the saved aspect ratio
1369                 if ( codec_context->codec_id != CODEC_ID_DVVIDEO )
1370                         mlt_properties_set_double( properties, "aspect_ratio", get_aspect_ratio( stream, this->video_codec, NULL ) );
1371
1372                 // Determine the fps first from the codec
1373                 double source_fps = (double) this->video_codec->time_base.den /
1374                                                                    ( this->video_codec->time_base.num == 0 ? 1 : this->video_codec->time_base.num );
1375
1376                 if ( mlt_properties_get( properties, "force_fps" ) )
1377                 {
1378                         source_fps = mlt_properties_get_double( properties, "force_fps" );
1379                         stream->time_base = av_d2q( source_fps, 255 );
1380                 }
1381                 else
1382                 {
1383                         // If the muxer reports a frame rate different than the codec
1384                         double muxer_fps = av_q2d( stream->r_frame_rate );
1385                         // Choose the lesser - the wrong tends to be off by some multiple of 10
1386                         source_fps = FFMIN( source_fps, muxer_fps );
1387                 }
1388
1389                 // We'll use fps if it's available
1390                 if ( source_fps > 0 )
1391                         mlt_properties_set_double( properties, "source_fps", source_fps );
1392                 else
1393                         mlt_properties_set_double( properties, "source_fps", mlt_producer_get_fps( this->parent ) );
1394         }
1395         return this->video_codec && this->video_index > -1;
1396 }
1397
1398 /** Set up video handling.
1399 */
1400
1401 static void producer_set_up_video( producer_avformat this, mlt_frame frame )
1402 {
1403         // Get the producer
1404         mlt_producer producer = this->parent;
1405
1406         // Get the properties
1407         mlt_properties properties = MLT_PRODUCER_PROPERTIES( producer );
1408
1409         // Fetch the video format context
1410         AVFormatContext *context = this->video_format;
1411
1412         // Get the video_index
1413         int index = mlt_properties_get_int( properties, "video_index" );
1414
1415         // Reopen the file if necessary
1416         if ( !context && index > -1 )
1417         {
1418                 mlt_events_block( properties, producer );
1419                 producer_open( this, mlt_service_profile( MLT_PRODUCER_SERVICE(producer) ),
1420                         mlt_properties_get( properties, "resource" ) );
1421                 context = this->video_format;
1422                 if ( this->dummy_context )
1423                 {
1424                         avformat_lock();
1425                         av_close_input_file( this->dummy_context );
1426                         avformat_unlock();
1427                 }
1428                 this->dummy_context = NULL;
1429                 mlt_events_unblock( properties, producer );
1430                 if ( this->audio_format )
1431                         get_audio_streams_info( this );
1432
1433                 // Process properties as AVOptions
1434                 apply_properties( context, properties, AV_OPT_FLAG_DECODING_PARAM );
1435         }
1436
1437         // Exception handling for video_index
1438         if ( context && index >= (int) context->nb_streams )
1439         {
1440                 // Get the last video stream
1441                 for ( index = context->nb_streams - 1;
1442                           index >= 0 && context->streams[ index ]->codec->codec_type != CODEC_TYPE_VIDEO;
1443                           index-- );
1444                 mlt_properties_set_int( properties, "video_index", index );
1445         }
1446         if ( context && index > -1 && context->streams[ index ]->codec->codec_type != CODEC_TYPE_VIDEO )
1447         {
1448                 // Invalidate the video stream
1449                 index = -1;
1450                 mlt_properties_set_int( properties, "video_index", index );
1451         }
1452
1453         // Update the video properties if the index changed
1454         if ( index != this->video_index )
1455         {
1456                 // Reset the video properties if the index changed
1457                 this->video_index = index;
1458                 if ( this->video_codec )
1459                 {
1460                         avformat_lock();
1461                         avcodec_close( this->video_codec );
1462                         avformat_unlock();
1463                 }
1464                 this->video_codec = NULL;
1465         }
1466
1467         // Get the frame properties
1468         mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame );
1469
1470         // Get the codec
1471         if ( context && index > -1 && video_codec_init( this, index, properties ) )
1472         {
1473                 // Set the frame properties
1474                 double force_aspect_ratio = mlt_properties_get_double( properties, "force_aspect_ratio" );
1475                 double aspect_ratio = ( force_aspect_ratio > 0.0 ) ?
1476                         force_aspect_ratio : mlt_properties_get_double( properties, "aspect_ratio" );
1477
1478                 // Set the width and height
1479                 mlt_properties_set_int( frame_properties, "width", this->video_codec->width );
1480                 mlt_properties_set_int( frame_properties, "height", this->video_codec->height );
1481                 mlt_properties_set_int( frame_properties, "real_width", this->video_codec->width );
1482                 mlt_properties_set_int( frame_properties, "real_height", this->video_codec->height );
1483                 mlt_properties_set_double( frame_properties, "aspect_ratio", aspect_ratio );
1484
1485                 // Workaround 1088 encodings missing cropping info.
1486                 if ( this->video_codec->height == 1088 && mlt_profile_dar( mlt_service_profile( MLT_PRODUCER_SERVICE( producer ) ) ) == 16.0/9.0 )
1487                         mlt_properties_set_int( frame_properties, "real_height", 1080 );
1488
1489                 // Add our image operation
1490                 mlt_frame_push_service( frame, this );
1491                 mlt_frame_push_get_image( frame, producer_get_image );
1492         }
1493         else
1494         {
1495                 // If something failed, use test card image
1496                 mlt_properties_set_int( frame_properties, "test_image", 1 );
1497         }
1498 }
1499
1500 static int seek_audio( producer_avformat this, mlt_position position, double timecode, int *ignore )
1501 {
1502         int paused = 0;
1503
1504         // Seek if necessary
1505         if ( position != this->audio_expected )
1506         {
1507                 if ( position + 1 == this->audio_expected )
1508                 {
1509                         // We're paused - silence required
1510                         paused = 1;
1511                 }
1512                 else if ( !this->seekable && position > this->audio_expected && ( position - this->audio_expected ) < 250 )
1513                 {
1514                         // Fast forward - seeking is inefficient for small distances - just ignore following frames
1515                         *ignore = position - this->audio_expected;
1516                 }
1517                 else if ( position < this->audio_expected || position - this->audio_expected >= 12 )
1518                 {
1519                         AVFormatContext *context = this->audio_format;
1520                         int64_t timestamp = ( int64_t )( timecode * AV_TIME_BASE + 0.5 );
1521                         if ( context->start_time != AV_NOPTS_VALUE )
1522                                 timestamp += context->start_time;
1523                         if ( timestamp < 0 )
1524                                 timestamp = 0;
1525
1526                         // Set to the real timecode
1527                         if ( av_seek_frame( context, -1, timestamp, AVSEEK_FLAG_BACKWARD ) != 0 )
1528                                 paused = 1;
1529
1530                         // Clear the usage in the audio buffer
1531                         int i = MAX_AUDIO_STREAMS + 1;
1532                         while ( --i )
1533                                 this->audio_used[i - 1] = 0;
1534                 }
1535         }
1536         return paused;
1537 }
1538
1539 static int decode_audio( producer_avformat this, int *ignore, AVPacket pkt, int channels, int samples, double timecode, double fps )
1540 {
1541         // Fetch the audio_format
1542         AVFormatContext *context = this->audio_format;
1543
1544         // Get the current stream index
1545         int index = pkt.stream_index;
1546
1547         // Get codec context
1548         AVCodecContext *codec_context = this->audio_codec[ index ];
1549
1550         // Obtain the resample context if it exists (not always needed)
1551         ReSampleContext *resample = this->audio_resample[ index ];
1552
1553         // Obtain the audio buffers
1554         int16_t *audio_buffer = this->audio_buffer[ index ];
1555         int16_t *decode_buffer = this->decode_buffer[ index ];
1556
1557         int audio_used = this->audio_used[ index ];
1558         uint8_t *ptr = pkt.data;
1559         int len = pkt.size;
1560         int ret = 0;
1561
1562         while ( ptr && ret >= 0 && len > 0 )
1563         {
1564                 int data_size = sizeof( int16_t ) * AVCODEC_MAX_AUDIO_FRAME_SIZE;
1565
1566                 // Decode the audio
1567 #if (LIBAVCODEC_VERSION_INT >= ((52<<16)+(26<<8)+0))
1568                 ret = avcodec_decode_audio3( codec_context, decode_buffer, &data_size, &pkt );
1569 #elif (LIBAVCODEC_VERSION_INT >= ((51<<16)+(29<<8)+0))
1570                 ret = avcodec_decode_audio2( codec_context, decode_buffer, &data_size, ptr, len );
1571 #else
1572                 ret = avcodec_decode_audio( codec_context, decode_buffer, &data_size, ptr, len );
1573 #endif
1574                 if ( ret < 0 )
1575                 {
1576                         mlt_log_warning( MLT_PRODUCER_SERVICE(this->parent), "audio decoding error %d\n", ret );
1577                         break;
1578                 }
1579
1580                 pkt.size = len -= ret;
1581                 pkt.data = ptr += ret;
1582
1583                 // If decoded successfully
1584                 if ( data_size > 0 )
1585                 {
1586                         // Figure out how many samples will be needed after resampling
1587                         int convert_samples = data_size / codec_context->channels / ( av_get_bits_per_sample_format( codec_context->sample_fmt ) / 8 );
1588                         int samples_needed = this->resample_factor * convert_samples + 1;
1589
1590                         // Resize audio buffer to prevent overflow
1591                         if ( audio_used * channels + samples_needed > this->audio_buffer_size[ index ] )
1592                         {
1593                                 this->audio_buffer_size[ index ] *= 2;
1594                                 audio_buffer = this->audio_buffer[ index ] = mlt_pool_realloc( audio_buffer, this->audio_buffer_size[ index ] * sizeof(int16_t) );
1595                         }
1596                         if ( resample )
1597                         {
1598                                 // Copy to audio buffer while resampling
1599                                 int16_t *source = decode_buffer;
1600                                 int16_t *dest = &audio_buffer[ audio_used * channels ];
1601                                 audio_used += audio_resample( resample, dest, source, convert_samples );
1602                         }
1603                         else
1604                         {
1605                                 // Straight copy to audio buffer
1606                                 memcpy( &audio_buffer[ audio_used * codec_context->channels ], decode_buffer, data_size );
1607                                 audio_used += convert_samples;
1608                         }
1609
1610                         // Handle ignore
1611                         while ( *ignore && audio_used > samples )
1612                         {
1613                                 *ignore -= 1;
1614                                 audio_used -= samples;
1615                                 memmove( audio_buffer, &audio_buffer[ samples * (resample? channels : codec_context->channels) ],
1616                                          audio_used * sizeof( int16_t ) );
1617                         }
1618                 }
1619         }
1620
1621         // If we're behind, ignore this packet
1622         if ( pkt.pts >= 0 )
1623         {
1624                 double current_pts = av_q2d( context->streams[ index ]->time_base ) * pkt.pts;
1625                 int req_position = ( int )( timecode * fps + 0.5 );
1626                 int int_position = ( int )( current_pts * fps + 0.5 );
1627                 if ( context->start_time != AV_NOPTS_VALUE )
1628                         int_position -= ( int )( fps * context->start_time / AV_TIME_BASE + 0.5 );
1629
1630                 if ( this->seekable && *ignore == 0 )
1631                 {
1632                         if ( int_position < req_position )
1633                                 // We are behind, so skip some
1634                                 *ignore = 1;
1635                         else if ( int_position > req_position + 2 )
1636                                 // We are ahead, so seek backwards some more
1637                                 seek_audio( this, req_position, timecode - 1.0, ignore );
1638                 }
1639         }
1640
1641         this->audio_used[ index ] = audio_used;
1642
1643         return ret;
1644 }
1645
1646 /** Get the audio from a frame.
1647 */
1648
1649 static int producer_get_audio( mlt_frame frame, void **buffer, mlt_audio_format *format, int *frequency, int *channels, int *samples )
1650 {
1651         // Get the producer
1652         producer_avformat this = mlt_frame_pop_audio( frame );
1653
1654         // Obtain the frame number of this frame
1655         mlt_position position = mlt_properties_get_position( MLT_FRAME_PROPERTIES( frame ), "avformat_position" );
1656
1657         // Calculate the real time code
1658         double real_timecode = producer_time_of_frame( this->parent, position );
1659
1660         // Get the producer fps
1661         double fps = mlt_producer_get_fps( this->parent );
1662
1663         // Number of frames to ignore (for ffwd)
1664         int ignore = 0;
1665
1666         // Flag for paused (silence)
1667         int paused = seek_audio( this, position, real_timecode, &ignore );
1668
1669         // Fetch the audio_format
1670         AVFormatContext *context = this->audio_format;
1671
1672         // Determine the tracks to use
1673         int index = this->audio_index;
1674         int index_max = this->audio_index + 1;
1675         if ( this->audio_index == INT_MAX )
1676         {
1677                 index = 0;
1678                 index_max = context->nb_streams;
1679                 *channels = this->total_channels;
1680                 *frequency = this->max_frequency;
1681         }
1682
1683         // Initialize the resamplers and buffers
1684         for ( ; index < index_max; index++ )
1685         {
1686                 // Get codec context
1687                 AVCodecContext *codec_context = this->audio_codec[ index ];
1688
1689                 if ( codec_context && !this->audio_buffer[ index ] )
1690                 {
1691                         // Check for resample and create if necessary
1692                         if ( codec_context->channels <= 2 )
1693                         {
1694                                 // Determine by how much resampling will increase number of samples
1695                                 double resample_factor = this->audio_index == INT_MAX ? 1 : (double) *channels / codec_context->channels;
1696                                 resample_factor *= (double) *frequency / codec_context->sample_rate;
1697                                 if ( resample_factor > this->resample_factor )
1698                                         this->resample_factor = resample_factor;
1699
1700                                 // Create the resampler
1701 #if (LIBAVCODEC_VERSION_INT >= ((52<<16)+(15<<8)+0))
1702                                 this->audio_resample[ index ] = av_audio_resample_init(
1703                                         this->audio_index == INT_MAX ? codec_context->channels : *channels,
1704                                         codec_context->channels, *frequency, codec_context->sample_rate,
1705                                         SAMPLE_FMT_S16, codec_context->sample_fmt, 16, 10, 0, 0.8 );
1706 #else
1707                                 this->audio_resample[ index ] = audio_resample_init(
1708                                         this->audio_index == INT_MAX ? codec_context->channels : *channels,
1709                                         codec_context->channels, *frequency, codec_context->sample_rate );
1710 #endif
1711                         }
1712                         else
1713                         {
1714                                 codec_context->request_channels = this->audio_index == INT_MAX ? codec_context->channels : *channels;
1715                         }
1716
1717                         // Check for audio buffer and create if necessary
1718                         this->audio_buffer_size[ index ] = AVCODEC_MAX_AUDIO_FRAME_SIZE;
1719                         this->audio_buffer[ index ] = mlt_pool_alloc( this->audio_buffer_size[ index ] * sizeof( int16_t ) );
1720
1721                         // Check for decoder buffer and create if necessary
1722                         this->decode_buffer[ index ] = av_malloc( AVCODEC_MAX_AUDIO_FRAME_SIZE * sizeof( int16_t ) );
1723                 }
1724         }
1725
1726         // Get the audio if required
1727         if ( !paused )
1728         {
1729                 int ret = 0;
1730                 int got_audio = 0;
1731                 AVPacket pkt;
1732
1733                 av_init_packet( &pkt );
1734
1735                 // If not resampling, give consumer more than requested.
1736                 // It requested number samples based on requested frame rate.
1737                 // Do not clean this up with a samples *= ...!
1738                 if ( this->audio_index != INT_MAX && ! this->audio_resample[ this->audio_index ] )
1739                         *samples = *samples * this->audio_codec[ this->audio_index ]->sample_rate / *frequency;
1740
1741                 while ( ret >= 0 && !got_audio )
1742                 {
1743                         // Check if the buffer already contains the samples required
1744                         if ( this->audio_index != INT_MAX && this->audio_used[ this->audio_index ] >= *samples && ignore == 0 )
1745                         {
1746                                 got_audio = 1;
1747                                 break;
1748                         }
1749
1750                         // Read a packet
1751                         ret = av_read_frame( context, &pkt );
1752
1753                         // We only deal with audio from the selected audio index
1754                         if ( ret >= 0 && pkt.data && pkt.size > 0 && ( pkt.stream_index == this->audio_index ||
1755                                  ( this->audio_index == INT_MAX && context->streams[ pkt.stream_index ]->codec->codec_type == CODEC_TYPE_AUDIO ) ) )
1756                         {
1757                                 int channels2 = this->audio_index == INT_MAX ? this->audio_codec[pkt.stream_index]->channels : *channels;
1758                                 ret = decode_audio( this, &ignore, pkt, channels2, *samples, real_timecode, fps );
1759                         }
1760                         av_free_packet( &pkt );
1761
1762                         if ( this->audio_index == INT_MAX && ret >= 0 )
1763                         {
1764                                 // Determine if there is enough audio for all streams
1765                                 got_audio = 1;
1766                                 for ( index = 0; index < context->nb_streams; index++ )
1767                                 {
1768                                         if ( this->audio_codec[ index ] && this->audio_used[ index ] < *samples )
1769                                                 got_audio = 0;
1770                                 }
1771                         }
1772                 }
1773
1774                 // Allocate and set the frame's audio buffer
1775                 int size = *samples * *channels * sizeof(int16_t);
1776                 *buffer = mlt_pool_alloc( size );
1777                 *format = mlt_audio_s16;
1778                 mlt_frame_set_audio( frame, *buffer, *format, size, mlt_pool_release );
1779
1780                 // Interleave tracks if audio_index=all
1781                 if ( this->audio_index == INT_MAX )
1782                 {
1783                         int16_t *dest = *buffer;
1784                         int i;
1785                         for ( i = 0; i < *samples; i++ )
1786                         {
1787                                 for ( index = 0; index < index_max; index++ )
1788                                 if ( this->audio_codec[ index ] )
1789                                 {
1790                                         int current_channels = this->audio_codec[ index ]->channels;
1791                                         int16_t *src = this->audio_buffer[ index ] + i * current_channels;
1792                                         memcpy( dest, src, current_channels * sizeof(int16_t) );
1793                                         dest += current_channels;
1794                                 }
1795                         }
1796                         for ( index = 0; index < index_max; index++ )
1797                         if ( this->audio_codec[ index ] && this->audio_used[ index ] >= *samples )
1798                         {
1799                                 int current_channels = this->audio_codec[ index ]->channels;
1800                                 int16_t *src = this->audio_buffer[ index ] + *samples * current_channels;
1801                                 this->audio_used[index] -= *samples;
1802                                 memmove( this->audio_buffer[ index ], src, this->audio_used[ index ] * current_channels * sizeof(int16_t) );
1803                         }
1804                 }
1805                 // Copy a single track to the output buffer
1806                 else
1807                 {
1808                         index = this->audio_index;
1809
1810                         // Now handle the audio if we have enough
1811                         if ( this->audio_used[ index ] >= *samples )
1812                         {
1813                                 int16_t *src = this->audio_buffer[ index ];
1814                                 memcpy( *buffer, src, *samples * *channels * sizeof(int16_t) );
1815                                 this->audio_used[ index ] -= *samples;
1816                                 memmove( src, &src[ *samples * *channels ], this->audio_used[ index ] * *channels * sizeof(int16_t) );
1817                         }
1818                         else
1819                         {
1820                                 // Otherwise fill with silence
1821                                 memset( *buffer, 0, *samples * *channels * sizeof(int16_t) );
1822                         }
1823                         if ( !this->audio_resample[ index ] )
1824                         {
1825                                 // TODO: uncomment and remove following line when full multi-channel support is ready
1826                                 // *channels = codec_context->channels;
1827                                 *frequency = this->audio_codec[ index ]->sample_rate;
1828                         }
1829                 }
1830         }
1831         else
1832         {
1833                 // Get silence and don't touch the context
1834                 mlt_frame_get_audio( frame, buffer, format, frequency, channels, samples );
1835         }
1836
1837         // Regardless of speed (other than paused), we expect to get the next frame
1838         if ( !paused )
1839                 this->audio_expected = position + 1;
1840
1841         return 0;
1842 }
1843
1844 /** Initialize the audio codec context.
1845 */
1846
1847 static int audio_codec_init( producer_avformat this, int index, mlt_properties properties )
1848 {
1849         // Initialise the codec if necessary
1850         if ( !this->audio_codec[ index ] )
1851         {
1852                 // Get codec context
1853                 AVCodecContext *codec_context = this->audio_format->streams[index]->codec;
1854
1855                 // Find the codec
1856                 AVCodec *codec = avcodec_find_decoder( codec_context->codec_id );
1857
1858                 // If we don't have a codec and we can't initialise it, we can't do much more...
1859                 avformat_lock( );
1860                 if ( codec && avcodec_open( codec_context, codec ) >= 0 )
1861                 {
1862                         // Now store the codec with its destructor
1863                         if ( this->audio_codec[ index ] )
1864                                 avcodec_close( this->audio_codec[ index ] );
1865                         this->audio_codec[ index ] = codec_context;
1866                 }
1867                 else
1868                 {
1869                         // Remember that we can't use this later
1870                         this->audio_index = -1;
1871                 }
1872                 avformat_unlock( );
1873
1874                 // Process properties as AVOptions
1875                 apply_properties( codec_context, properties, AV_OPT_FLAG_AUDIO_PARAM | AV_OPT_FLAG_DECODING_PARAM );
1876         }
1877         return this->audio_codec[ index ] && this->audio_index > -1;
1878 }
1879
1880 /** Set up audio handling.
1881 */
1882
1883 static void producer_set_up_audio( producer_avformat this, mlt_frame frame )
1884 {
1885         // Get the producer
1886         mlt_producer producer = this->parent;
1887
1888         // Get the properties
1889         mlt_properties properties = MLT_PRODUCER_PROPERTIES( producer );
1890
1891         // Fetch the audio format context
1892         AVFormatContext *context = this->audio_format;
1893
1894         mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame );
1895
1896         // Get the audio_index
1897         int index = mlt_properties_get_int( properties, "audio_index" );
1898
1899         // Handle all audio tracks
1900         if ( this->audio_index > -1 &&
1901              mlt_properties_get( properties, "audio_index" ) &&
1902              !strcmp( mlt_properties_get( properties, "audio_index" ), "all" ) )
1903                 index = INT_MAX;
1904
1905         // Reopen the file if necessary
1906         if ( !context && this->audio_index > -1 && index > -1 )
1907         {
1908                 mlt_events_block( properties, producer );
1909                 producer_open( this, mlt_service_profile( MLT_PRODUCER_SERVICE(producer) ),
1910                         mlt_properties_get( properties, "resource" ) );
1911                 context = this->audio_format;
1912                 if ( this->dummy_context )
1913                 {
1914                         avformat_lock();
1915                         av_close_input_file( this->dummy_context );
1916                         avformat_unlock();
1917                 }
1918                 this->dummy_context = NULL;
1919                 mlt_events_unblock( properties, producer );
1920                 get_audio_streams_info( this );
1921         }
1922
1923         // Exception handling for audio_index
1924         if ( context && index >= (int) context->nb_streams && index < INT_MAX )
1925         {
1926                 for ( index = context->nb_streams - 1;
1927                           index >= 0 && context->streams[ index ]->codec->codec_type != CODEC_TYPE_AUDIO;
1928                           index-- );
1929                 mlt_properties_set_int( properties, "audio_index", index );
1930         }
1931         if ( context && index > -1 && index < INT_MAX &&
1932                  context->streams[ index ]->codec->codec_type != CODEC_TYPE_AUDIO )
1933         {
1934                 index = this->audio_index;
1935                 mlt_properties_set_int( properties, "audio_index", index );
1936         }
1937
1938         // Update the audio properties if the index changed
1939         if ( context && index > -1 && index != this->audio_index )
1940         {
1941                 if ( this->audio_codec[ this->audio_index ] )
1942                 {
1943                         avformat_lock();
1944                         avcodec_close( this->audio_codec[ this->audio_index ] );
1945                         avformat_unlock();
1946                 }
1947                 this->audio_codec[ this->audio_index ] = NULL;
1948         }
1949         if ( this->audio_index != -1 )
1950                 this->audio_index = index;
1951         else
1952                 index = -1;
1953
1954         // Get the codec(s)
1955         if ( context && index == INT_MAX )
1956         {
1957                 mlt_properties_set_int( frame_properties, "frequency", this->max_frequency );
1958                 mlt_properties_set_int( frame_properties, "channels", this->total_channels );
1959                 for ( index = 0; index < context->nb_streams; index++ )
1960                 {
1961                         if ( context->streams[ index ]->codec->codec_type == CODEC_TYPE_AUDIO )
1962                                 audio_codec_init( this, index, properties );
1963                 }
1964         }
1965         else if ( context && index > -1 && audio_codec_init( this, index, properties ) )
1966         {
1967                 // Set the frame properties
1968                 if ( index < INT_MAX )
1969                 {
1970                         mlt_properties_set_int( frame_properties, "frequency", this->audio_codec[ index ]->sample_rate );
1971                         mlt_properties_set_int( frame_properties, "channels", this->audio_codec[ index ]->channels );
1972                 }
1973         }
1974         if ( context && index > -1 )
1975         {
1976                 // Add our audio operation
1977                 mlt_frame_push_audio( frame, this );
1978                 mlt_frame_push_audio( frame, producer_get_audio );
1979         }
1980 }
1981
1982 /** Our get frame implementation.
1983 */
1984
1985 static int producer_get_frame( mlt_producer producer, mlt_frame_ptr frame, int index )
1986 {
1987         // Access the private data
1988         mlt_service service = MLT_PRODUCER_SERVICE( producer );
1989         mlt_cache_item cache_item = mlt_service_cache_get( service, "producer_avformat" );
1990         producer_avformat this = mlt_cache_item_data( cache_item, NULL );
1991
1992         // If cache miss
1993         if ( !this )
1994         {
1995                 this = calloc( 1, sizeof( struct producer_avformat_s ) );
1996                 producer->child = this;
1997                 this->parent = producer;
1998                 mlt_service_cache_put( service, "producer_avformat", this, 0, (mlt_destructor) producer_avformat_close );
1999                 cache_item = mlt_service_cache_get( service, "producer_avformat" );
2000         }
2001
2002         // Create an empty frame
2003         *frame = mlt_frame_init( service);
2004
2005         if ( *frame )
2006         {
2007                 mlt_properties_set_data( MLT_FRAME_PROPERTIES(*frame), "avformat_cache", cache_item, 0, (mlt_destructor) mlt_cache_item_close, NULL );
2008         }
2009         else
2010         {
2011                 mlt_cache_item_close( cache_item );
2012                 return 1;
2013         }
2014
2015         // Update timecode on the frame we're creating
2016         mlt_frame_set_position( *frame, mlt_producer_position( producer ) );
2017
2018         // Set the position of this producer
2019         mlt_properties_set_position( MLT_FRAME_PROPERTIES( *frame ), "avformat_position", mlt_producer_frame( producer ) );
2020
2021         // Set up the video
2022         producer_set_up_video( this, *frame );
2023
2024         // Set up the audio
2025         producer_set_up_audio( this, *frame );
2026
2027         // Calculate the next timecode
2028         mlt_producer_prepare_next( producer );
2029
2030         return 0;
2031 }
2032
2033 static void producer_avformat_close( producer_avformat this )
2034 {
2035         mlt_log_debug( NULL, "producer_avformat_close\n" );
2036         // Close the file
2037         av_free( this->av_frame );
2038         avformat_lock();
2039         int i;
2040         for ( i = 0; i < MAX_AUDIO_STREAMS; i++ )
2041         {
2042                 if ( this->audio_resample[i] )
2043                         audio_resample_close( this->audio_resample[i] );
2044                 mlt_pool_release( this->audio_buffer[i] );
2045                 av_free( this->decode_buffer[i] );
2046                 if ( this->audio_codec[i] )
2047                         avcodec_close( this->audio_codec[i] );
2048         }
2049         if ( this->video_codec )
2050                 avcodec_close( this->video_codec );
2051         if ( this->dummy_context )
2052                 av_close_input_file( this->dummy_context );
2053         if ( this->audio_format )
2054                 av_close_input_file( this->audio_format );
2055         if ( this->video_format )
2056                 av_close_input_file( this->video_format );
2057         avformat_unlock();
2058 #ifdef VDPAU
2059         vdpau_producer_close( this );
2060 #endif
2061 #ifdef SWSCALE
2062         sws_freeContext( this->swscale );
2063 #endif
2064         if ( this->image_cache )
2065                 mlt_cache_close( this->image_cache );
2066         free( this );
2067 }
2068
2069 static void producer_close( mlt_producer parent )
2070 {
2071         // Close the parent
2072         parent->close = NULL;
2073         mlt_producer_close( parent );
2074
2075         // Free the memory
2076         free( parent );
2077 }