git.sesse.net Git - mlt/blob - src/modules/avformat/producer_avformat.c

   1 /*
   2  * producer_avformat.c -- avformat producer
   3  * Copyright (C) 2003-2004 Ushodaya Enterprises Limited
   4  * Author: Charles Yates <charles.yates@pandora.be>
   5  * Much code borrowed from ffmpeg.c: Copyright (c) 2000-2003 Fabrice Bellard
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2.1 of the License, or (at your option) any later version.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with this library; if not, write to the Free Software
  19  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20  */
  21
  22 // MLT Header files
  23 #include <framework/mlt_producer.h>
  24 #include <framework/mlt_frame.h>
  25 #include <framework/mlt_profile.h>
  26 #include <framework/mlt_log.h>
  27
  28 // ffmpeg Header files
  29 #include <avformat.h>
  30 #include <opt.h>
  31 #ifdef SWSCALE
  32 #  include <swscale.h>
  33 #endif
  34 #if (LIBAVCODEC_VERSION_INT >= ((51<<16)+(71<<8)+0))
  35 #  include "audioconvert.h"
  36 #endif
  37
  38 // System header files
  39 #include <stdlib.h>
  40 #include <string.h>
  41 #include <pthread.h>
  42
  43 #if LIBAVUTIL_VERSION_INT < (50<<16)
  44 #define PIX_FMT_RGB32 PIX_FMT_RGBA32
  45 #define PIX_FMT_YUYV422 PIX_FMT_YUV422
  46 #endif
  47
  48 #define POSITION_INITIAL (-2)
  49 #define POSITION_INVALID (-1)
  50
  51 void avformat_lock( );
  52 void avformat_unlock( );
  53
  54 struct producer_avformat_s
  55 {
  56         struct mlt_producer_s parent;
  57         AVFormatContext *dummy_context;
  58         AVFormatContext *audio_context;
  59         AVFormatContext *video_context;
  60         AVCodec *audio_codec;
  61         AVCodec *video_codec;
  62         AVFrame *av_frame;
  63         ReSampleContext *audio_resample;
  64         mlt_position audio_expected;
  65         mlt_position video_expected;
  66         int audio_index;
  67         int video_index;
  68         double start_time;
  69         int first_pts;
  70         int last_position;
  71         int seekable;
  72         int current_position;
  73         int got_picture;
  74         int top_field_first;
  75         int16_t *audio_buffer;
  76         int16_t *decode_buffer;
  77         int audio_used;
  78 };
  79 typedef struct producer_avformat_s *producer_avformat;
  80
  81 // Forward references.
  82 static int producer_open( producer_avformat this, mlt_profile profile, char *file );
  83 static int producer_get_frame( mlt_producer this, mlt_frame_ptr frame, int index );
  84 static void producer_file_close( void *context );
  85 static void producer_close( mlt_producer parent );
  86
  87 /** Constructor for libavformat.
  88 */
  89
  90 mlt_producer producer_avformat_init( mlt_profile profile, char *file )
  91 {
  92         int skip = 0;
  93
  94         // Report information about available demuxers and codecs as YAML Tiny
  95         if ( file && strstr( file, "f-list" ) )
  96         {
  97                 fprintf( stderr, "---\nformats:\n" );
  98                 AVInputFormat *format = NULL;
  99                 while ( ( format = av_iformat_next( format ) ) )
 100                         fprintf( stderr, "  - %s\n", format->name );
 101                 fprintf( stderr, "...\n" );
 102                 skip = 1;
 103         }
 104         if ( file && strstr( file, "acodec-list" ) )
 105         {
 106                 fprintf( stderr, "---\naudio_codecs:\n" );
 107                 AVCodec *codec = NULL;
 108                 while ( ( codec = av_codec_next( codec ) ) )
 109                         if ( codec->decode && codec->type == CODEC_TYPE_AUDIO )
 110                                 fprintf( stderr, "  - %s\n", codec->name );
 111                 fprintf( stderr, "...\n" );
 112                 skip = 1;
 113         }
 114         if ( file && strstr( file, "vcodec-list" ) )
 115         {
 116                 fprintf( stderr, "---\nvideo_codecs:\n" );
 117                 AVCodec *codec = NULL;
 118                 while ( ( codec = av_codec_next( codec ) ) )
 119                         if ( codec->decode && codec->type == CODEC_TYPE_VIDEO )
 120                                 fprintf( stderr, "  - %s\n", codec->name );
 121                 fprintf( stderr, "...\n" );
 122                 skip = 1;
 123         }
 124
 125         // Check that we have a non-NULL argument
 126         if ( !skip && file )
 127         {
 128                 // Construct the producer
 129                 producer_avformat this = calloc( 1, sizeof( struct producer_avformat_s ) );
 130
 131                 // Initialise it
 132                 if ( mlt_producer_init( &this->parent, this ) == 0 )
 133                 {
 134                         mlt_producer producer = &this->parent;
 135
 136                         // Get the properties
 137                         mlt_properties properties = MLT_PRODUCER_PROPERTIES( producer );
 138
 139                         // Set the resource property (required for all producers)
 140                         mlt_properties_set( properties, "resource", file );
 141
 142                         // Register transport implementation with the producer
 143                         producer->close = (mlt_destructor) producer_close;
 144
 145                         // Register our get_frame implementation
 146                         producer->get_frame = producer_get_frame;
 147
 148                         // Open the file
 149                         if ( producer_open( this, profile, file ) != 0 )
 150                         {
 151                                 // Clean up
 152                                 mlt_producer_close( producer );
 153                                 this = NULL;
 154                         }
 155                         else
 156                         {
 157                                 // Close the file to release resources for large playlists - reopen later as needed
 158                                 producer_file_close( this->dummy_context );
 159                                 this->dummy_context = NULL;
 160                                 producer_file_close( this->audio_context );
 161                                 this->audio_context = NULL;
 162                                 producer_file_close( this->video_context );
 163                                 this->video_context = NULL;
 164
 165                                 // Default the user-selectable indices from the auto-detected indices
 166                                 mlt_properties_set_int( properties, "audio_index",  this->audio_index );
 167                                 mlt_properties_set_int( properties, "video_index",  this->video_index );
 168                         }
 169                         return producer;
 170                 }
 171         }
 172         return NULL;
 173 }
 174
 175 /** Find the default streams.
 176 */
 177
 178 static mlt_properties find_default_streams( mlt_properties meta_media, AVFormatContext *context, int *audio_index, int *video_index )
 179 {
 180         int i;
 181         char key[200];
 182
 183         mlt_properties_set_int( meta_media, "meta.media.nb_streams", context->nb_streams );
 184
 185         // Allow for multiple audio and video streams in the file and select first of each (if available)
 186         for( i = 0; i < context->nb_streams; i++ )
 187         {
 188                 // Get the codec context
 189                 AVStream *stream = context->streams[ i ];
 190                 if ( ! stream ) continue;
 191                 AVCodecContext *codec_context = stream->codec;
 192                 if ( ! codec_context ) continue;
 193                 AVCodec *codec = avcodec_find_decoder( codec_context->codec_id );
 194                 if ( ! codec ) continue;
 195
 196                 snprintf( key, sizeof(key), "meta.media.%d.stream.type", i );
 197
 198                 // Determine the type and obtain the first index of each type
 199                 switch( codec_context->codec_type )
 200                 {
 201                         case CODEC_TYPE_VIDEO:
 202                                 if ( *video_index < 0 )
 203                                         *video_index = i;
 204                                 mlt_properties_set( meta_media, key, "video" );
 205                                 snprintf( key, sizeof(key), "meta.media.%d.stream.frame_rate", i );
 206                                 mlt_properties_set_double( meta_media, key, av_q2d( context->streams[ i ]->r_frame_rate ) );
 207 #if LIBAVFORMAT_VERSION_INT >= ((52<<16)+(21<<8)+0)
 208                                 snprintf( key, sizeof(key), "meta.media.%d.stream.sample_aspect_ratio", i );
 209                                 mlt_properties_set_double( meta_media, key, av_q2d( context->streams[ i ]->sample_aspect_ratio ) );
 210 #endif
 211                                 snprintf( key, sizeof(key), "meta.media.%d.codec.pix_fmt", i );
 212                                 mlt_properties_set( meta_media, key, avcodec_get_pix_fmt_name( codec_context->pix_fmt ) );
 213                                 snprintf( key, sizeof(key), "meta.media.%d.codec.sample_aspect_ratio", i );
 214                                 mlt_properties_set_double( meta_media, key, av_q2d( codec_context->sample_aspect_ratio ) );
 215                                 break;
 216                         case CODEC_TYPE_AUDIO:
 217                                 if ( *audio_index < 0 )
 218                                         *audio_index = i;
 219                                 mlt_properties_set( meta_media, key, "audio" );
 220 #if (LIBAVCODEC_VERSION_INT >= ((51<<16)+(71<<8)+0))
 221                                 snprintf( key, sizeof(key), "meta.media.%d.codec.sample_fmt", i );
 222                                 mlt_properties_set( meta_media, key, avcodec_get_sample_fmt_name( codec_context->sample_fmt ) );
 223 #endif
 224                                 snprintf( key, sizeof(key), "meta.media.%d.codec.sample_rate", i );
 225                                 mlt_properties_set_int( meta_media, key, codec_context->sample_rate );
 226                                 snprintf( key, sizeof(key), "meta.media.%d.codec.channels", i );
 227                                 mlt_properties_set_int( meta_media, key, codec_context->channels );
 228                                 break;
 229                         default:
 230                                 break;
 231                 }
 232 //              snprintf( key, sizeof(key), "meta.media.%d.stream.time_base", i );
 233 //              mlt_properties_set_double( meta_media, key, av_q2d( context->streams[ i ]->time_base ) );
 234                 snprintf( key, sizeof(key), "meta.media.%d.codec.name", i );
 235                 mlt_properties_set( meta_media, key, codec->name );
 236 #if (LIBAVCODEC_VERSION_INT >= ((51<<16)+(55<<8)+0))
 237                 snprintf( key, sizeof(key), "meta.media.%d.codec.long_name", i );
 238                 mlt_properties_set( meta_media, key, codec->long_name );
 239 #endif
 240                 snprintf( key, sizeof(key), "meta.media.%d.codec.bit_rate", i );
 241                 mlt_properties_set_int( meta_media, key, codec_context->bit_rate );
 242 //              snprintf( key, sizeof(key), "meta.media.%d.codec.time_base", i );
 243 //              mlt_properties_set_double( meta_media, key, av_q2d( codec_context->time_base ) );
 244                 snprintf( key, sizeof(key), "meta.media.%d.codec.profile", i );
 245                 mlt_properties_set_int( meta_media, key, codec_context->profile );
 246                 snprintf( key, sizeof(key), "meta.media.%d.codec.level", i );
 247                 mlt_properties_set_int( meta_media, key, codec_context->level );
 248         }
 249
 250         return meta_media;
 251 }
 252
 253 /** Producer file destructor.
 254 */
 255
 256 static void producer_file_close( void *context )
 257 {
 258         if ( context )
 259         {
 260                 // Lock the mutex now
 261                 avformat_lock( );
 262
 263                 // Close the file
 264                 av_close_input_file( context );
 265
 266                 // Unlock the mutex now
 267                 avformat_unlock( );
 268         }
 269 }
 270
 271 /** Producer file destructor.
 272 */
 273
 274 static void producer_codec_close( void *codec )
 275 {
 276         if ( codec )
 277         {
 278                 // Lock the mutex now
 279                 avformat_lock( );
 280
 281                 // Close the file
 282                 avcodec_close( codec );
 283
 284                 // Unlock the mutex now
 285                 avformat_unlock( );
 286         }
 287 }
 288
 289 static inline int dv_is_pal( AVPacket *pkt )
 290 {
 291         return pkt->data[3] & 0x80;
 292 }
 293
 294 static int dv_is_wide( AVPacket *pkt )
 295 {
 296         int i = 80 /* block size */ *3 /* VAUX starts at block 3 */ +3 /* skip block header */;
 297
 298         for ( ; i < pkt->size; i += 5 /* packet size */ )
 299         {
 300                 if ( pkt->data[ i ] == 0x61 )
 301                 {
 302                         uint8_t x = pkt->data[ i + 2 ] & 0x7;
 303                         return ( x == 2 ) || ( x == 7 );
 304                 }
 305         }
 306         return 0;
 307 }
 308
 309 static double get_aspect_ratio( AVStream *stream, AVCodecContext *codec_context, AVPacket *pkt )
 310 {
 311         double aspect_ratio = 1.0;
 312
 313         if ( codec_context->codec_id == CODEC_ID_DVVIDEO )
 314         {
 315                 if ( pkt )
 316                 {
 317                         if ( dv_is_pal( pkt ) )
 318                         {
 319                                 aspect_ratio = dv_is_wide( pkt )
 320                                         ? 64.0/45.0 // 16:9 PAL
 321                                         : 16.0/15.0; // 4:3 PAL
 322                         }
 323                         else
 324                         {
 325                                 aspect_ratio = dv_is_wide( pkt )
 326                                         ? 32.0/27.0 // 16:9 NTSC
 327                                         : 8.0/9.0; // 4:3 NTSC
 328                         }
 329                 }
 330                 else
 331                 {
 332                         AVRational ar =
 333 #if LIBAVFORMAT_VERSION_INT >= ((52<<16)+(21<<8)+0)
 334                                 stream->sample_aspect_ratio;
 335 #else
 336                                 codec_context->sample_aspect_ratio;
 337 #endif
 338                         // Override FFmpeg's notion of DV aspect ratios, which are
 339                         // based upon a width of 704. Since we do not have a normaliser
 340                         // that crops (nor is cropping 720 wide ITU-R 601 video always desirable)
 341                         // we just coerce the values to facilitate a passive behaviour through
 342                         // the rescale normaliser when using equivalent producers and consumers.
 343                         // = display_aspect / (width * height)
 344                         if ( ar.num == 10 && ar.den == 11 )
 345                                 aspect_ratio = 8.0/9.0; // 4:3 NTSC
 346                         else if ( ar.num == 59 && ar.den == 54 )
 347                                 aspect_ratio = 16.0/15.0; // 4:3 PAL
 348                         else if ( ar.num == 40 && ar.den == 33 )
 349                                 aspect_ratio = 32.0/27.0; // 16:9 NTSC
 350                         else if ( ar.num == 118 && ar.den == 81 )
 351                                 aspect_ratio = 64.0/45.0; // 16:9 PAL
 352                 }
 353         }
 354         else
 355         {
 356                 AVRational codec_sar = codec_context->sample_aspect_ratio;
 357                 AVRational stream_sar =
 358 #if LIBAVFORMAT_VERSION_INT >= ((52<<16)+(21<<8)+0)
 359                         stream->sample_aspect_ratio;
 360 #else
 361                         { 0, 1 };
 362 #endif
 363                 if ( codec_sar.num > 0 )
 364                         aspect_ratio = av_q2d( codec_sar );
 365                 else if ( stream_sar.num > 0 )
 366                         aspect_ratio = av_q2d( stream_sar );
 367         }
 368         return aspect_ratio;
 369 }
 370
 371 /** Open the file.
 372 */
 373
 374 static int producer_open( producer_avformat this, mlt_profile profile, char *file )
 375 {
 376         // Return an error code (0 == no error)
 377         int error = 0;
 378
 379         // Context for avformat
 380         AVFormatContext *context = NULL;
 381
 382         // Get the properties
 383         mlt_properties properties = MLT_PRODUCER_PROPERTIES( &this->parent );
 384
 385         // We will treat everything with the producer fps
 386         double fps = mlt_profile_fps( profile );
 387
 388         // Lock the mutex now
 389         avformat_lock( );
 390
 391         // If "MRL", then create AVInputFormat
 392         AVInputFormat *format = NULL;
 393         AVFormatParameters *params = NULL;
 394         char *standard = NULL;
 395         char *mrl = strchr( file, ':' );
 396
 397         // AV option (0 = both, 1 = video, 2 = audio)
 398         int av = 0;
 399
 400         // Only if there is not a protocol specification that avformat can handle
 401         if ( mrl && !url_exist( file ) )
 402         {
 403                 // 'file' becomes format abbreviation
 404                 mrl[0] = 0;
 405
 406                 // Lookup the format
 407                 format = av_find_input_format( file );
 408
 409                 // Eat the format designator
 410                 file = ++mrl;
 411
 412                 if ( format )
 413                 {
 414                         // Allocate params
 415                         params = calloc( sizeof( AVFormatParameters ), 1 );
 416
 417                         // These are required by video4linux (defaults)
 418                         params->width = 640;
 419                         params->height = 480;
 420                         params->time_base= (AVRational){1,25};
 421                         // params->device = file;
 422                         params->channels = 2;
 423                         params->sample_rate = 48000;
 424                 }
 425
 426                 // XXX: this does not work anymore since avdevice
 427                 // TODO: make producer_avddevice?
 428                 // Parse out params
 429                 mrl = strchr( file, '?' );
 430                 while ( mrl )
 431                 {
 432                         mrl[0] = 0;
 433                         char *name = strdup( ++mrl );
 434                         char *value = strchr( name, ':' );
 435                         if ( value )
 436                         {
 437                                 value[0] = 0;
 438                                 value++;
 439                                 char *t = strchr( value, '&' );
 440                                 if ( t )
 441                                         t[0] = 0;
 442                                 if ( !strcmp( name, "frame_rate" ) )
 443                                         params->time_base.den = atoi( value );
 444                                 else if ( !strcmp( name, "frame_rate_base" ) )
 445                                         params->time_base.num = atoi( value );
 446                                 else if ( !strcmp( name, "sample_rate" ) )
 447                                         params->sample_rate = atoi( value );
 448                                 else if ( !strcmp( name, "channels" ) )
 449                                         params->channels = atoi( value );
 450                                 else if ( !strcmp( name, "width" ) )
 451                                         params->width = atoi( value );
 452                                 else if ( !strcmp( name, "height" ) )
 453                                         params->height = atoi( value );
 454                                 else if ( !strcmp( name, "standard" ) )
 455                                 {
 456                                         standard = strdup( value );
 457                                         params->standard = standard;
 458                                 }
 459                                 else if ( !strcmp( name, "av" ) )
 460                                         av = atoi( value );
 461                         }
 462                         free( name );
 463                         mrl = strchr( mrl, '&' );
 464                 }
 465         }
 466
 467         // Now attempt to open the file
 468         error = av_open_input_file( &context, file, format, 0, params ) < 0;
 469
 470         // Cleanup AVFormatParameters
 471         free( standard );
 472         free( params );
 473
 474         // If successful, then try to get additional info
 475         if ( !error )
 476         {
 477                 // Get the stream info
 478                 error = av_find_stream_info( context ) < 0;
 479
 480                 // Continue if no error
 481                 if ( !error )
 482                 {
 483                         // We will default to the first audio and video streams found
 484                         int audio_index = -1;
 485                         int video_index = -1;
 486
 487                         // Now set properties where we can (use default unknowns if required)
 488                         if ( context->duration != AV_NOPTS_VALUE )
 489                         {
 490                                 // This isn't going to be accurate for all formats
 491                                 mlt_position frames = ( mlt_position )( ( ( double )context->duration / ( double )AV_TIME_BASE ) * fps + 0.5 );
 492                                 mlt_properties_set_position( properties, "out", frames - 1 );
 493                                 mlt_properties_set_position( properties, "length", frames );
 494                         }
 495
 496                         // Find default audio and video streams
 497                         find_default_streams( properties, context, &audio_index, &video_index );
 498
 499                         if ( context->start_time != AV_NOPTS_VALUE )
 500                                 this->start_time = context->start_time;
 501
 502                         // Check if we're seekable (something funny about mpeg here :-/)
 503                         if ( strncmp( file, "pipe:", 5 ) &&
 504                                  strncmp( file, "http:", 5 ) &&
 505                                  strncmp( file, "udp:", 4 )  &&
 506                                  strncmp( file, "tcp:", 4 )  &&
 507                                  strncmp( file, "rtsp:", 5 ) &&
 508                                  strncmp( file, "rtp:", 4 ) )
 509                         {
 510                                 this->seekable = av_seek_frame( context, -1, this->start_time, AVSEEK_FLAG_BACKWARD ) >= 0;
 511                                 mlt_properties_set_int( properties, "seekable", this->seekable );
 512                                 producer_file_close( this->dummy_context );
 513                                 this->dummy_context = context;
 514                                 av_open_input_file( &context, file, NULL, 0, NULL );
 515                                 av_find_stream_info( context );
 516                         }
 517
 518                         // Store selected audio and video indexes on properties
 519                         this->audio_index = audio_index;
 520                         this->video_index = video_index;
 521                         this->first_pts = -1;
 522                         this->last_position = POSITION_INITIAL;
 523
 524                         // Fetch the width, height and aspect ratio
 525                         if ( video_index != -1 )
 526                         {
 527                                 AVCodecContext *codec_context = context->streams[ video_index ]->codec;
 528                                 mlt_properties_set_int( properties, "width", codec_context->width );
 529                                 mlt_properties_set_int( properties, "height", codec_context->height );
 530
 531                                 if ( codec_context->codec_id == CODEC_ID_DVVIDEO )
 532                                 {
 533                                         // Fetch the first frame of DV so we can read it directly
 534                                         AVPacket pkt;
 535                                         int ret = 0;
 536                                         while ( ret >= 0 )
 537                                         {
 538                                                 ret = av_read_frame( context, &pkt );
 539                                                 if ( ret >= 0 && pkt.stream_index == video_index && pkt.size > 0 )
 540                                                 {
 541                                                         mlt_properties_set_double( properties, "aspect_ratio",
 542                                                                 get_aspect_ratio( context->streams[ video_index ], codec_context, &pkt ) );
 543                                                         break;
 544                                                 }
 545                                         }
 546                                 }
 547                                 else
 548                                 {
 549                                         mlt_properties_set_double( properties, "aspect_ratio",
 550                                                 get_aspect_ratio( context->streams[ video_index ], codec_context, NULL ) );
 551                                 }
 552                         }
 553
 554                         // Read Metadata
 555                         if ( context->title )
 556                                 mlt_properties_set(properties, "meta.attr.title.markup", context->title );
 557                         if ( context->author )
 558                                 mlt_properties_set(properties, "meta.attr.author.markup", context->author );
 559                         if ( context->copyright )
 560                                 mlt_properties_set(properties, "meta.attr.copyright.markup", context->copyright );
 561                         if ( context->comment )
 562                                 mlt_properties_set(properties, "meta.attr.comment.markup", context->comment );
 563                         if ( context->album )
 564                                 mlt_properties_set(properties, "meta.attr.album.markup", context->album );
 565                         if ( context->year )
 566                                 mlt_properties_set_int(properties, "meta.attr.year.markup", context->year );
 567                         if ( context->track )
 568                                 mlt_properties_set_int(properties, "meta.attr.track.markup", context->track );
 569
 570                         // We're going to cheat here - for a/v files, we will have two contexts (reasoning will be clear later)
 571                         if ( av == 0 && audio_index != -1 && video_index != -1 )
 572                         {
 573                                 // We'll use the open one as our video_context
 574                                 producer_file_close( this->video_context );
 575                                 this->video_context = context;
 576
 577                                 // And open again for our audio context
 578                                 av_open_input_file( &context, file, NULL, 0, NULL );
 579                                 av_find_stream_info( context );
 580
 581                                 // Audio context
 582                                 producer_file_close( this->audio_context );
 583                                 this->audio_context = context;
 584                         }
 585                         else if ( av != 2 && video_index != -1 )
 586                         {
 587                                 // We only have a video context
 588                                 producer_file_close( this->video_context );
 589                                 this->video_context = context;
 590                         }
 591                         else if ( audio_index != -1 )
 592                         {
 593                                 // We only have an audio context
 594                                 producer_file_close( this->audio_context );
 595                                 this->audio_context = context;
 596                         }
 597                         else
 598                         {
 599                                 // Something has gone wrong
 600                                 error = -1;
 601                         }
 602                 }
 603         }
 604
 605         // Unlock the mutex now
 606         avformat_unlock( );
 607
 608         return error;
 609 }
 610
 611 /** Convert a frame position to a time code.
 612 */
 613
 614 static double producer_time_of_frame( mlt_producer this, mlt_position position )
 615 {
 616         return ( double )position / mlt_producer_get_fps( this );
 617 }
 618
 619 static inline void convert_image( AVFrame *frame, uint8_t *buffer, int pix_fmt, mlt_image_format *format, int width, int height )
 620 {
 621 #ifdef SWSCALE
 622         if ( pix_fmt == PIX_FMT_RGB32 )
 623         {
 624                 *format = mlt_image_rgb24a;
 625                 struct SwsContext *context = sws_getContext( width, height, pix_fmt,
 626                         width, height, PIX_FMT_RGBA, SWS_FAST_BILINEAR, NULL, NULL, NULL);
 627                 AVPicture output;
 628                 avpicture_fill( &output, buffer, PIX_FMT_RGBA, width, height );
 629                 sws_scale( context, frame->data, frame->linesize, 0, height,
 630                         output.data, output.linesize);
 631                 sws_freeContext( context );
 632         }
 633         else if ( *format == mlt_image_yuv420p )
 634         {
 635                 struct SwsContext *context = sws_getContext( width, height, pix_fmt,
 636                         width, height, PIX_FMT_YUV420P, SWS_FAST_BILINEAR, NULL, NULL, NULL);
 637                 AVPicture output;
 638                 output.data[0] = buffer;
 639                 output.data[1] = buffer + width * height;
 640                 output.data[2] = buffer + ( 3 * width * height ) / 2;
 641                 output.linesize[0] = width;
 642                 output.linesize[1] = width >> 1;
 643                 output.linesize[2] = width >> 1;
 644                 sws_scale( context, frame->data, frame->linesize, 0, height,
 645                         output.data, output.linesize);
 646                 sws_freeContext( context );
 647         }
 648         else if ( *format == mlt_image_rgb24 )
 649         {
 650                 struct SwsContext *context = sws_getContext( width, height, pix_fmt,
 651                         width, height, PIX_FMT_RGB24, SWS_FAST_BILINEAR, NULL, NULL, NULL);
 652                 AVPicture output;
 653                 avpicture_fill( &output, buffer, PIX_FMT_RGB24, width, height );
 654                 sws_scale( context, frame->data, frame->linesize, 0, height,
 655                         output.data, output.linesize);
 656                 sws_freeContext( context );
 657         }
 658         else if ( *format == mlt_image_rgb24a || *format == mlt_image_opengl )
 659         {
 660                 struct SwsContext *context = sws_getContext( width, height, pix_fmt,
 661                         width, height, PIX_FMT_RGBA, SWS_FAST_BILINEAR, NULL, NULL, NULL);
 662                 AVPicture output;
 663                 avpicture_fill( &output, buffer, PIX_FMT_RGBA, width, height );
 664                 sws_scale( context, frame->data, frame->linesize, 0, height,
 665                         output.data, output.linesize);
 666                 sws_freeContext( context );
 667         }
 668         else
 669         {
 670                 struct SwsContext *context = sws_getContext( width, height, pix_fmt,
 671                         width, height, PIX_FMT_YUYV422, SWS_FAST_BILINEAR, NULL, NULL, NULL);
 672                 AVPicture output;
 673                 avpicture_fill( &output, buffer, PIX_FMT_YUYV422, width, height );
 674                 sws_scale( context, frame->data, frame->linesize, 0, height,
 675                         output.data, output.linesize);
 676                 sws_freeContext( context );
 677         }
 678 #else
 679         if ( *format == mlt_image_yuv420p )
 680         {
 681                 AVPicture pict;
 682                 pict.data[0] = buffer;
 683                 pict.data[1] = buffer + width * height;
 684                 pict.data[2] = buffer + ( 3 * width * height ) / 2;
 685                 pict.linesize[0] = width;
 686                 pict.linesize[1] = width >> 1;
 687                 pict.linesize[2] = width >> 1;
 688                 img_convert( &pict, PIX_FMT_YUV420P, (AVPicture *)frame, pix_fmt, width, height );
 689         }
 690         else if ( *format == mlt_image_rgb24 )
 691         {
 692                 AVPicture output;
 693                 avpicture_fill( &output, buffer, PIX_FMT_RGB24, width, height );
 694                 img_convert( &output, PIX_FMT_RGB24, (AVPicture *)frame, pix_fmt, width, height );
 695         }
 696         else if ( format == mlt_image_rgb24a || format == mlt_image_opengl )
 697         {
 698                 AVPicture output;
 699                 avpicture_fill( &output, buffer, PIX_FMT_RGB32, width, height );
 700                 img_convert( &output, PIX_FMT_RGB32, (AVPicture *)frame, pix_fmt, width, height );
 701         }
 702         else
 703         {
 704                 AVPicture output;
 705                 avpicture_fill( &output, buffer, PIX_FMT_YUYV422, width, height );
 706                 img_convert( &output, PIX_FMT_YUYV422, (AVPicture *)frame, pix_fmt, width, height );
 707         }
 708 #endif
 709 }
 710
 711 /** Allocate the image buffer and set it on the frame.
 712 */
 713
 714 static int allocate_buffer( mlt_properties frame_properties, AVCodecContext *codec_context, uint8_t **buffer, mlt_image_format *format, int *width, int *height )
 715 {
 716         int size = 0;
 717
 718         if ( codec_context->width == 0 || codec_context->height == 0 )
 719                 return size;
 720
 721         *width = codec_context->width;
 722         *height = codec_context->height;
 723         mlt_properties_set_int( frame_properties, "width", *width );
 724         mlt_properties_set_int( frame_properties, "height", *height );
 725
 726         if ( codec_context->pix_fmt == PIX_FMT_RGB32 )
 727                 size = *width * ( *height + 1 ) * 4;
 728         else switch ( *format )
 729         {
 730                 case mlt_image_yuv420p:
 731                         size = *width * 3 * ( *height + 1 ) / 2;
 732                         break;
 733                 case mlt_image_rgb24:
 734                         size = *width * ( *height + 1 ) * 3;
 735                         break;
 736                 case mlt_image_rgb24a:
 737                 case mlt_image_opengl:
 738                         size = *width * ( *height + 1 ) * 4;
 739                         break;
 740                 default:
 741                         *format = mlt_image_yuv422;
 742                         size = *width * ( *height + 1 ) * 2;
 743                         break;
 744         }
 745
 746         // Construct the output image
 747         *buffer = mlt_pool_alloc( size );
 748         if ( *buffer )
 749                 mlt_properties_set_data( frame_properties, "image", *buffer, size, mlt_pool_release, NULL );
 750         else
 751                 size = 0;
 752
 753         return size;
 754 }
 755
 756 /** Get an image from a frame.
 757 */
 758
 759 static int producer_get_image( mlt_frame frame, uint8_t **buffer, mlt_image_format *format, int *width, int *height, int writable )
 760 {
 761         // Get the producer
 762         producer_avformat this = mlt_frame_pop_service( frame );
 763         mlt_producer producer = &this->parent;
 764
 765         // Get the properties from the frame
 766         mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame );
 767
 768         // Obtain the frame number of this frame
 769         mlt_position position = mlt_properties_get_position( frame_properties, "avformat_position" );
 770
 771         // Get the producer properties
 772         mlt_properties properties = MLT_PRODUCER_PROPERTIES( producer );
 773
 774         avformat_lock();
 775
 776         // Fetch the video_context
 777         AVFormatContext *context = this->video_context;
 778
 779         // Get the video stream
 780         AVStream *stream = context->streams[ this->video_index ];
 781
 782         // Get codec context
 783         AVCodecContext *codec_context = stream->codec;
 784
 785         // Packet
 786         AVPacket pkt;
 787
 788         // Special case pause handling flag
 789         int paused = 0;
 790
 791         // Special case ffwd handling
 792         int ignore = 0;
 793
 794         // We may want to use the source fps if available
 795         double source_fps = mlt_properties_get_double( properties, "source_fps" );
 796         double fps = mlt_producer_get_fps( producer );
 797
 798         // This is the physical frame position in the source
 799         int req_position = ( int )( position / fps * source_fps + 0.5 );
 800
 801         // Determines if we have to decode all frames in a sequence
 802         // Temporary hack to improve intra frame only
 803         int must_decode = strcmp( codec_context->codec->name, "dnxhd" ) &&
 804                                   strcmp( codec_context->codec->name, "dvvideo" ) &&
 805                                   strcmp( codec_context->codec->name, "huffyuv" ) &&
 806                                   strcmp( codec_context->codec->name, "mjpeg" ) &&
 807                                   strcmp( codec_context->codec->name, "rawvideo" );
 808
 809         int last_position = this->last_position;
 810
 811         // Turn on usage of new seek API and PTS for seeking
 812         int use_new_seek = codec_context->codec_id == CODEC_ID_H264 && !strcmp( context->iformat->name, "mpegts" );
 813         if ( mlt_properties_get( properties, "new_seek" ) )
 814                 use_new_seek = mlt_properties_get_int( properties, "new_seek" );
 815
 816         // Seek if necessary
 817         if ( position != this->video_expected || last_position < 0 )
 818         {
 819                 if ( this->av_frame && position + 1 == this->video_expected )
 820                 {
 821                         // We're paused - use last image
 822                         paused = 1;
 823                 }
 824                 else if ( !this->seekable && position > this->video_expected && ( position - this->video_expected ) < 250 )
 825                 {
 826                         // Fast forward - seeking is inefficient for small distances - just ignore following frames
 827                         ignore = ( int )( ( position - this->video_expected ) / fps * source_fps );
 828                         codec_context->skip_loop_filter = AVDISCARD_NONREF;
 829                 }
 830                 else if ( this->seekable && ( position < this->video_expected || position - this->video_expected >= 12 || last_position < 0 ) )
 831                 {
 832                         if ( use_new_seek && last_position == POSITION_INITIAL )
 833                         {
 834                                 // find first key frame
 835                                 int ret = 0;
 836                                 int toscan = 100;
 837
 838                                 while ( ret >= 0 && toscan-- > 0 )
 839                                 {
 840                                         ret = av_read_frame( context, &pkt );
 841                                         if ( ret >= 0 && ( pkt.flags & PKT_FLAG_KEY ) && pkt.stream_index == this->video_index )
 842                                         {
 843                                                 mlt_log_verbose( MLT_PRODUCER_SERVICE(producer), "first_pts %lld dts %lld pts_dts_delta %d\n", pkt.pts, pkt.dts, (int)(pkt.pts - pkt.dts) );
 844                                                 this->first_pts = pkt.pts;
 845                                                 toscan = 0;
 846                                         }
 847                                         av_free_packet( &pkt );
 848                                 }
 849                                 // Rewind
 850                                 av_seek_frame( context, -1, 0, AVSEEK_FLAG_BACKWARD );
 851                         }
 852
 853                         // Calculate the timestamp for the requested frame
 854                         int64_t timestamp;
 855                         if ( use_new_seek )
 856                         {
 857                                 timestamp = ( req_position - 0.1 / source_fps ) /
 858                                         ( av_q2d( stream->time_base ) * source_fps );
 859                                 mlt_log_verbose( MLT_PRODUCER_SERVICE(producer), "pos %d pts %lld ", req_position, timestamp );
 860                                 if ( this->first_pts > 0 )
 861                                         timestamp += this->first_pts;
 862                                 else if ( context->start_time != AV_NOPTS_VALUE )
 863                                         timestamp += context->start_time;
 864                         }
 865                         else
 866                         {
 867                                 timestamp = ( int64_t )( ( double )req_position / source_fps * AV_TIME_BASE + 0.5 );
 868                                 if ( context->start_time != AV_NOPTS_VALUE )
 869                                         timestamp += context->start_time;
 870                         }
 871                         if ( must_decode )
 872                                 timestamp -= AV_TIME_BASE;
 873                         if ( timestamp < 0 )
 874                                 timestamp = 0;
 875                         mlt_log_debug( MLT_PRODUCER_SERVICE(producer), "seeking timestamp %lld position %d expected %d last_pos %d\n",
 876                                 timestamp, position, this->video_expected, last_position );
 877
 878                         // Seek to the timestamp
 879                         if ( use_new_seek )
 880                         {
 881                                 codec_context->skip_loop_filter = AVDISCARD_NONREF;
 882                                 av_seek_frame( context, this->video_index, timestamp, AVSEEK_FLAG_BACKWARD );
 883                         }
 884                         else
 885                         {
 886                                 av_seek_frame( context, -1, timestamp, AVSEEK_FLAG_BACKWARD );
 887                         }
 888
 889                         // Remove the cached info relating to the previous position
 890                         this->current_position = POSITION_INVALID;
 891                         this->last_position = POSITION_INVALID;
 892                         av_freep( &this->av_frame );
 893
 894                         if ( use_new_seek )
 895                         {
 896                                 // flush any pictures still in decode buffer
 897                                 avcodec_flush_buffers( codec_context );
 898                         }
 899                 }
 900         }
 901
 902         // Duplicate the last image if necessary (see comment on rawvideo below)
 903         if ( this->av_frame && this->got_picture && this->seekable
 904                  && ( paused
 905                           || this->current_position == req_position
 906                           || ( !use_new_seek && this->current_position > req_position ) ) )
 907         {
 908                 // Duplicate it
 909                 if ( allocate_buffer( frame_properties, codec_context, buffer, format, width, height ) )
 910                         convert_image( this->av_frame, *buffer, codec_context->pix_fmt, format, *width, *height );
 911                 else
 912                         mlt_frame_get_image( frame, buffer, format, width, height, writable );
 913         }
 914         else
 915         {
 916                 int ret = 0;
 917                 int int_position = 0;
 918                 int decode_errors = 0;
 919                 int got_picture = 0;
 920
 921                 av_init_packet( &pkt );
 922
 923                 // Construct an AVFrame for YUV422 conversion
 924                 if ( !this->av_frame )
 925                         this->av_frame = avcodec_alloc_frame( );
 926
 927                 while( ret >= 0 && !got_picture )
 928                 {
 929                         // Read a packet
 930                         ret = av_read_frame( context, &pkt );
 931
 932                         // We only deal with video from the selected video_index
 933                         if ( ret >= 0 && pkt.stream_index == this->video_index && pkt.size > 0 )
 934                         {
 935                                 // Determine time code of the packet
 936                                 if ( use_new_seek )
 937                                 {
 938                                         int64_t pts = pkt.pts;
 939                                         if ( this->first_pts > 0 )
 940                                                 pts -= this->first_pts;
 941                                         else if ( context->start_time != AV_NOPTS_VALUE )
 942                                                 pts -= context->start_time;
 943                                         int_position = ( int )( av_q2d( stream->time_base ) * pts * source_fps + 0.1 );
 944                                 }
 945                                 else
 946                                 {
 947                                         if ( pkt.dts != AV_NOPTS_VALUE )
 948                                         {
 949                                                 int_position = ( int )( av_q2d( stream->time_base ) * pkt.dts * source_fps + 0.5 );
 950                                                 if ( context->start_time != AV_NOPTS_VALUE )
 951                                                         int_position -= ( int )( context->start_time * source_fps / AV_TIME_BASE + 0.5 );
 952                                                 last_position = this->last_position;
 953                                                 if ( int_position == last_position )
 954                                                         int_position = last_position + 1;
 955                                         }
 956                                         else
 957                                         {
 958                                                 int_position = req_position;
 959                                         }
 960                                         mlt_log_debug( MLT_PRODUCER_SERVICE(producer), "pkt.dts %llu req_pos %d cur_pos %d pkt_pos %d",
 961                                                 pkt.dts, req_position, this->current_position, int_position );
 962                                         // Make a dumb assumption on streams that contain wild timestamps
 963                                         if ( abs( req_position - int_position ) > 999 )
 964                                         {
 965                                                 int_position = req_position;
 966                                                 mlt_log_debug( MLT_PRODUCER_SERVICE(producer), " WILD TIMESTAMP!" );
 967                                         }
 968                                 }
 969                                 this->last_position = int_position;
 970
 971                                 // Decode the image
 972                                 if ( must_decode || int_position >= req_position )
 973                                 {
 974                                         codec_context->reordered_opaque = pkt.pts;
 975                                         if ( int_position >= req_position )
 976                                                 codec_context->skip_loop_filter = AVDISCARD_NONE;
 977                                         ret = avcodec_decode_video( codec_context, this->av_frame, &got_picture, pkt.data, pkt.size );
 978                                         // Note: decode may fail at the beginning of MPEGfile (B-frames referencing before first I-frame), so allow a few errors.
 979                                         if ( ret < 0 )
 980                                         {
 981                                                 if ( ++decode_errors <= 10 )
 982                                                         ret = 0;
 983                                         }
 984                                         else
 985                                         {
 986                                                 decode_errors = 0;
 987                                         }
 988                                 }
 989
 990                                 if ( got_picture )
 991                                 {
 992                                         if ( use_new_seek )
 993                                         {
 994                                                 // Determine time code of the packet
 995                                                 int64_t pts = this->av_frame->reordered_opaque;
 996                                                 if ( this->first_pts > 0 )
 997                                                         pts -= this->first_pts;
 998                                                 else if ( context->start_time != AV_NOPTS_VALUE )
 999                                                         pts -= context->start_time;
1000                                                 int_position = ( int )( av_q2d( stream->time_base) * pts * source_fps + 0.1 );
1001                                                 mlt_log_verbose( MLT_PRODUCER_SERVICE(producer), "got frame %d, key %d\n", int_position, this->av_frame->key_frame );
1002                                         }
1003                                         // Handle ignore
1004                                         if ( int_position < req_position )
1005                                         {
1006                                                 ignore = 0;
1007                                                 got_picture = 0;
1008                                         }
1009                                         else if ( int_position >= req_position )
1010                                         {
1011                                                 ignore = 0;
1012                                                 codec_context->skip_loop_filter = AVDISCARD_NONE;
1013                                         }
1014                                         else if ( ignore -- )
1015                                         {
1016                                                 got_picture = 0;
1017                                         }
1018                                 }
1019                                 mlt_log_debug( MLT_PRODUCER_SERVICE(producer), " got_pic %d key %d\n", got_picture, pkt.flags & PKT_FLAG_KEY );
1020                                 av_free_packet( &pkt );
1021                         }
1022                         else if ( ret >= 0 )
1023                         {
1024                                 av_free_packet( &pkt );
1025                         }
1026
1027                         // Now handle the picture if we have one
1028                         if ( got_picture )
1029                         {
1030                                 if ( allocate_buffer( frame_properties, codec_context, buffer, format, width, height ) )
1031                                 {
1032                                         convert_image( this->av_frame, *buffer, codec_context->pix_fmt, format, *width, *height );
1033                                         if ( !mlt_properties_get( properties, "force_progressive" ) )
1034                                                 mlt_properties_set_int( frame_properties, "progressive", !this->av_frame->interlaced_frame );
1035                                         this->top_field_first = this->av_frame->top_field_first;
1036                                         this->current_position = int_position;
1037                                         this->got_picture = 1;
1038                                 }
1039                                 else
1040                                 {
1041                                         got_picture = 0;
1042                                 }
1043                         }
1044                 }
1045                 if ( !got_picture )
1046                         mlt_frame_get_image( frame, buffer, format, width, height, writable );
1047         }
1048
1049         // Very untidy - for rawvideo, the packet contains the frame, hence the free packet
1050         // above will break the pause behaviour - so we wipe the frame now
1051         if ( !strcmp( codec_context->codec->name, "rawvideo" ) )
1052                 av_freep( &this->av_frame );
1053
1054         avformat_unlock();
1055
1056         // Set the field order property for this frame
1057         mlt_properties_set_int( frame_properties, "top_field_first", this->top_field_first );
1058
1059         // Regardless of speed, we expect to get the next frame (cos we ain't too bright)
1060         this->video_expected = position + 1;
1061
1062         return 0;
1063 }
1064
1065 /** Process properties as AVOptions and apply to AV context obj
1066 */
1067
1068 static void apply_properties( void *obj, mlt_properties properties, int flags )
1069 {
1070         int i;
1071         int count = mlt_properties_count( properties );
1072         for ( i = 0; i < count; i++ )
1073         {
1074                 const char *opt_name = mlt_properties_get_name( properties, i );
1075                 const AVOption *opt = av_find_opt( obj, opt_name, NULL, flags, flags );
1076                 if ( opt )
1077 #if LIBAVCODEC_VERSION_INT >= ((52<<16)+(7<<8)+0)
1078                         av_set_string3( obj, opt_name, mlt_properties_get( properties, opt_name), 0, NULL );
1079 #elif LIBAVCODEC_VERSION_INT >= ((51<<16)+(59<<8)+0)
1080                         av_set_string2( obj, opt_name, mlt_properties_get( properties, opt_name), 0 );
1081 #else
1082                         av_set_string( obj, opt_name, mlt_properties_get( properties, opt_name) );
1083 #endif
1084         }
1085 }
1086
1087 /** Set up video handling.
1088 */
1089
1090 static void producer_set_up_video( producer_avformat this, mlt_frame frame )
1091 {
1092         // Get the producer
1093         mlt_producer producer = &this->parent;
1094
1095         // Get the properties
1096         mlt_properties properties = MLT_PRODUCER_PROPERTIES( producer );
1097
1098         // Fetch the video_context
1099         AVFormatContext *context = this->video_context;
1100
1101         // Get the video_index
1102         int index = mlt_properties_get_int( properties, "video_index" );
1103
1104         // Reopen the file if necessary
1105         if ( !context && index > -1 )
1106         {
1107                 mlt_events_block( properties, producer );
1108                 producer_open( this, mlt_service_profile( MLT_PRODUCER_SERVICE(producer) ),
1109                         mlt_properties_get( properties, "resource" ) );
1110                 context = this->video_context;
1111                 producer_file_close( this->dummy_context );
1112                 this->dummy_context = NULL;
1113                 mlt_events_unblock( properties, producer );
1114
1115                 // Process properties as AVOptions
1116                 apply_properties( context, properties, AV_OPT_FLAG_DECODING_PARAM );
1117         }
1118
1119         // Exception handling for video_index
1120         if ( context && index >= (int) context->nb_streams )
1121         {
1122                 // Get the last video stream
1123                 for ( index = context->nb_streams - 1;
1124                           index >= 0 && context->streams[ index ]->codec->codec_type != CODEC_TYPE_VIDEO;
1125                           index-- );
1126                 mlt_properties_set_int( properties, "video_index", index );
1127         }
1128         if ( context && index > -1 && context->streams[ index ]->codec->codec_type != CODEC_TYPE_VIDEO )
1129         {
1130                 // Invalidate the video stream
1131                 index = -1;
1132                 mlt_properties_set_int( properties, "video_index", index );
1133         }
1134
1135         // Get the frame properties
1136         mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame );
1137
1138         if ( context && index > -1 )
1139         {
1140                 // Get the video stream
1141                 AVStream *stream = context->streams[ index ];
1142
1143                 // Get codec context
1144                 AVCodecContext *codec_context = stream->codec;
1145
1146                 // Get the codec
1147                 AVCodec *codec = this->video_codec;
1148
1149                 // Update the video properties if the index changed
1150                 if ( index != this->video_index )
1151                 {
1152                         // Reset the video properties if the index changed
1153                         this->video_index = index;
1154                         producer_codec_close( this->video_codec );
1155                         this->video_codec = NULL;
1156                         mlt_properties_set_int( properties, "width", codec_context->width );
1157                         mlt_properties_set_int( properties, "height", codec_context->height );
1158                         // TODO: get the first usable AVPacket and reset the stream position
1159                         mlt_properties_set_double( properties, "aspect_ratio",
1160                                 get_aspect_ratio( context->streams[ index ], codec_context, NULL ) );
1161                         codec = NULL;
1162                 }
1163
1164                 // Initialise the codec if necessary
1165                 if ( !codec )
1166                 {
1167                         // Initialise multi-threading
1168                         int thread_count = mlt_properties_get_int( properties, "threads" );
1169                         if ( thread_count == 0 && getenv( "MLT_AVFORMAT_THREADS" ) )
1170                                 thread_count = atoi( getenv( "MLT_AVFORMAT_THREADS" ) );
1171                         if ( thread_count > 1 )
1172                         {
1173                                 avcodec_thread_init( codec_context, thread_count );
1174                                 codec_context->thread_count = thread_count;
1175                         }
1176
1177                         // Find the codec
1178                         codec = avcodec_find_decoder( codec_context->codec_id );
1179
1180                         // If we don't have a codec and we can't initialise it, we can't do much more...
1181                         avformat_lock( );
1182                         if ( codec && avcodec_open( codec_context, codec ) >= 0 )
1183                         {
1184                                 // Now store the codec with its destructor
1185                                 producer_codec_close( this->video_codec );
1186                                 this->video_codec = codec_context;
1187                         }
1188                         else
1189                         {
1190                                 // Remember that we can't use this later
1191                                 this->video_index = index = -1;
1192                         }
1193                         avformat_unlock( );
1194
1195                         // Process properties as AVOptions
1196                         apply_properties( codec_context, properties, AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM );
1197                 }
1198
1199                 // No codec, no show...
1200                 if ( codec && index > -1 )
1201                 {
1202                         double source_fps = 0;
1203                         double force_aspect_ratio = mlt_properties_get_double( properties, "force_aspect_ratio" );
1204                         double aspect_ratio = ( force_aspect_ratio > 0.0 ) ?
1205                                 force_aspect_ratio : mlt_properties_get_double( properties, "aspect_ratio" );
1206
1207                         // Determine the fps
1208                         source_fps = ( double )codec_context->time_base.den /
1209                                                                  ( codec_context->time_base.num == 0 ? 1 : codec_context->time_base.num );
1210
1211                         // If the muxer reports a frame rate different than the codec
1212                         double muxer_fps = av_q2d( context->streams[ index ]->r_frame_rate );
1213                         if ( source_fps != muxer_fps )
1214                                 // Choose the lesser - the wrong tends to be off by some multiple of 10
1215                                 source_fps = muxer_fps < source_fps ? muxer_fps : source_fps;
1216
1217                         // We'll use fps if it's available
1218                         if ( source_fps > 0 )
1219                                 mlt_properties_set_double( properties, "source_fps", source_fps );
1220                         else
1221                                 mlt_properties_set_double( properties, "source_fps", mlt_producer_get_fps( producer ) );
1222                         mlt_properties_set_double( properties, "aspect_ratio", aspect_ratio );
1223
1224                         // Set the width and height
1225                         mlt_properties_set_int( frame_properties, "width", codec_context->width );
1226                         mlt_properties_set_int( frame_properties, "height", codec_context->height );
1227                         mlt_properties_set_int( frame_properties, "real_width", codec_context->width );
1228                         mlt_properties_set_int( frame_properties, "real_height", codec_context->height );
1229                         mlt_properties_set_double( frame_properties, "aspect_ratio", aspect_ratio );
1230                         if ( mlt_properties_get( properties, "force_progressive" ) )
1231                                 mlt_properties_set_int( frame_properties, "progressive", mlt_properties_get_int( properties, "force_progressive" ) );
1232
1233                         mlt_frame_push_service( frame, this );
1234                         mlt_frame_push_get_image( frame, producer_get_image );
1235                 }
1236                 else
1237                 {
1238                         mlt_properties_set_int( frame_properties, "test_image", 1 );
1239                 }
1240         }
1241         else
1242         {
1243                 mlt_properties_set_int( frame_properties, "test_image", 1 );
1244         }
1245 }
1246
1247 /** Get the audio from a frame.
1248 */
1249
1250 static int producer_get_audio( mlt_frame frame, void **buffer, mlt_audio_format *format, int *frequency, int *channels, int *samples )
1251 {
1252         // Get the producer
1253         producer_avformat this = mlt_frame_pop_audio( frame );
1254         mlt_producer producer = &this->parent;
1255
1256         // Get the properties from the frame
1257         mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame );
1258
1259         // Obtain the frame number of this frame
1260         mlt_position position = mlt_properties_get_position( frame_properties, "avformat_position" );
1261
1262         // Get the producer properties
1263         mlt_properties properties = MLT_PRODUCER_PROPERTIES( producer );
1264
1265         // Fetch the audio_context
1266         AVFormatContext *context = this->audio_context;
1267
1268         // Get the audio stream
1269         AVStream *stream = context->streams[ this->audio_index ];
1270
1271         // Get codec context
1272         AVCodecContext *codec_context = stream->codec;
1273
1274         // Packet
1275         AVPacket pkt;
1276
1277         // Obtain the resample context if it exists (not always needed)
1278         ReSampleContext *resample = this->audio_resample;
1279
1280         // Obtain the audio buffers
1281         int16_t *audio_buffer = this->audio_buffer;
1282         int16_t *decode_buffer = this->decode_buffer;
1283
1284         // Get amount of audio used
1285         int audio_used =  this->audio_used;
1286
1287         // Calculate the real time code
1288         double real_timecode = producer_time_of_frame( producer, position );
1289
1290         // Number of frames to ignore (for ffwd)
1291         int ignore = 0;
1292
1293         // Flag for paused (silence)
1294         int paused = 0;
1295
1296         // Check for resample and create if necessary
1297         if ( !resample && codec_context->channels <= 2 )
1298         {
1299                 // Create the resampler
1300 #if (LIBAVCODEC_VERSION_INT >= ((52<<16)+(15<<8)+0))
1301                 resample = av_audio_resample_init( *channels, codec_context->channels, *frequency, codec_context->sample_rate,
1302                         SAMPLE_FMT_S16, codec_context->sample_fmt, 16, 10, 0, 0.8 );
1303 #else
1304                 resample = audio_resample_init( *channels, codec_context->channels, *frequency, codec_context->sample_rate );
1305 #endif
1306
1307                 // And store it on properties
1308                 if ( this->audio_resample ) audio_resample_close( this->audio_resample );
1309                 this->audio_resample = resample;
1310         }
1311         else if ( !resample )
1312         {
1313                 // TODO: uncomment and remove following line when full multi-channel support is ready
1314                 // *channels = codec_context->channels;
1315                 codec_context->request_channels = *channels;
1316
1317                 *frequency = codec_context->sample_rate;
1318         }
1319
1320         // Check for audio buffer and create if necessary
1321         if ( !audio_buffer )
1322                 this->audio_buffer = audio_buffer = mlt_pool_alloc( AVCODEC_MAX_AUDIO_FRAME_SIZE * sizeof( int16_t ) );
1323
1324         // Check for decoder buffer and create if necessary
1325         if ( !decode_buffer )
1326                 this->decode_buffer = decode_buffer = av_malloc( AVCODEC_MAX_AUDIO_FRAME_SIZE * sizeof( int16_t ) );
1327
1328         // Seek if necessary
1329         if ( position != this->audio_expected )
1330         {
1331                 if ( position + 1 == this->audio_expected )
1332                 {
1333                         // We're paused - silence required
1334                         paused = 1;
1335                 }
1336                 else if ( !this->seekable && position > this->audio_expected && ( position - this->audio_expected ) < 250 )
1337                 {
1338                         // Fast forward - seeking is inefficient for small distances - just ignore following frames
1339                         ignore = position - this->audio_expected;
1340                 }
1341                 else if ( position < this->audio_expected || position - this->audio_expected >= 12 )
1342                 {
1343                         int64_t timestamp = ( int64_t )( real_timecode * AV_TIME_BASE + 0.5 );
1344                         if ( context->start_time != AV_NOPTS_VALUE )
1345                                 timestamp += context->start_time;
1346                         if ( timestamp < 0 )
1347                                 timestamp = 0;
1348
1349                         // Set to the real timecode
1350                         if ( av_seek_frame( context, -1, timestamp, AVSEEK_FLAG_BACKWARD ) != 0 )
1351                                 paused = 1;
1352
1353                         // Clear the usage in the audio buffer
1354                         audio_used = 0;
1355                 }
1356         }
1357
1358         // Get the audio if required
1359         if ( !paused )
1360         {
1361                 int ret = 0;
1362                 int got_audio = 0;
1363
1364                 av_init_packet( &pkt );
1365
1366                 while( ret >= 0 && !got_audio )
1367                 {
1368                         // Check if the buffer already contains the samples required
1369                         if ( audio_used >= *samples && ignore == 0 )
1370                         {
1371                                 got_audio = 1;
1372                                 break;
1373                         }
1374
1375                         // Read a packet
1376                         ret = av_read_frame( context, &pkt );
1377
1378                         int len = pkt.size;
1379                         uint8_t *ptr = pkt.data;
1380
1381                         // We only deal with audio from the selected audio_index
1382                         while ( ptr && ret >= 0 && pkt.stream_index == this->audio_index && len > 0 )
1383                         {
1384                                 int data_size = sizeof( int16_t ) * AVCODEC_MAX_AUDIO_FRAME_SIZE;
1385
1386                                 // Decode the audio
1387 #if (LIBAVCODEC_VERSION_INT >= ((51<<16)+(29<<8)+0))
1388                                 ret = avcodec_decode_audio2( codec_context, decode_buffer, &data_size, ptr, len );
1389 #else
1390                                 ret = avcodec_decode_audio( codec_context, decode_buffer, &data_size, ptr, len );
1391 #endif
1392                                 if ( ret < 0 )
1393                                 {
1394                                         ret = 0;
1395                                         break;
1396                                 }
1397
1398                                 len -= ret;
1399                                 ptr += ret;
1400
1401                                 if ( data_size > 0 && ( audio_used * *channels + data_size < AVCODEC_MAX_AUDIO_FRAME_SIZE ) )
1402                                 {
1403                                         if ( resample )
1404                                         {
1405                                                 int16_t *source = decode_buffer;
1406                                                 int16_t *dest = &audio_buffer[ audio_used * *channels ];
1407                                                 int convert_samples = data_size / av_get_bits_per_sample_format( codec_context->sample_fmt )
1408                                                                                           * 8 / codec_context->channels;
1409
1410                                                 audio_used += audio_resample( resample, dest, source, convert_samples );
1411                                         }
1412                                         else
1413                                         {
1414                                                 memcpy( &audio_buffer[ audio_used * *channels ], decode_buffer, data_size );
1415                                                 audio_used += data_size / *channels / av_get_bits_per_sample_format( codec_context->sample_fmt ) * 8;
1416                                         }
1417
1418                                         // Handle ignore
1419                                         while ( ignore && audio_used > *samples )
1420                                         {
1421                                                 ignore --;
1422                                                 audio_used -= *samples;
1423                                                 memmove( audio_buffer, &audio_buffer[ *samples * *channels ], audio_used * sizeof( int16_t ) );
1424                                         }
1425                                 }
1426
1427                                 // If we're behind, ignore this packet
1428                                 if ( pkt.pts >= 0 )
1429                                 {
1430                                         double current_pts = av_q2d( stream->time_base ) * pkt.pts;
1431                                         double source_fps = mlt_properties_get_double( properties, "source_fps" );
1432                                         int req_position = ( int )( real_timecode * source_fps + 0.5 );
1433                                         int int_position = ( int )( current_pts * source_fps + 0.5 );
1434
1435                                         if ( context->start_time != AV_NOPTS_VALUE )
1436                                                 int_position -= ( int )( context->start_time * source_fps / AV_TIME_BASE + 0.5 );
1437                                         if ( this->seekable && !ignore && int_position < req_position )
1438                                                 ignore = 1;
1439                                 }
1440                         }
1441
1442                         // We're finished with this packet regardless
1443                         av_free_packet( &pkt );
1444                 }
1445
1446                 int size = *samples * *channels * sizeof( int16_t );
1447                 *format = mlt_audio_s16;
1448                 *buffer = mlt_pool_alloc( size );
1449                 mlt_frame_set_audio( frame, *buffer, *format, size, mlt_pool_release );
1450
1451                 // Now handle the audio if we have enough
1452                 if ( audio_used >= *samples )
1453                 {
1454                         memcpy( *buffer, audio_buffer, *samples * *channels * sizeof( int16_t ) );
1455                         audio_used -= *samples;
1456                         memmove( audio_buffer, &audio_buffer[ *samples * *channels ], audio_used * *channels * sizeof( int16_t ) );
1457                 }
1458                 else
1459                 {
1460                         memset( *buffer, 0, *samples * *channels * sizeof( int16_t ) );
1461                 }
1462
1463                 // Store the number of audio samples still available
1464                 this->audio_used = audio_used;
1465         }
1466         else
1467         {
1468                 // Get silence and don't touch the context
1469                 mlt_frame_get_audio( frame, buffer, format, frequency, channels, samples );
1470         }
1471
1472         // Regardless of speed (other than paused), we expect to get the next frame
1473         if ( !paused )
1474                 this->audio_expected = position + 1;
1475
1476         return 0;
1477 }
1478
1479 /** Set up audio handling.
1480 */
1481
1482 static void producer_set_up_audio( producer_avformat this, mlt_frame frame )
1483 {
1484         // Get the producer
1485         mlt_producer producer = &this->parent;
1486
1487         // Get the properties
1488         mlt_properties properties = MLT_PRODUCER_PROPERTIES( producer );
1489
1490         // Fetch the audio_context
1491         AVFormatContext *context = this->audio_context;
1492
1493         // Get the audio_index
1494         int index = mlt_properties_get_int( properties, "audio_index" );
1495
1496         // Reopen the file if necessary
1497         if ( !context && index > -1 )
1498         {
1499                 mlt_events_block( properties, producer );
1500                 producer_open( this, mlt_service_profile( MLT_PRODUCER_SERVICE(producer) ),
1501                         mlt_properties_get( properties, "resource" ) );
1502                 context = this->audio_context;
1503                 producer_file_close( this->dummy_context );
1504                 this->dummy_context = NULL;
1505                 mlt_events_unblock( properties, producer );
1506         }
1507
1508         // Exception handling for audio_index
1509         if ( context && index >= (int) context->nb_streams )
1510         {
1511                 for ( index = context->nb_streams - 1;
1512                           index >= 0 && context->streams[ index ]->codec->codec_type != CODEC_TYPE_AUDIO;
1513                           index-- );
1514                 mlt_properties_set_int( properties, "audio_index", index );
1515         }
1516         if ( context && index > -1 && context->streams[ index ]->codec->codec_type != CODEC_TYPE_AUDIO )
1517         {
1518                 index = -1;
1519                 mlt_properties_set_int( properties, "audio_index", index );
1520         }
1521
1522         // Update the audio properties if the index changed
1523         if ( index > -1 && index != this->audio_index )
1524         {
1525                 this->audio_index = index;
1526                 producer_codec_close( this->audio_codec );
1527                 this->audio_codec = NULL;
1528         }
1529
1530         // Deal with audio context
1531         if ( context && index > -1 )
1532         {
1533                 // Get the frame properties
1534                 mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame );
1535
1536                 // Get the audio stream
1537                 AVStream *stream = context->streams[ index ];
1538
1539                 // Get codec context
1540                 AVCodecContext *codec_context = stream->codec;
1541
1542                 // Get the codec
1543                 AVCodec *codec = this->audio_codec;
1544
1545                 // Initialise the codec if necessary
1546                 if ( !codec )
1547                 {
1548                         // Find the codec
1549                         codec = avcodec_find_decoder( codec_context->codec_id );
1550
1551                         // If we don't have a codec and we can't initialise it, we can't do much more...
1552                         avformat_lock( );
1553                         if ( codec && avcodec_open( codec_context, codec ) >= 0 )
1554                         {
1555                                 // Now store the codec with its destructor
1556                                 producer_codec_close( this->audio_codec );
1557                                 this->audio_codec = codec_context;
1558
1559                         }
1560                         else
1561                         {
1562                                 // Remember that we can't use this later
1563                                 this->audio_index = index = -1;
1564                         }
1565                         avformat_unlock( );
1566
1567                         // Process properties as AVOptions
1568                         apply_properties( codec_context, properties, AV_OPT_FLAG_AUDIO_PARAM | AV_OPT_FLAG_DECODING_PARAM );
1569                 }
1570
1571                 // No codec, no show...
1572                 if ( codec && index > -1 )
1573                 {
1574                         mlt_frame_push_audio( frame, this );
1575                         mlt_frame_push_audio( frame, producer_get_audio );
1576                         mlt_properties_set_int( frame_properties, "frequency", codec_context->sample_rate );
1577                         mlt_properties_set_int( frame_properties, "channels", codec_context->channels );
1578                 }
1579         }
1580 }
1581
1582 /** Our get frame implementation.
1583 */
1584
1585 static int producer_get_frame( mlt_producer producer, mlt_frame_ptr frame, int index )
1586 {
1587         // Access the private data
1588         producer_avformat this = producer->child;
1589
1590         // Create an empty frame
1591         *frame = mlt_frame_init( MLT_PRODUCER_SERVICE( producer ) );
1592
1593         // Update timecode on the frame we're creating
1594         mlt_frame_set_position( *frame, mlt_producer_position( producer ) );
1595
1596         // Set the position of this producer
1597         mlt_properties_set_position( MLT_FRAME_PROPERTIES( *frame ), "avformat_position", mlt_producer_frame( producer ) );
1598
1599         // Set up the video
1600         producer_set_up_video( this, *frame );
1601
1602         // Set up the audio
1603         producer_set_up_audio( this, *frame );
1604
1605         // Set the aspect_ratio
1606         mlt_properties_set_double( MLT_FRAME_PROPERTIES( *frame ), "aspect_ratio",
1607                 mlt_properties_get_double( MLT_PRODUCER_PROPERTIES( producer ), "aspect_ratio" ) );
1608
1609         // Calculate the next timecode
1610         mlt_producer_prepare_next( producer );
1611
1612         return 0;
1613 }
1614
1615 static void producer_close( mlt_producer parent )
1616 {
1617         // Obtain this
1618         producer_avformat this = parent->child;
1619
1620         // Close the file
1621         av_free( this->av_frame );
1622         if ( this->audio_resample )
1623                 audio_resample_close( this->audio_resample );
1624         mlt_pool_release( this->audio_buffer );
1625         av_free( this->decode_buffer );
1626         producer_file_close( this->dummy_context );
1627         producer_file_close( this->audio_context );
1628         producer_file_close( this->video_context );
1629         producer_codec_close( this->audio_codec );
1630         producer_codec_close( this->video_codec );
1631
1632         // Close the parent
1633         parent->close = NULL;
1634         mlt_producer_close( parent );
1635
1636         // Free the memory
1637         free( this );
1638 }