X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=src%2Fmodules%2Favformat%2Fproducer_avformat.c;h=8f4274cbf42dd3e532f638c3edc57ab9ffe05620;hb=0d8eb5b94dcfadffd8e4d67efd62ec09a0934c7f;hp=1afa5daeb808cb9fadd08bcebe9f5fa90ab42334;hpb=2e7094eebd8d7ade29c187e421491c35122ddfe6;p=mlt diff --git a/src/modules/avformat/producer_avformat.c b/src/modules/avformat/producer_avformat.c index 1afa5dae..8f4274cb 100644 --- a/src/modules/avformat/producer_avformat.c +++ b/src/modules/avformat/producer_avformat.c @@ -1,7 +1,8 @@ /* * producer_avformat.c -- avformat producer - * Copyright (C) 2003-2004 Ushodaya Enterprises Limited + * Copyright (C) 2003-2009 Ushodaya Enterprises Limited * Author: Charles Yates + * Author: Dan Dennedy * Much code borrowed from ffmpeg.c: Copyright (c) 2000-2003 Fabrice Bellard * * This library is free software; you can redistribute it and/or @@ -24,6 +25,9 @@ #include #include #include +#include +#include +#include // ffmpeg Header files #include @@ -34,30 +38,96 @@ #if (LIBAVCODEC_VERSION_INT >= ((51<<16)+(71<<8)+0)) # include "audioconvert.h" #endif +#ifdef VDPAU +#include +#endif // System header files #include #include #include +#include #if LIBAVUTIL_VERSION_INT < (50<<16) #define PIX_FMT_RGB32 PIX_FMT_RGBA32 #define PIX_FMT_YUYV422 PIX_FMT_YUV422 #endif +#define POSITION_INITIAL (-2) +#define POSITION_INVALID (-1) + +#define MAX_AUDIO_STREAMS (10) +#define MAX_VDPAU_SURFACES (10) + void avformat_lock( ); void avformat_unlock( ); +struct producer_avformat_s +{ + mlt_producer parent; + AVFormatContext *dummy_context; + AVFormatContext *audio_format; + AVFormatContext *video_format; + AVCodecContext *audio_codec[ MAX_AUDIO_STREAMS ]; + AVCodecContext *video_codec; + AVFrame *av_frame; + ReSampleContext *audio_resample[ MAX_AUDIO_STREAMS ]; + mlt_position audio_expected; + mlt_position video_expected; + int audio_index; + int video_index; + double start_time; + int first_pts; + int last_position; + int seekable; + int current_position; + int got_picture; + int top_field_first; + int16_t *audio_buffer[ MAX_AUDIO_STREAMS ]; + size_t audio_buffer_size[ MAX_AUDIO_STREAMS ]; + int16_t *decode_buffer[ MAX_AUDIO_STREAMS ]; + int audio_used[ MAX_AUDIO_STREAMS ]; + int audio_streams; + int audio_max_stream; + int total_channels; + int max_channel; + int max_frequency; + unsigned int invalid_pts_counter; + double resample_factor; + mlt_cache image_cache; +#ifdef VDPAU + struct + { + // from FFmpeg + struct vdpau_render_state render_states[MAX_VDPAU_SURFACES]; + + // internal + mlt_deque deque; + int b_age; + int ip_age[2]; + int is_decoded; + uint8_t *buffer; + } *vdpau; +#endif +}; +typedef struct producer_avformat_s *producer_avformat; + // Forward references. -static int producer_open( mlt_producer this, mlt_profile profile, char *file ); +static int producer_open( producer_avformat this, mlt_profile profile, char *file ); static int producer_get_frame( mlt_producer this, mlt_frame_ptr frame, int index ); +static void producer_avformat_close( producer_avformat ); +static void producer_close( mlt_producer parent ); + +#ifdef VDPAU +#include "vdpau.c" +#endif /** Constructor for libavformat. */ -mlt_producer producer_avformat_init( mlt_profile profile, char *file ) +mlt_producer producer_avformat_init( mlt_profile profile, const char *service, char *file ) { - int error = 0; + int skip = 0; // Report information about available demuxers and codecs as YAML Tiny if ( file && strstr( file, "f-list" ) ) @@ -67,7 +137,7 @@ mlt_producer producer_avformat_init( mlt_profile profile, char *file ) while ( ( format = av_iformat_next( format ) ) ) fprintf( stderr, " - %s\n", format->name ); fprintf( stderr, "...\n" ); - error = 1; + skip = 1; } if ( file && strstr( file, "acodec-list" ) ) { @@ -77,7 +147,7 @@ mlt_producer producer_avformat_init( mlt_profile profile, char *file ) if ( codec->decode && codec->type == CODEC_TYPE_AUDIO ) fprintf( stderr, " - %s\n", codec->name ); fprintf( stderr, "...\n" ); - error = 1; + skip = 1; } if ( file && strstr( file, "vcodec-list" ) ) { @@ -87,53 +157,78 @@ mlt_producer producer_avformat_init( mlt_profile profile, char *file ) if ( codec->decode && codec->type == CODEC_TYPE_VIDEO ) fprintf( stderr, " - %s\n", codec->name ); fprintf( stderr, "...\n" ); - error = 1; + skip = 1; } - if ( error ) - return NULL; - - mlt_producer this = NULL; // Check that we have a non-NULL argument - if ( file != NULL ) + if ( !skip && file ) { // Construct the producer - this = calloc( 1, sizeof( struct mlt_producer_s ) ); + mlt_producer producer = calloc( 1, sizeof( struct mlt_producer_s ) ); + producer_avformat this = calloc( 1, sizeof( struct producer_avformat_s ) ); // Initialise it - if ( mlt_producer_init( this, NULL ) == 0 ) + if ( mlt_producer_init( producer, this ) == 0 ) { + this->parent = producer; + // Get the properties - mlt_properties properties = MLT_PRODUCER_PROPERTIES( this ); + mlt_properties properties = MLT_PRODUCER_PROPERTIES( producer ); // Set the resource property (required for all producers) mlt_properties_set( properties, "resource", file ); - // Register our get_frame implementation - this->get_frame = producer_get_frame; + // Register transport implementation with the producer + producer->close = (mlt_destructor) producer_close; - // Open the file - if ( producer_open( this, profile, file ) != 0 ) + // Register our get_frame implementation + producer->get_frame = producer_get_frame; + + if ( strcmp( service, "avformat-novalidate" ) ) { - // Clean up - mlt_producer_close( this ); - this = NULL; + // Open the file + if ( producer_open( this, profile, file ) != 0 ) + { + // Clean up + mlt_producer_close( producer ); + producer = NULL; + } + else + { + // Close the file to release resources for large playlists - reopen later as needed + avformat_lock(); + if ( this->dummy_context ) + av_close_input_file( this->dummy_context ); + this->dummy_context = NULL; + if ( this->audio_format ) + av_close_input_file( this->audio_format ); + this->audio_format = NULL; + if ( this->video_format ) + av_close_input_file( this->video_format ); + this->video_format = NULL; + avformat_unlock(); + + // Default the user-selectable indices from the auto-detected indices + mlt_properties_set_int( properties, "audio_index", this->audio_index ); + mlt_properties_set_int( properties, "video_index", this->video_index ); + +#ifdef VDPAU + mlt_service_cache_set_size( MLT_PRODUCER_SERVICE(producer), "producer_avformat", 5 ); +#endif + mlt_service_cache_put( MLT_PRODUCER_SERVICE(producer), "producer_avformat", this, 0, (mlt_destructor) producer_avformat_close ); + } } else { - // Close the file to release resources for large playlists - reopen later as needed - mlt_properties_set_data( properties, "dummy_context", NULL, 0, NULL, NULL ); - mlt_properties_set_data( properties, "audio_context", NULL, 0, NULL, NULL ); - mlt_properties_set_data( properties, "video_context", NULL, 0, NULL, NULL ); - - // Default the user-selectable indices from the auto-detected indices - mlt_properties_set_int( properties, "audio_index", mlt_properties_get_int( properties, "_audio_index" ) ); - mlt_properties_set_int( properties, "video_index", mlt_properties_get_int( properties, "_video_index" ) ); +#ifdef VDPAU + mlt_service_cache_set_size( MLT_PRODUCER_SERVICE(producer), "producer_avformat", 5 ); +#endif + mlt_service_cache_put( MLT_PRODUCER_SERVICE(producer), "producer_avformat", this, 0, (mlt_destructor) producer_avformat_close ); } + return producer; } } - - return this; + return NULL; } /** Find the default streams. @@ -172,6 +267,9 @@ static mlt_properties find_default_streams( mlt_properties meta_media, AVFormatC snprintf( key, sizeof(key), "meta.media.%d.stream.sample_aspect_ratio", i ); mlt_properties_set_double( meta_media, key, av_q2d( context->streams[ i ]->sample_aspect_ratio ) ); #endif + snprintf( key, sizeof(key), "meta.media.%d.codec.frame_rate", i ); + mlt_properties_set_double( meta_media, key, (double) codec_context->time_base.den / + ( codec_context->time_base.num == 0 ? 1 : codec_context->time_base.num ) ); snprintf( key, sizeof(key), "meta.media.%d.codec.pix_fmt", i ); mlt_properties_set( meta_media, key, avcodec_get_pix_fmt_name( codec_context->pix_fmt ) ); snprintf( key, sizeof(key), "meta.media.%d.codec.sample_aspect_ratio", i ); @@ -205,51 +303,15 @@ static mlt_properties find_default_streams( mlt_properties meta_media, AVFormatC mlt_properties_set_int( meta_media, key, codec_context->bit_rate ); // snprintf( key, sizeof(key), "meta.media.%d.codec.time_base", i ); // mlt_properties_set_double( meta_media, key, av_q2d( codec_context->time_base ) ); - snprintf( key, sizeof(key), "meta.media.%d.codec.profile", i ); - mlt_properties_set_int( meta_media, key, codec_context->profile ); - snprintf( key, sizeof(key), "meta.media.%d.codec.level", i ); - mlt_properties_set_int( meta_media, key, codec_context->level ); +// snprintf( key, sizeof(key), "meta.media.%d.codec.profile", i ); +// mlt_properties_set_int( meta_media, key, codec_context->profile ); +// snprintf( key, sizeof(key), "meta.media.%d.codec.level", i ); +// mlt_properties_set_int( meta_media, key, codec_context->level ); } return meta_media; } -/** Producer file destructor. -*/ - -static void producer_file_close( void *context ) -{ - if ( context != NULL ) - { - // Lock the mutex now - avformat_lock( ); - - // Close the file - av_close_input_file( context ); - - // Unlock the mutex now - avformat_unlock( ); - } -} - -/** Producer file destructor. -*/ - -static void producer_codec_close( void *codec ) -{ - if ( codec != NULL ) - { - // Lock the mutex now - avformat_lock( ); - - // Close the file - avcodec_close( codec ); - - // Unlock the mutex now - avformat_unlock( ); - } -} - static inline int dv_is_pal( AVPacket *pkt ) { return pkt->data[3] & 0x80; @@ -335,7 +397,7 @@ static double get_aspect_ratio( AVStream *stream, AVCodecContext *codec_context, /** Open the file. */ -static int producer_open( mlt_producer this, mlt_profile profile, char *file ) +static int producer_open( producer_avformat this, mlt_profile profile, char *file ) { // Return an error code (0 == no error) int error = 0; @@ -344,7 +406,7 @@ static int producer_open( mlt_producer this, mlt_profile profile, char *file ) AVFormatContext *context = NULL; // Get the properties - mlt_properties properties = MLT_PRODUCER_PROPERTIES( this ); + mlt_properties properties = MLT_PRODUCER_PROPERTIES( this->parent ); // We will treat everything with the producer fps double fps = mlt_profile_fps( profile ); @@ -387,8 +449,6 @@ static int producer_open( mlt_producer this, mlt_profile profile, char *file ) params->sample_rate = 48000; } - // XXX: this does not work anymore since avdevice - // TODO: make producer_avddevice? // Parse out params mrl = strchr( file, '?' ); while ( mrl ) @@ -436,18 +496,17 @@ static int producer_open( mlt_producer this, mlt_profile profile, char *file ) free( params ); // If successful, then try to get additional info - if ( error == 0 ) + if ( !error ) { // Get the stream info error = av_find_stream_info( context ) < 0; // Continue if no error - if ( error == 0 ) + if ( !error ) { // We will default to the first audio and video streams found int audio_index = -1; int video_index = -1; - int av_bypass = 0; // Now set properties where we can (use default unknowns if required) if ( context->duration != AV_NOPTS_VALUE ) @@ -462,23 +521,29 @@ static int producer_open( mlt_producer this, mlt_profile profile, char *file ) find_default_streams( properties, context, &audio_index, &video_index ); if ( context->start_time != AV_NOPTS_VALUE ) - mlt_properties_set_double( properties, "_start_time", context->start_time ); + this->start_time = context->start_time; // Check if we're seekable (something funny about mpeg here :-/) - if ( strcmp( file, "pipe:" ) && strncmp( file, "http://", 6 ) && strncmp( file, "udp:", 4 ) && strncmp( file, "tcp:", 4 ) && strncmp( file, "rtsp:", 5 ) && strncmp( file, "rtp:", 4 ) ) + if ( strncmp( file, "pipe:", 5 ) && + strncmp( file, "/dev/", 5 ) && + strncmp( file, "http:", 5 ) && + strncmp( file, "udp:", 4 ) && + strncmp( file, "tcp:", 4 ) && + strncmp( file, "rtsp:", 5 ) && + strncmp( file, "rtp:", 4 ) ) { - mlt_properties_set_int( properties, "seekable", av_seek_frame( context, -1, mlt_properties_get_double( properties, "_start_time" ), AVSEEK_FLAG_BACKWARD ) >= 0 ); - mlt_properties_set_data( properties, "dummy_context", context, 0, producer_file_close, NULL ); + this->seekable = av_seek_frame( context, -1, this->start_time, AVSEEK_FLAG_BACKWARD ) >= 0; + mlt_properties_set_int( properties, "seekable", this->seekable ); + this->dummy_context = context; av_open_input_file( &context, file, NULL, 0, NULL ); av_find_stream_info( context ); } - else - av_bypass = 1; // Store selected audio and video indexes on properties - mlt_properties_set_int( properties, "_audio_index", audio_index ); - mlt_properties_set_int( properties, "_video_index", video_index ); - mlt_properties_set_int( properties, "_last_position", -1 ); + this->audio_index = audio_index; + this->video_index = video_index; + this->first_pts = -1; + this->last_position = POSITION_INITIAL; // Fetch the width, height and aspect ratio if ( video_index != -1 ) @@ -511,51 +576,49 @@ static int producer_open( mlt_producer this, mlt_profile profile, char *file ) } // Read Metadata - if (context->title != NULL) + if ( context->title ) mlt_properties_set(properties, "meta.attr.title.markup", context->title ); - if (context->author != NULL) + if ( context->author ) mlt_properties_set(properties, "meta.attr.author.markup", context->author ); - if (context->copyright != NULL) + if ( context->copyright ) mlt_properties_set(properties, "meta.attr.copyright.markup", context->copyright ); - if (context->comment != NULL) + if ( context->comment ) mlt_properties_set(properties, "meta.attr.comment.markup", context->comment ); - if (context->album != NULL) + if ( context->album ) mlt_properties_set(properties, "meta.attr.album.markup", context->album ); - if (context->year != 0) + if ( context->year ) mlt_properties_set_int(properties, "meta.attr.year.markup", context->year ); - if (context->track != 0) + if ( context->track ) mlt_properties_set_int(properties, "meta.attr.track.markup", context->track ); // We're going to cheat here - for a/v files, we will have two contexts (reasoning will be clear later) if ( av == 0 && audio_index != -1 && video_index != -1 ) { - // We'll use the open one as our video_context - mlt_properties_set_data( properties, "video_context", context, 0, producer_file_close, NULL ); + // We'll use the open one as our video_format + this->video_format = context; // And open again for our audio context av_open_input_file( &context, file, NULL, 0, NULL ); av_find_stream_info( context ); // Audio context - mlt_properties_set_data( properties, "audio_context", context, 0, producer_file_close, NULL ); + this->audio_format = context; } else if ( av != 2 && video_index != -1 ) { // We only have a video context - mlt_properties_set_data( properties, "video_context", context, 0, producer_file_close, NULL ); + this->video_format = context; } else if ( audio_index != -1 ) { // We only have an audio context - mlt_properties_set_data( properties, "audio_context", context, 0, producer_file_close, NULL ); + this->audio_format = context; } else { // Something has gone wrong error = -1; } - - mlt_properties_set_int( properties, "av_bypass", av_bypass ); } } @@ -573,6 +636,46 @@ static double producer_time_of_frame( mlt_producer this, mlt_position position ) return ( double )position / mlt_producer_get_fps( this ); } + // Collect information about all audio streams + +static void get_audio_streams_info( producer_avformat this ) +{ + // Fetch the audio format context + AVFormatContext *context = this->audio_format; + int i; + + for ( i = 0; + i < context->nb_streams; + i++ ) + { + if ( context->streams[i]->codec->codec_type == CODEC_TYPE_AUDIO ) + { + AVCodecContext *codec_context = context->streams[i]->codec; + AVCodec *codec = avcodec_find_decoder( codec_context->codec_id ); + + // If we don't have a codec and we can't initialise it, we can't do much more... + avformat_lock( ); + if ( codec && avcodec_open( codec_context, codec ) >= 0 ) + { + this->audio_streams++; + this->audio_max_stream = i; + this->total_channels += codec_context->channels; + if ( codec_context->channels > this->max_channel ) + this->max_channel = codec_context->channels; + if ( codec_context->sample_rate > this->max_frequency ) + this->max_frequency = codec_context->sample_rate; + avcodec_close( codec_context ); + } + avformat_unlock( ); + } + } + mlt_log_verbose( NULL, "[producer avformat] audio: total_streams %d max_stream %d total_channels %d max_channels %d\n", + this->audio_streams, this->audio_max_stream, this->total_channels, this->max_channel ); + + // Other audio-specific initializations + this->resample_factor = 1.0; +} + static inline void convert_image( AVFrame *frame, uint8_t *buffer, int pix_fmt, mlt_image_format *format, int width, int height ) { #ifdef SWSCALE @@ -580,7 +683,7 @@ static inline void convert_image( AVFrame *frame, uint8_t *buffer, int pix_fmt, { *format = mlt_image_rgb24a; struct SwsContext *context = sws_getContext( width, height, pix_fmt, - width, height, PIX_FMT_RGBA, SWS_FAST_BILINEAR, NULL, NULL, NULL); + width, height, PIX_FMT_RGBA, SWS_BILINEAR, NULL, NULL, NULL); AVPicture output; avpicture_fill( &output, buffer, PIX_FMT_RGBA, width, height ); sws_scale( context, frame->data, frame->linesize, 0, height, @@ -590,11 +693,11 @@ static inline void convert_image( AVFrame *frame, uint8_t *buffer, int pix_fmt, else if ( *format == mlt_image_yuv420p ) { struct SwsContext *context = sws_getContext( width, height, pix_fmt, - width, height, PIX_FMT_YUV420P, SWS_FAST_BILINEAR, NULL, NULL, NULL); + width, height, PIX_FMT_YUV420P, SWS_BILINEAR, NULL, NULL, NULL); AVPicture output; output.data[0] = buffer; output.data[1] = buffer + width * height; - output.data[2] = buffer + ( 3 * width * height ) / 2; + output.data[2] = buffer + ( 5 * width * height ) / 4; output.linesize[0] = width; output.linesize[1] = width >> 1; output.linesize[2] = width >> 1; @@ -605,7 +708,7 @@ static inline void convert_image( AVFrame *frame, uint8_t *buffer, int pix_fmt, else if ( *format == mlt_image_rgb24 ) { struct SwsContext *context = sws_getContext( width, height, pix_fmt, - width, height, PIX_FMT_RGB24, SWS_FAST_BILINEAR, NULL, NULL, NULL); + width, height, PIX_FMT_RGB24, SWS_BILINEAR, NULL, NULL, NULL); AVPicture output; avpicture_fill( &output, buffer, PIX_FMT_RGB24, width, height ); sws_scale( context, frame->data, frame->linesize, 0, height, @@ -615,7 +718,7 @@ static inline void convert_image( AVFrame *frame, uint8_t *buffer, int pix_fmt, else if ( *format == mlt_image_rgb24a || *format == mlt_image_opengl ) { struct SwsContext *context = sws_getContext( width, height, pix_fmt, - width, height, PIX_FMT_RGBA, SWS_FAST_BILINEAR, NULL, NULL, NULL); + width, height, PIX_FMT_RGBA, SWS_BILINEAR, NULL, NULL, NULL); AVPicture output; avpicture_fill( &output, buffer, PIX_FMT_RGBA, width, height ); sws_scale( context, frame->data, frame->linesize, 0, height, @@ -625,7 +728,7 @@ static inline void convert_image( AVFrame *frame, uint8_t *buffer, int pix_fmt, else { struct SwsContext *context = sws_getContext( width, height, pix_fmt, - width, height, PIX_FMT_YUYV422, SWS_FAST_BILINEAR, NULL, NULL, NULL); + width, height, PIX_FMT_YUYV422, SWS_BILINEAR, NULL, NULL, NULL); AVPicture output; avpicture_fill( &output, buffer, PIX_FMT_YUYV422, width, height ); sws_scale( context, frame->data, frame->linesize, 0, height, @@ -638,7 +741,7 @@ static inline void convert_image( AVFrame *frame, uint8_t *buffer, int pix_fmt, AVPicture pict; pict.data[0] = buffer; pict.data[1] = buffer + width * height; - pict.data[2] = buffer + ( 3 * width * height ) / 2; + pict.data[2] = buffer + ( 5 * width * height ) / 4; pict.linesize[0] = width; pict.linesize[1] = width >> 1; pict.linesize[2] = width >> 1; @@ -703,7 +806,7 @@ static int allocate_buffer( mlt_properties frame_properties, AVCodecContext *cod // Construct the output image *buffer = mlt_pool_alloc( size ); if ( *buffer ) - mlt_properties_set_data( frame_properties, "image", *buffer, size, (mlt_destructor)mlt_pool_release, NULL ); + mlt_properties_set_data( frame_properties, "image", *buffer, size, mlt_pool_release, NULL ); else size = 0; @@ -715,39 +818,78 @@ static int allocate_buffer( mlt_properties frame_properties, AVCodecContext *cod static int producer_get_image( mlt_frame frame, uint8_t **buffer, mlt_image_format *format, int *width, int *height, int writable ) { + // Get the producer + producer_avformat this = mlt_frame_pop_service( frame ); + mlt_producer producer = this->parent; + // Get the properties from the frame mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame ); // Obtain the frame number of this frame mlt_position position = mlt_properties_get_position( frame_properties, "avformat_position" ); - // Get the producer - mlt_producer this = mlt_properties_get_data( frame_properties, "avformat_producer", NULL ); - // Get the producer properties - mlt_properties properties = MLT_PRODUCER_PROPERTIES( this ); + mlt_properties properties = MLT_PRODUCER_PROPERTIES( producer ); - // Fetch the video_context - AVFormatContext *context = mlt_properties_get_data( properties, "video_context", NULL ); - - // Get the video_index - int index = mlt_properties_get_int( properties, "video_index" ); - - // Obtain the expected frame numer - mlt_position expected = mlt_properties_get_position( properties, "_video_expected" ); + // Fetch the video format context + AVFormatContext *context = this->video_format; // Get the video stream - AVStream *stream = context->streams[ index ]; + AVStream *stream = context->streams[ this->video_index ]; // Get codec context AVCodecContext *codec_context = stream->codec; + // Get the image cache + if ( ! this->image_cache && ! mlt_properties_get_int( properties, "noimagecache" ) ) + this->image_cache = mlt_cache_init(); + if ( this->image_cache ) + { + mlt_cache_item item = mlt_cache_get( this->image_cache, (void*) position ); + *buffer = mlt_cache_item_data( item, format ); + if ( *buffer ) + { + // Set the resolution + *width = codec_context->width; + *height = codec_context->height; + mlt_properties_set_int( frame_properties, "width", *width ); + mlt_properties_set_int( frame_properties, "height", *height ); + + // Cache hit + int size; + switch ( *format ) + { + case mlt_image_yuv420p: + size = *width * 3 * ( *height + 1 ) / 2; + break; + case mlt_image_rgb24: + size = *width * ( *height + 1 ) * 3; + break; + case mlt_image_rgb24a: + case mlt_image_opengl: + size = *width * ( *height + 1 ) * 4; + break; + default: + *format = mlt_image_yuv422; + size = *width * ( *height + 1 ) * 2; + break; + } + mlt_properties_set_data( frame_properties, "avformat.image_cache", item, 0, ( mlt_destructor )mlt_cache_item_close, NULL ); + mlt_properties_set_data( frame_properties, "image", *buffer, size, NULL, NULL ); + this->top_field_first = mlt_properties_get_int( frame_properties, "top_field_first" ); + this->got_picture = 1; + + goto exit_get_image; + } + } + // Cache miss + int image_size = 0; + + avformat_lock(); + // Packet AVPacket pkt; - // Get the conversion frame - AVFrame *av_frame = mlt_properties_get_data( properties, "av_frame", NULL ); - // Special case pause handling flag int paused = 0; @@ -756,74 +898,135 @@ static int producer_get_image( mlt_frame frame, uint8_t **buffer, mlt_image_form // We may want to use the source fps if available double source_fps = mlt_properties_get_double( properties, "source_fps" ); - double fps = mlt_producer_get_fps( this ); + double fps = mlt_producer_get_fps( producer ); // This is the physical frame position in the source int req_position = ( int )( position / fps * source_fps + 0.5 ); - // Get the seekable status - int seekable = mlt_properties_get_int( properties, "seekable" ); - - // Hopefully provide better support for streams... - int av_bypass = mlt_properties_get_int( properties, "av_bypass" ); - // Determines if we have to decode all frames in a sequence - int must_decode = 1; - // Temporary hack to improve intra frame only - must_decode = strcmp( codec_context->codec->name, "dnxhd" ) && + int must_decode = strcmp( codec_context->codec->name, "dnxhd" ) && strcmp( codec_context->codec->name, "dvvideo" ) && strcmp( codec_context->codec->name, "huffyuv" ) && strcmp( codec_context->codec->name, "mjpeg" ) && strcmp( codec_context->codec->name, "rawvideo" ); - int last_position = mlt_properties_get_int( properties, "_last_position" ); + int last_position = this->last_position; + + // Turn on usage of new seek API and PTS for seeking + int use_new_seek = codec_context->codec_id == CODEC_ID_H264 && !strcmp( context->iformat->name, "mpegts" ); + if ( mlt_properties_get( properties, "new_seek" ) ) + use_new_seek = mlt_properties_get_int( properties, "new_seek" ); // Seek if necessary - if ( position != expected || last_position == -1 ) + if ( position != this->video_expected || last_position < 0 ) { - if ( av_frame != NULL && position + 1 == expected ) + if ( this->av_frame && position + 1 == this->video_expected ) { // We're paused - use last image paused = 1; } - else if ( !seekable && position > expected && ( position - expected ) < 250 ) + else if ( !this->seekable && position > this->video_expected && ( position - this->video_expected ) < 250 ) { // Fast forward - seeking is inefficient for small distances - just ignore following frames - ignore = ( int )( ( position - expected ) / fps * source_fps ); + ignore = ( int )( ( position - this->video_expected ) / fps * source_fps ); + codec_context->skip_loop_filter = AVDISCARD_NONREF; } - else if ( seekable && ( position < expected || position - expected >= 12 || last_position == -1 ) ) + else if ( this->seekable && ( position < this->video_expected || position - this->video_expected >= 12 || last_position < 0 ) ) { + if ( use_new_seek && last_position == POSITION_INITIAL ) + { + // find first key frame + int ret = 0; + int toscan = 100; + + while ( ret >= 0 && toscan-- > 0 ) + { + ret = av_read_frame( context, &pkt ); + if ( ret >= 0 && ( pkt.flags & PKT_FLAG_KEY ) && pkt.stream_index == this->video_index ) + { + mlt_log_verbose( MLT_PRODUCER_SERVICE(producer), "first_pts %lld dts %lld pts_dts_delta %d\n", pkt.pts, pkt.dts, (int)(pkt.pts - pkt.dts) ); + this->first_pts = pkt.pts; + toscan = 0; + } + av_free_packet( &pkt ); + } + // Rewind + av_seek_frame( context, -1, 0, AVSEEK_FLAG_BACKWARD ); + } + // Calculate the timestamp for the requested frame - int64_t timestamp = ( int64_t )( ( double )req_position / source_fps * AV_TIME_BASE + 0.5 ); - if ( ( uint64_t )context->start_time != AV_NOPTS_VALUE ) - timestamp += context->start_time; + int64_t timestamp; + if ( use_new_seek ) + { + timestamp = ( req_position - 0.1 / source_fps ) / + ( av_q2d( stream->time_base ) * source_fps ); + mlt_log_verbose( MLT_PRODUCER_SERVICE(producer), "pos %d pts %lld ", req_position, timestamp ); + if ( this->first_pts > 0 ) + timestamp += this->first_pts; + else if ( context->start_time != AV_NOPTS_VALUE ) + timestamp += context->start_time; + } + else + { + timestamp = ( int64_t )( ( double )req_position / source_fps * AV_TIME_BASE + 0.5 ); + if ( context->start_time != AV_NOPTS_VALUE ) + timestamp += context->start_time; + } if ( must_decode ) timestamp -= AV_TIME_BASE; if ( timestamp < 0 ) timestamp = 0; + mlt_log_debug( MLT_PRODUCER_SERVICE(producer), "seeking timestamp %lld position %d expected %d last_pos %d\n", + timestamp, position, this->video_expected, last_position ); - // Set to the timestamp - mlt_log_debug( MLT_PRODUCER_SERVICE( this ), "seeking timestamp %lld position %d expected %d last_pos %d\n", - timestamp, position, expected, last_position ); - av_seek_frame( context, -1, timestamp, AVSEEK_FLAG_BACKWARD ); + // Seek to the timestamp + if ( use_new_seek ) + { + codec_context->skip_loop_filter = AVDISCARD_NONREF; + av_seek_frame( context, this->video_index, timestamp, AVSEEK_FLAG_BACKWARD ); + } + else + { + av_seek_frame( context, -1, timestamp, AVSEEK_FLAG_BACKWARD ); + } // Remove the cached info relating to the previous position - mlt_properties_set_int( properties, "_current_position", -1 ); - mlt_properties_set_int( properties, "_last_position", -1 ); - mlt_properties_set_data( properties, "av_frame", NULL, 0, NULL, NULL ); - av_frame = NULL; + this->current_position = POSITION_INVALID; + this->last_position = POSITION_INVALID; + av_freep( &this->av_frame ); + + if ( use_new_seek ) + { + // flush any pictures still in decode buffer + avcodec_flush_buffers( codec_context ); + } } } // Duplicate the last image if necessary (see comment on rawvideo below) - int current_position = mlt_properties_get_int( properties, "_current_position" ); - int got_picture = mlt_properties_get_int( properties, "_got_picture" ); - if ( av_frame != NULL && got_picture && ( paused || current_position >= req_position ) && av_bypass == 0 ) + if ( this->av_frame && this->av_frame->linesize[0] && this->got_picture && this->seekable + && ( paused + || this->current_position == req_position + || ( !use_new_seek && this->current_position > req_position ) ) ) { // Duplicate it - if ( allocate_buffer( frame_properties, codec_context, buffer, format, width, height ) ) - convert_image( av_frame, *buffer, codec_context->pix_fmt, format, *width, *height ); + if ( ( image_size = allocate_buffer( frame_properties, codec_context, buffer, format, width, height ) ) ) +#ifdef VDPAU + if ( this->vdpau && this->vdpau->buffer ) + { + AVPicture picture; + picture.data[0] = this->vdpau->buffer; + picture.data[2] = this->vdpau->buffer + codec_context->width * codec_context->height; + picture.data[1] = this->vdpau->buffer + codec_context->width * codec_context->height * 5 / 4; + picture.linesize[0] = codec_context->width; + picture.linesize[1] = codec_context->width / 2; + picture.linesize[2] = codec_context->width / 2; + convert_image( (AVFrame*) &picture, *buffer, PIX_FMT_YUV420P, format, *width, *height ); + } + else +#endif + convert_image( this->av_frame, *buffer, codec_context->pix_fmt, format, *width, *height ); else mlt_frame_get_image( frame, buffer, format, width, height, writable ); } @@ -831,13 +1034,14 @@ static int producer_get_image( mlt_frame frame, uint8_t **buffer, mlt_image_form { int ret = 0; int int_position = 0; - got_picture = 0; + int decode_errors = 0; + int got_picture = 0; av_init_packet( &pkt ); // Construct an AVFrame for YUV422 conversion - if ( av_frame == NULL ) - av_frame = avcodec_alloc_frame( ); + if ( !this->av_frame ) + this->av_frame = avcodec_alloc_frame( ); while( ret >= 0 && !got_picture ) { @@ -845,38 +1049,109 @@ static int producer_get_image( mlt_frame frame, uint8_t **buffer, mlt_image_form ret = av_read_frame( context, &pkt ); // We only deal with video from the selected video_index - if ( ret >= 0 && pkt.stream_index == index && pkt.size > 0 ) + if ( ret >= 0 && pkt.stream_index == this->video_index && pkt.size > 0 ) { // Determine time code of the packet - if (pkt.dts != AV_NOPTS_VALUE) + if ( use_new_seek ) { - int_position = ( int )( av_q2d( stream->time_base ) * pkt.dts * source_fps + 0.5 ); - if ( context->start_time != AV_NOPTS_VALUE ) - int_position -= ( int )( context->start_time * source_fps / AV_TIME_BASE + 0.5 ); - last_position = mlt_properties_get_int( properties, "_last_position" ); - if ( int_position == last_position ) - int_position = last_position + 1; + int64_t pts = pkt.pts; + if ( this->first_pts > 0 ) + pts -= this->first_pts; + else if ( context->start_time != AV_NOPTS_VALUE ) + pts -= context->start_time; + int_position = ( int )( av_q2d( stream->time_base ) * pts * source_fps + 0.1 ); + if ( pkt.pts == AV_NOPTS_VALUE ) + { + this->invalid_pts_counter++; + if ( this->invalid_pts_counter > 20 ) + { + mlt_log_panic( MLT_PRODUCER_SERVICE(producer), "\ainvalid PTS; DISABLING NEW_SEEK!\n" ); + mlt_properties_set_int( properties, "new_seek", 0 ); + int_position = req_position; + use_new_seek = 0; + } + } + else + { + this->invalid_pts_counter = 0; + } + mlt_log_debug( MLT_PRODUCER_SERVICE(producer), "pkt.pts %llu req_pos %d cur_pos %d pkt_pos %d\n", + pkt.pts, req_position, this->current_position, int_position ); } else { - int_position = req_position; - } - mlt_log_debug( MLT_PRODUCER_SERVICE(this), "pkt.dts %llu req_pos %d cur_pos %d pkt_pos %d", - pkt.dts, req_position, current_position, int_position ); - // Make a dumb assumption on streams that contain wild timestamps - if ( (unsigned) req_position - (unsigned) int_position > 999 ) - { - int_position = req_position; - mlt_log_debug( MLT_PRODUCER_SERVICE(this), " WILD TIMESTAMP!" ); + if ( pkt.dts != AV_NOPTS_VALUE ) + { + int_position = ( int )( av_q2d( stream->time_base ) * pkt.dts * source_fps + 0.5 ); + if ( context->start_time != AV_NOPTS_VALUE ) + int_position -= ( int )( context->start_time * source_fps / AV_TIME_BASE + 0.5 ); + last_position = this->last_position; + if ( int_position == last_position ) + int_position = last_position + 1; + } + else + { + int_position = req_position; + } + mlt_log_debug( MLT_PRODUCER_SERVICE(producer), "pkt.dts %llu req_pos %d cur_pos %d pkt_pos %d\n", + pkt.dts, req_position, this->current_position, int_position ); + // Make a dumb assumption on streams that contain wild timestamps + if ( abs( req_position - int_position ) > 999 ) + { + int_position = req_position; + mlt_log_debug( MLT_PRODUCER_SERVICE(producer), " WILD TIMESTAMP!" ); + } } - mlt_properties_set_int( properties, "_last_position", int_position ); + this->last_position = int_position; // Decode the image if ( must_decode || int_position >= req_position ) - ret = avcodec_decode_video( codec_context, av_frame, &got_picture, pkt.data, pkt.size ); + { +#ifdef VDPAU + if ( g_vdpau && this->vdpau ) + { + if ( g_vdpau->producer != this ) + { + vdpau_decoder_close(); + vdpau_decoder_init( this ); + } + if ( this->vdpau ) + this->vdpau->is_decoded = 0; + } +#endif + codec_context->reordered_opaque = pkt.pts; + if ( int_position >= req_position ) + codec_context->skip_loop_filter = AVDISCARD_NONE; +#if (LIBAVCODEC_VERSION_INT >= ((52<<16)+(26<<8)+0)) + ret = avcodec_decode_video2( codec_context, this->av_frame, &got_picture, &pkt ); +#else + ret = avcodec_decode_video( codec_context, this->av_frame, &got_picture, pkt.data, pkt.size ); +#endif + // Note: decode may fail at the beginning of MPEGfile (B-frames referencing before first I-frame), so allow a few errors. + if ( ret < 0 ) + { + if ( ++decode_errors <= 10 ) + ret = 0; + } + else + { + decode_errors = 0; + } + } if ( got_picture ) { + if ( use_new_seek ) + { + // Determine time code of the packet + int64_t pts = this->av_frame->reordered_opaque; + if ( this->first_pts > 0 ) + pts -= this->first_pts; + else if ( context->start_time != AV_NOPTS_VALUE ) + pts -= context->start_time; + int_position = ( int )( av_q2d( stream->time_base) * pts * source_fps + 0.1 ); + mlt_log_debug( MLT_PRODUCER_SERVICE(producer), "got frame %d, key %d\n", int_position, this->av_frame->key_frame ); + } // Handle ignore if ( int_position < req_position ) { @@ -886,13 +1161,14 @@ static int producer_get_image( mlt_frame frame, uint8_t **buffer, mlt_image_form else if ( int_position >= req_position ) { ignore = 0; + codec_context->skip_loop_filter = AVDISCARD_NONE; } else if ( ignore -- ) { got_picture = 0; } } - mlt_log_debug( MLT_PRODUCER_SERVICE(this), " got_pic %d key %d\n", got_picture, pkt.flags & PKT_FLAG_KEY ); + mlt_log_debug( MLT_PRODUCER_SERVICE(producer), " got_pic %d key %d\n", got_picture, pkt.flags & PKT_FLAG_KEY ); av_free_packet( &pkt ); } else if ( ret >= 0 ) @@ -903,15 +1179,51 @@ static int producer_get_image( mlt_frame frame, uint8_t **buffer, mlt_image_form // Now handle the picture if we have one if ( got_picture ) { - if ( allocate_buffer( frame_properties, codec_context, buffer, format, width, height ) ) + if ( ( image_size = allocate_buffer( frame_properties, codec_context, buffer, format, width, height ) ) ) { - convert_image( av_frame, *buffer, codec_context->pix_fmt, format, *width, *height ); - if ( !mlt_properties_get( properties, "force_progressive" ) ) - mlt_properties_set_int( frame_properties, "progressive", !av_frame->interlaced_frame ); - mlt_properties_set_int( properties, "top_field_first", av_frame->top_field_first ); - mlt_properties_set_int( properties, "_current_position", int_position ); - mlt_properties_set_int( properties, "_got_picture", 1 ); - mlt_properties_set_data( properties, "av_frame", av_frame, 0, av_free, NULL ); +#ifdef VDPAU + if ( this->vdpau ) + { + if ( this->vdpau->is_decoded ) + { + struct vdpau_render_state *render = (struct vdpau_render_state*) this->av_frame->data[0]; + void *planes[3]; + uint32_t pitches[3]; + VdpYCbCrFormat dest_format = VDP_YCBCR_FORMAT_YV12; + AVPicture picture; + + if ( !this->vdpau->buffer ) + this->vdpau->buffer = mlt_pool_alloc( codec_context->width * codec_context->height * 3 / 2 ); + picture.data[0] = planes[0] = this->vdpau->buffer; + picture.data[2] = planes[1] = this->vdpau->buffer + codec_context->width * codec_context->height; + picture.data[1] = planes[2] = this->vdpau->buffer + codec_context->width * codec_context->height * 5 / 4; + picture.linesize[0] = pitches[0] = codec_context->width; + picture.linesize[1] = pitches[1] = codec_context->width / 2; + picture.linesize[2] = pitches[2] = codec_context->width / 2; + + VdpStatus status = vdp_surface_get_bits( render->surface, dest_format, planes, pitches ); + if ( status == VDP_STATUS_OK ) + { + convert_image( (AVFrame*) &picture, *buffer, PIX_FMT_YUV420P, format, *width, *height ); + } + else + { + mlt_log_error( MLT_PRODUCER_SERVICE(producer), "VDPAU Error: %s\n", vdp_get_error_string( status ) ); + this->vdpau->is_decoded = 0; + } + } + else + { + mlt_log_error( MLT_PRODUCER_SERVICE(producer), "VDPAU error in VdpDecoderRender\n" ); + got_picture = 0; + } + } + else +#endif + convert_image( this->av_frame, *buffer, codec_context->pix_fmt, format, *width, *height ); + this->top_field_first |= this->av_frame->top_field_first; + this->current_position = int_position; + this->got_picture = 1; } else { @@ -919,22 +1231,32 @@ static int producer_get_image( mlt_frame frame, uint8_t **buffer, mlt_image_form } } } - if ( !got_picture ) - mlt_frame_get_image( frame, buffer, format, width, height, writable ); } - // Very untidy - for rawvideo, the packet contains the frame, hence the free packet - // above will break the pause behaviour - so we wipe the frame now - if ( !strcmp( codec_context->codec->name, "rawvideo" ) ) - mlt_properties_set_data( properties, "av_frame", NULL, 0, NULL, NULL ); + avformat_unlock(); + + if ( this->got_picture && image_size > 0 && this->image_cache ) + { + // Copy buffer to image cache + uint8_t *image = mlt_pool_alloc( image_size ); + memcpy( image, *buffer, image_size ); + mlt_cache_put( this->image_cache, (void*) position, image, *format, mlt_pool_release ); + } + +exit_get_image: + // Set the progressive flag + if ( mlt_properties_get( properties, "force_progressive" ) ) + mlt_properties_set_int( frame_properties, "progressive", !!mlt_properties_get_int( properties, "force_progressive" ) ); + else if ( this->av_frame ) + mlt_properties_set_int( frame_properties, "progressive", !this->av_frame->interlaced_frame ); // Set the field order property for this frame - mlt_properties_set_int( frame_properties, "top_field_first", mlt_properties_get_int( properties, "top_field_first" ) ); + mlt_properties_set_int( frame_properties, "top_field_first", this->top_field_first ); // Regardless of speed, we expect to get the next frame (cos we ain't too bright) - mlt_properties_set_position( properties, "_video_expected", position + 1 ); + this->video_expected = position + 1; - return 0; + return !this->got_picture; } /** Process properties as AVOptions and apply to AV context obj @@ -948,27 +1270,126 @@ static void apply_properties( void *obj, mlt_properties properties, int flags ) { const char *opt_name = mlt_properties_get_name( properties, i ); const AVOption *opt = av_find_opt( obj, opt_name, NULL, flags, flags ); - if ( opt != NULL ) + if ( opt_name && mlt_properties_get( properties, opt_name ) ) + { + if ( opt ) #if LIBAVCODEC_VERSION_INT >= ((52<<16)+(7<<8)+0) - av_set_string3( obj, opt_name, mlt_properties_get( properties, opt_name), 0, NULL ); + av_set_string3( obj, opt_name, mlt_properties_get( properties, opt_name), 0, NULL ); #elif LIBAVCODEC_VERSION_INT >= ((51<<16)+(59<<8)+0) - av_set_string2( obj, opt_name, mlt_properties_get( properties, opt_name), 0 ); + av_set_string2( obj, opt_name, mlt_properties_get( properties, opt_name), 0 ); #else - av_set_string( obj, opt_name, mlt_properties_get( properties, opt_name) ); + av_set_string( obj, opt_name, mlt_properties_get( properties, opt_name) ); #endif + } } } +/** Initialize the video codec context. + */ + +static int video_codec_init( producer_avformat this, int index, mlt_properties properties ) +{ + // Initialise the codec if necessary + if ( !this->video_codec ) + { + // Get the video stream + AVStream *stream = this->video_format->streams[ index ]; + + // Get codec context + AVCodecContext *codec_context = stream->codec; + + // Find the codec + AVCodec *codec = avcodec_find_decoder( codec_context->codec_id ); +#ifdef VDPAU + if ( codec_context->codec_id == CODEC_ID_H264 ) + { + if ( ( codec = avcodec_find_decoder_by_name( "h264_vdpau" ) ) ) + { + if ( vdpau_init( this ) ) + { + this->video_codec = codec_context; + if ( !vdpau_decoder_init( this ) ) + vdpau_decoder_close(); + } + } + if ( !this->vdpau ) + codec = avcodec_find_decoder( codec_context->codec_id ); + } +#endif + + // Initialise multi-threading + int thread_count = mlt_properties_get_int( properties, "threads" ); + if ( thread_count == 0 && getenv( "MLT_AVFORMAT_THREADS" ) ) + thread_count = atoi( getenv( "MLT_AVFORMAT_THREADS" ) ); + if ( thread_count > 1 ) + { + avcodec_thread_init( codec_context, thread_count ); + codec_context->thread_count = thread_count; + } + + // If we don't have a codec and we can't initialise it, we can't do much more... + avformat_lock( ); + if ( codec && avcodec_open( codec_context, codec ) >= 0 ) + { + // Now store the codec with its destructor + this->video_codec = codec_context; + } + else + { + // Remember that we can't use this later + this->video_index = -1; + } + avformat_unlock( ); + + // Process properties as AVOptions + apply_properties( codec_context, properties, AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM ); + + // Reset some image properties + mlt_properties_set_int( properties, "width", this->video_codec->width ); + mlt_properties_set_int( properties, "height", this->video_codec->height ); + // For DV, we'll just use the saved aspect ratio + if ( codec_context->codec_id != CODEC_ID_DVVIDEO ) + mlt_properties_set_double( properties, "aspect_ratio", get_aspect_ratio( stream, this->video_codec, NULL ) ); + + // Determine the fps first from the codec + double source_fps = (double) this->video_codec->time_base.den / + ( this->video_codec->time_base.num == 0 ? 1 : this->video_codec->time_base.num ); + + if ( mlt_properties_get( properties, "force_fps" ) ) + { + source_fps = mlt_properties_get_double( properties, "force_fps" ); + stream->time_base = av_d2q( source_fps, 255 ); + } + else + { + // If the muxer reports a frame rate different than the codec + double muxer_fps = av_q2d( stream->r_frame_rate ); + // Choose the lesser - the wrong tends to be off by some multiple of 10 + source_fps = FFMIN( source_fps, muxer_fps ); + } + + // We'll use fps if it's available + if ( source_fps > 0 ) + mlt_properties_set_double( properties, "source_fps", source_fps ); + else + mlt_properties_set_double( properties, "source_fps", mlt_producer_get_fps( this->parent ) ); + } + return this->video_codec && this->video_index > -1; +} + /** Set up video handling. */ -static void producer_set_up_video( mlt_producer this, mlt_frame frame ) +static void producer_set_up_video( producer_avformat this, mlt_frame frame ) { + // Get the producer + mlt_producer producer = this->parent; + // Get the properties - mlt_properties properties = MLT_PRODUCER_PROPERTIES( this ); + mlt_properties properties = MLT_PRODUCER_PROPERTIES( producer ); - // Fetch the video_context - AVFormatContext *context = mlt_properties_get_data( properties, "video_context", NULL ); + // Fetch the video format context + AVFormatContext *context = this->video_format; // Get the video_index int index = mlt_properties_get_int( properties, "video_index" ); @@ -976,12 +1397,20 @@ static void producer_set_up_video( mlt_producer this, mlt_frame frame ) // Reopen the file if necessary if ( !context && index > -1 ) { - mlt_events_block( properties, this ); - producer_open( this, mlt_service_profile( MLT_PRODUCER_SERVICE(this) ), + mlt_events_block( properties, producer ); + producer_open( this, mlt_service_profile( MLT_PRODUCER_SERVICE(producer) ), mlt_properties_get( properties, "resource" ) ); - context = mlt_properties_get_data( properties, "video_context", NULL ); - mlt_properties_set_data( properties, "dummy_context", NULL, 0, NULL, NULL ); - mlt_events_unblock( properties, this ); + context = this->video_format; + if ( this->dummy_context ) + { + avformat_lock(); + av_close_input_file( this->dummy_context ); + avformat_unlock(); + } + this->dummy_context = NULL; + mlt_events_unblock( properties, producer ); + if ( this->audio_format ) + get_audio_streams_info( this ); // Process properties as AVOptions apply_properties( context, properties, AV_OPT_FLAG_DECODING_PARAM ); @@ -991,7 +1420,9 @@ static void producer_set_up_video( mlt_producer this, mlt_frame frame ) if ( context && index >= (int) context->nb_streams ) { // Get the last video stream - for ( index = context->nb_streams - 1; index >= 0 && context->streams[ index ]->codec->codec_type != CODEC_TYPE_VIDEO; --index ); + for ( index = context->nb_streams - 1; + index >= 0 && context->streams[ index ]->codec->codec_type != CODEC_TYPE_VIDEO; + index-- ); mlt_properties_set_int( properties, "video_index", index ); } if ( context && index > -1 && context->streams[ index ]->codec->codec_type != CODEC_TYPE_VIDEO ) @@ -1001,114 +1432,193 @@ static void producer_set_up_video( mlt_producer this, mlt_frame frame ) mlt_properties_set_int( properties, "video_index", index ); } + // Update the video properties if the index changed + if ( index != this->video_index ) + { + // Reset the video properties if the index changed + this->video_index = index; + if ( this->video_codec ) + { + avformat_lock(); + avcodec_close( this->video_codec ); + avformat_unlock(); + } + this->video_codec = NULL; + } + // Get the frame properties mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame ); - if ( context && index > -1 ) + // Get the codec + if ( context && index > -1 && video_codec_init( this, index, properties ) ) { - // Get the video stream - AVStream *stream = context->streams[ index ]; + // Set the frame properties + double force_aspect_ratio = mlt_properties_get_double( properties, "force_aspect_ratio" ); + double aspect_ratio = ( force_aspect_ratio > 0.0 ) ? + force_aspect_ratio : mlt_properties_get_double( properties, "aspect_ratio" ); + + // Set the width and height + mlt_properties_set_int( frame_properties, "width", this->video_codec->width ); + mlt_properties_set_int( frame_properties, "height", this->video_codec->height ); + mlt_properties_set_int( frame_properties, "real_width", this->video_codec->width ); + mlt_properties_set_int( frame_properties, "real_height", this->video_codec->height ); + mlt_properties_set_double( frame_properties, "aspect_ratio", aspect_ratio ); + + // Add our image operation + mlt_frame_push_service( frame, this ); + mlt_frame_push_get_image( frame, producer_get_image ); + } + else + { + // If something failed, use test card image + mlt_properties_set_int( frame_properties, "test_image", 1 ); + } +} - // Get codec context - AVCodecContext *codec_context = stream->codec; +static int seek_audio( producer_avformat this, mlt_position position, double timecode, int *ignore ) +{ + int paused = 0; + + // Seek if necessary + if ( position != this->audio_expected ) + { + if ( position + 1 == this->audio_expected ) + { + // We're paused - silence required + paused = 1; + } + else if ( !this->seekable && position > this->audio_expected && ( position - this->audio_expected ) < 250 ) + { + // Fast forward - seeking is inefficient for small distances - just ignore following frames + *ignore = position - this->audio_expected; + } + else if ( position < this->audio_expected || position - this->audio_expected >= 12 ) + { + AVFormatContext *context = this->audio_format; + int64_t timestamp = ( int64_t )( timecode * AV_TIME_BASE + 0.5 ); + if ( context->start_time != AV_NOPTS_VALUE ) + timestamp += context->start_time; + if ( timestamp < 0 ) + timestamp = 0; + + // Set to the real timecode + if ( av_seek_frame( context, -1, timestamp, AVSEEK_FLAG_BACKWARD ) != 0 ) + paused = 1; + + // Clear the usage in the audio buffer + int i = MAX_AUDIO_STREAMS + 1; + while ( --i ) + this->audio_used[i - 1] = 0; + } + } + return paused; +} + +static int decode_audio( producer_avformat this, int *ignore, AVPacket *pkt, int channels, int samples, double timecode, double fps ) +{ + // Fetch the audio_format + AVFormatContext *context = this->audio_format; + + // Get the current stream index + int index = pkt->stream_index; + + // Get codec context + AVCodecContext *codec_context = this->audio_codec[ index ]; - // Get the codec - AVCodec *codec = mlt_properties_get_data( properties, "video_codec", NULL ); + // Obtain the resample context if it exists (not always needed) + ReSampleContext *resample = this->audio_resample[ index ]; + + // Obtain the audio buffers + int16_t *audio_buffer = this->audio_buffer[ index ]; + int16_t *decode_buffer = this->decode_buffer[ index ]; + + int audio_used = this->audio_used[ index ]; + uint8_t *ptr = pkt->data; + int len = pkt->size; + int ret = 0; - // Update the video properties if the index changed - if ( index != mlt_properties_get_int( properties, "_video_index" ) ) + while ( ptr && ret >= 0 && len > 0 ) + { + int data_size = sizeof( int16_t ) * AVCODEC_MAX_AUDIO_FRAME_SIZE; + + // Decode the audio +#if (LIBAVCODEC_VERSION_INT >= ((52<<16)+(26<<8)+0)) + ret = avcodec_decode_audio3( codec_context, decode_buffer, &data_size, pkt ); +#elif (LIBAVCODEC_VERSION_INT >= ((51<<16)+(29<<8)+0)) + ret = avcodec_decode_audio2( codec_context, decode_buffer, &data_size, ptr, len ); +#else + ret = avcodec_decode_audio( codec_context, decode_buffer, &data_size, ptr, len ); +#endif + if ( ret < 0 ) { - // Reset the video properties if the index changed - mlt_properties_set_int( properties, "_video_index", index ); - mlt_properties_set_data( properties, "video_codec", NULL, 0, NULL, NULL ); - mlt_properties_set_int( properties, "width", codec_context->width ); - mlt_properties_set_int( properties, "height", codec_context->height ); - // TODO: get the first usable AVPacket and reset the stream position - mlt_properties_set_double( properties, "aspect_ratio", - get_aspect_ratio( context->streams[ index ], codec_context, NULL ) ); - codec = NULL; + ret = 0; + break; } - // Initialise the codec if necessary - if ( codec == NULL ) + len -= ret; + ptr += ret; + + // If decoded successfully + if ( data_size > 0 ) { - // Initialise multi-threading - int thread_count = mlt_properties_get_int( properties, "threads" ); - if ( thread_count == 0 && getenv( "MLT_AVFORMAT_THREADS" ) ) - thread_count = atoi( getenv( "MLT_AVFORMAT_THREADS" ) ); - if ( thread_count > 1 ) + // Figure out how many samples will be needed after resampling + int convert_samples = data_size / codec_context->channels / ( av_get_bits_per_sample_format( codec_context->sample_fmt ) / 8 ); + int samples_needed = this->resample_factor * convert_samples + 1; + + // Resize audio buffer to prevent overflow + if ( audio_used * channels + samples_needed > this->audio_buffer_size[ index ] ) { - avcodec_thread_init( codec_context, thread_count ); - codec_context->thread_count = thread_count; + this->audio_buffer_size[ index ] *= 2; + audio_buffer = this->audio_buffer[ index ] = mlt_pool_realloc( audio_buffer, this->audio_buffer_size[ index ] * sizeof(int16_t) ); } - - // Find the codec - codec = avcodec_find_decoder( codec_context->codec_id ); - - // If we don't have a codec and we can't initialise it, we can't do much more... - avformat_lock( ); - if ( codec != NULL && avcodec_open( codec_context, codec ) >= 0 ) + if ( resample ) { - // Now store the codec with its destructor - mlt_properties_set_data( properties, "video_codec", codec_context, 0, producer_codec_close, NULL ); + // Copy to audio buffer while resampling + int16_t *source = decode_buffer; + int16_t *dest = &audio_buffer[ audio_used * channels ]; + audio_used += audio_resample( resample, dest, source, convert_samples ); } else { - // Remember that we can't use this later - mlt_properties_set_int( properties, "video_index", -1 ); - index = -1; + // Straight copy to audio buffer + memcpy( &audio_buffer[ audio_used * codec_context->channels ], decode_buffer, data_size ); + audio_used += convert_samples; } - avformat_unlock( ); - // Process properties as AVOptions - apply_properties( codec_context, properties, AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM ); + // Handle ignore + while ( *ignore && audio_used > samples ) + { + *ignore -= 1; + audio_used -= samples; + memmove( audio_buffer, &audio_buffer[ samples * (resample? channels : codec_context->channels) ], + audio_used * sizeof( int16_t ) ); + } } + } - // No codec, no show... - if ( codec && index > -1 ) - { - double source_fps = 0; - double force_aspect_ratio = mlt_properties_get_double( properties, "force_aspect_ratio" ); - double aspect_ratio = ( force_aspect_ratio > 0.0 ) ? - force_aspect_ratio : mlt_properties_get_double( properties, "aspect_ratio" ); - - // Determine the fps - source_fps = ( double )codec_context->time_base.den / ( codec_context->time_base.num == 0 ? 1 : codec_context->time_base.num ); + // If we're behind, ignore this packet + if ( pkt->pts >= 0 ) + { + double current_pts = av_q2d( context->streams[ index ]->time_base ) * pkt->pts; + int req_position = ( int )( timecode * fps + 0.5 ); + int int_position = ( int )( current_pts * fps + 0.5 ); + if ( context->start_time != AV_NOPTS_VALUE ) + int_position -= ( int )( fps * context->start_time / AV_TIME_BASE + 0.5 ); - // If the muxer reports a frame rate different than the codec - double muxer_fps = av_q2d( context->streams[ index ]->r_frame_rate ); - if ( source_fps != muxer_fps ) - // Choose the lesser - the wrong tends to be off by some multiple of 10 - source_fps = muxer_fps < source_fps ? muxer_fps : source_fps; - - // We'll use fps if it's available - if ( source_fps > 0 ) - mlt_properties_set_double( properties, "source_fps", source_fps ); - else - mlt_properties_set_double( properties, "source_fps", mlt_producer_get_fps( this ) ); - mlt_properties_set_double( properties, "aspect_ratio", aspect_ratio ); - - // Set the width and height - mlt_properties_set_int( frame_properties, "width", codec_context->width ); - mlt_properties_set_int( frame_properties, "height", codec_context->height ); - mlt_properties_set_int( frame_properties, "real_width", codec_context->width ); - mlt_properties_set_int( frame_properties, "real_height", codec_context->height ); - mlt_properties_set_double( frame_properties, "aspect_ratio", aspect_ratio ); - if ( mlt_properties_get( properties, "force_progressive" ) ) - mlt_properties_set_int( frame_properties, "progressive", mlt_properties_get_int( properties, "force_progressive" ) ); - - mlt_frame_push_get_image( frame, producer_get_image ); - mlt_properties_set_data( frame_properties, "avformat_producer", this, 0, NULL, NULL ); - } - else + if ( this->seekable && *ignore == 0 ) { - mlt_properties_set_int( frame_properties, "test_image", 1 ); + if ( int_position < req_position ) + // We are behind, so skip some + *ignore = 1; + else if ( int_position > req_position + 2 ) + // We are ahead, so seek backwards some more + seek_audio( this, req_position, timecode - 1.0, ignore ); } } - else - { - mlt_properties_set_int( frame_properties, "test_image", 1 ); - } + + this->audio_used[ index ] = audio_used; + + return ret; } /** Get the audio from a frame. @@ -1116,128 +1626,78 @@ static void producer_set_up_video( mlt_producer this, mlt_frame frame ) static int producer_get_audio( mlt_frame frame, void **buffer, mlt_audio_format *format, int *frequency, int *channels, int *samples ) { - // Get the properties from the frame - mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame ); - - // Obtain the frame number of this frame - mlt_position position = mlt_properties_get_position( frame_properties, "avformat_position" ); - // Get the producer - mlt_producer this = mlt_properties_get_data( frame_properties, "avformat_producer", NULL ); - - // Get the producer properties - mlt_properties properties = MLT_PRODUCER_PROPERTIES( this ); - - // Fetch the audio_context - AVFormatContext *context = mlt_properties_get_data( properties, "audio_context", NULL ); - - // Get the audio_index - int index = mlt_properties_get_int( properties, "audio_index" ); - - // Get the seekable status - int seekable = mlt_properties_get_int( properties, "seekable" ); - - // Obtain the expected frame numer - mlt_position expected = mlt_properties_get_position( properties, "_audio_expected" ); + producer_avformat this = mlt_frame_pop_audio( frame ); - // Obtain the resample context if it exists (not always needed) - ReSampleContext *resample = mlt_properties_get_data( properties, "audio_resample", NULL ); - - // Obtain the audio buffers - int16_t *audio_buffer = mlt_properties_get_data( properties, "audio_buffer", NULL ); - int16_t *decode_buffer = mlt_properties_get_data( properties, "decode_buffer", NULL ); - - // Get amount of audio used - int audio_used = mlt_properties_get_int( properties, "_audio_used" ); + // Obtain the frame number of this frame + mlt_position position = mlt_properties_get_position( MLT_FRAME_PROPERTIES( frame ), "avformat_position" ); // Calculate the real time code - double real_timecode = producer_time_of_frame( this, position ); - - // Get the audio stream - AVStream *stream = context->streams[ index ]; + double real_timecode = producer_time_of_frame( this->parent, position ); - // Get codec context - AVCodecContext *codec_context = stream->codec; - - // Packet - AVPacket pkt; + // Get the producer fps + double fps = mlt_producer_get_fps( this->parent ); // Number of frames to ignore (for ffwd) int ignore = 0; // Flag for paused (silence) - int paused = 0; - - // Check for resample and create if necessary - if ( resample == NULL && codec_context->channels <= 2 ) - { - // Create the resampler -#if (LIBAVCODEC_VERSION_INT >= ((52<<16)+(15<<8)+0)) - resample = av_audio_resample_init( *channels, codec_context->channels, *frequency, codec_context->sample_rate, - SAMPLE_FMT_S16, codec_context->sample_fmt, 16, 10, 0, 0.8 ); -#else - resample = audio_resample_init( *channels, codec_context->channels, *frequency, codec_context->sample_rate ); -#endif - - // And store it on properties - mlt_properties_set_data( properties, "audio_resample", resample, 0, ( mlt_destructor )audio_resample_close, NULL ); - } - else if ( resample == NULL ) + int paused = seek_audio( this, position, real_timecode, &ignore ); + + // Fetch the audio_format + AVFormatContext *context = this->audio_format; + + // Determine the tracks to use + int index = this->audio_index; + int index_max = this->audio_index + 1; + if ( this->audio_index == INT_MAX ) { - // TODO: uncomment and remove following line when full multi-channel support is ready - // *channels = codec_context->channels; - codec_context->request_channels = *channels; - - *frequency = codec_context->sample_rate; + index = 0; + index_max = context->nb_streams; + *channels = this->total_channels; + *frequency = this->max_frequency; } - // Check for audio buffer and create if necessary - if ( audio_buffer == NULL ) + // Initialize the resamplers and buffers + for ( ; index < index_max; index++ ) { - // Allocate the audio buffer - audio_buffer = mlt_pool_alloc( AVCODEC_MAX_AUDIO_FRAME_SIZE * sizeof( int16_t ) ); - - // And store it on properties for reuse - mlt_properties_set_data( properties, "audio_buffer", audio_buffer, 0, ( mlt_destructor )mlt_pool_release, NULL ); - } - - // Check for decoder buffer and create if necessary - if ( decode_buffer == NULL ) - { - // Allocate the audio buffer - decode_buffer = av_malloc( AVCODEC_MAX_AUDIO_FRAME_SIZE * sizeof( int16_t ) ); - - // And store it on properties for reuse - mlt_properties_set_data( properties, "decode_buffer", decode_buffer, 0, ( mlt_destructor )av_free, NULL ); - } + // Get codec context + AVCodecContext *codec_context = this->audio_codec[ index ]; - // Seek if necessary - if ( position != expected ) - { - if ( position + 1 == expected ) + if ( codec_context && !this->audio_buffer[ index ] ) { - // We're paused - silence required - paused = 1; - } - else if ( !seekable && position > expected && ( position - expected ) < 250 ) - { - // Fast forward - seeking is inefficient for small distances - just ignore following frames - ignore = position - expected; - } - else if ( position < expected || position - expected >= 12 ) - { - int64_t timestamp = ( int64_t )( real_timecode * AV_TIME_BASE + 0.5 ); - if ( context->start_time != AV_NOPTS_VALUE ) - timestamp += context->start_time; - if ( timestamp < 0 ) - timestamp = 0; + // Check for resample and create if necessary + if ( codec_context->channels <= 2 ) + { + // Determine by how much resampling will increase number of samples + double resample_factor = this->audio_index == INT_MAX ? 1 : (double) *channels / codec_context->channels; + resample_factor *= (double) *frequency / codec_context->sample_rate; + if ( resample_factor > this->resample_factor ) + this->resample_factor = resample_factor; + + // Create the resampler +#if (LIBAVCODEC_VERSION_INT >= ((52<<16)+(15<<8)+0)) + this->audio_resample[ index ] = av_audio_resample_init( + this->audio_index == INT_MAX ? codec_context->channels : *channels, + codec_context->channels, *frequency, codec_context->sample_rate, + SAMPLE_FMT_S16, codec_context->sample_fmt, 16, 10, 0, 0.8 ); +#else + this->audio_resample[ index ] = audio_resample_init( + this->audio_index == INT_MAX ? codec_context->channels : *channels, + codec_context->channels, *frequency, codec_context->sample_rate ); +#endif + } + else + { + codec_context->request_channels = this->audio_index == INT_MAX ? codec_context->channels : *channels; + } - // Set to the real timecode - if ( av_seek_frame( context, -1, timestamp, AVSEEK_FLAG_BACKWARD ) != 0 ) - paused = 1; + // Check for audio buffer and create if necessary + this->audio_buffer_size[ index ] = AVCODEC_MAX_AUDIO_FRAME_SIZE; + this->audio_buffer[ index ] = mlt_pool_alloc( this->audio_buffer_size[ index ] * sizeof( int16_t ) ); - // Clear the usage in the audio buffer - audio_used = 0; + // Check for decoder buffer and create if necessary + this->decode_buffer[ index ] = av_malloc( AVCODEC_MAX_AUDIO_FRAME_SIZE * sizeof( int16_t ) ); } } @@ -1246,13 +1706,20 @@ static int producer_get_audio( mlt_frame frame, void **buffer, mlt_audio_format { int ret = 0; int got_audio = 0; + AVPacket pkt; av_init_packet( &pkt ); - - while( ret >= 0 && !got_audio ) + + // If not resampling, give consumer more than requested. + // It requested number samples based on requested frame rate. + // Do not clean this up with a samples *= ...! + if ( this->audio_index != INT_MAX && ! this->audio_resample[ this->audio_index ] ) + *samples = *samples * this->audio_codec[ this->audio_index ]->sample_rate / *frequency; + + while ( ret >= 0 && !got_audio ) { // Check if the buffer already contains the samples required - if ( audio_used >= *samples && ignore == 0 ) + if ( this->audio_index != INT_MAX && this->audio_used[ this->audio_index ] >= *samples && ignore == 0 ) { got_audio = 1; break; @@ -1261,227 +1728,325 @@ static int producer_get_audio( mlt_frame frame, void **buffer, mlt_audio_format // Read a packet ret = av_read_frame( context, &pkt ); - int len = pkt.size; - uint8_t *ptr = pkt.data; - - // We only deal with audio from the selected audio_index - while ( ptr != NULL && ret >= 0 && pkt.stream_index == index && len > 0 ) + // We only deal with audio from the selected audio index + if ( ret >= 0 && pkt.data && pkt.size > 0 && ( pkt.stream_index == this->audio_index || + ( this->audio_index == INT_MAX && context->streams[ pkt.stream_index ]->codec->codec_type == CODEC_TYPE_AUDIO ) ) ) { - int data_size = sizeof( int16_t ) * AVCODEC_MAX_AUDIO_FRAME_SIZE; - - // Decode the audio -#if (LIBAVCODEC_VERSION_INT >= ((51<<16)+(29<<8)+0)) - ret = avcodec_decode_audio2( codec_context, decode_buffer, &data_size, ptr, len ); -#else - ret = avcodec_decode_audio( codec_context, decode_buffer, &data_size, ptr, len ); -#endif - if ( ret < 0 ) - { - ret = 0; - break; - } - - len -= ret; - ptr += ret; - - if ( data_size > 0 && ( audio_used * *channels + data_size < AVCODEC_MAX_AUDIO_FRAME_SIZE ) ) - { - if ( resample ) - { - int16_t *source = decode_buffer; - int16_t *dest = &audio_buffer[ audio_used * *channels ]; - int convert_samples = data_size / av_get_bits_per_sample_format( codec_context->sample_fmt ) * 8 / codec_context->channels; - - audio_used += audio_resample( resample, dest, source, convert_samples ); - } - else - { - memcpy( &audio_buffer[ audio_used * *channels ], decode_buffer, data_size ); - audio_used += data_size / *channels / av_get_bits_per_sample_format( codec_context->sample_fmt ) * 8; - } - - // Handle ignore - while ( ignore && audio_used > *samples ) - { - ignore --; - audio_used -= *samples; - memmove( audio_buffer, &audio_buffer[ *samples * *channels ], audio_used * sizeof( int16_t ) ); - } - } + int channels2 = this->audio_index == INT_MAX ? this->audio_codec[pkt.stream_index]->channels : *channels; + ret = decode_audio( this, &ignore, &pkt, channels2, *samples, real_timecode, fps ); + } + av_free_packet( &pkt ); - // If we're behind, ignore this packet - if ( pkt.pts >= 0 ) + if ( this->audio_index == INT_MAX && ret >= 0 ) + { + // Determine if there is enough audio for all streams + got_audio = 1; + for ( index = 0; index < context->nb_streams; index++ ) { - double current_pts = av_q2d( stream->time_base ) * pkt.pts; - double source_fps = mlt_properties_get_double( properties, "source_fps" ); - int req_position = ( int )( real_timecode * source_fps + 0.5 ); - int int_position = ( int )( current_pts * source_fps + 0.5 ); - - if ( context->start_time != AV_NOPTS_VALUE ) - int_position -= ( int )( context->start_time * source_fps / AV_TIME_BASE + 0.5 ); - if ( seekable && !ignore && int_position < req_position ) - ignore = 1; + if ( this->audio_codec[ index ] && this->audio_used[ index ] < *samples ) + got_audio = 0; } } - - // We're finished with this packet regardless - av_free_packet( &pkt ); } - - int size = *samples * *channels * sizeof( int16_t ); - *format = mlt_audio_s16; + + // Allocate and set the frame's audio buffer + int size = *samples * *channels * sizeof(int16_t); *buffer = mlt_pool_alloc( size ); + *format = mlt_audio_s16; mlt_frame_set_audio( frame, *buffer, *format, size, mlt_pool_release ); - // Now handle the audio if we have enough - if ( audio_used >= *samples ) + // Interleave tracks if audio_index=all + if ( this->audio_index == INT_MAX ) { - memcpy( *buffer, audio_buffer, *samples * *channels * sizeof( int16_t ) ); - audio_used -= *samples; - memmove( audio_buffer, &audio_buffer[ *samples * *channels ], audio_used * *channels * sizeof( int16_t ) ); + int16_t *dest = *buffer; + int i; + for ( i = 0; i < *samples; i++ ) + { + for ( index = 0; index < index_max; index++ ) + if ( this->audio_codec[ index ] ) + { + int current_channels = this->audio_codec[ index ]->channels; + int16_t *src = this->audio_buffer[ index ] + i * current_channels; + memcpy( dest, src, current_channels * sizeof(int16_t) ); + dest += current_channels; + } + } + for ( index = 0; index < index_max; index++ ) + if ( this->audio_codec[ index ] && this->audio_used[ index ] >= *samples ) + { + int current_channels = this->audio_codec[ index ]->channels; + int16_t *src = this->audio_buffer[ index ] + *samples * current_channels; + this->audio_used[index] -= *samples; + memmove( this->audio_buffer[ index ], src, this->audio_used[ index ] * current_channels * sizeof(int16_t) ); + } } + // Copy a single track to the output buffer else { - memset( *buffer, 0, *samples * *channels * sizeof( int16_t ) ); - } + index = this->audio_index; - // Store the number of audio samples still available - mlt_properties_set_int( properties, "_audio_used", audio_used ); + // Now handle the audio if we have enough + if ( this->audio_used[ index ] >= *samples ) + { + int16_t *src = this->audio_buffer[ index ]; + memcpy( *buffer, src, *samples * *channels * sizeof(int16_t) ); + this->audio_used[ index ] -= *samples; + memmove( src, &src[ *samples * *channels ], this->audio_used[ index ] * *channels * sizeof(int16_t) ); + } + else + { + // Otherwise fill with silence + memset( *buffer, 0, *samples * *channels * sizeof(int16_t) ); + } + if ( !this->audio_resample[ index ] ) + { + // TODO: uncomment and remove following line when full multi-channel support is ready + // *channels = codec_context->channels; + *frequency = this->audio_codec[ index ]->sample_rate; + } + } } else { // Get silence and don't touch the context mlt_frame_get_audio( frame, buffer, format, frequency, channels, samples ); } - + // Regardless of speed (other than paused), we expect to get the next frame if ( !paused ) - mlt_properties_set_position( properties, "_audio_expected", position + 1 ); + this->audio_expected = position + 1; return 0; } +/** Initialize the audio codec context. +*/ + +static int audio_codec_init( producer_avformat this, int index, mlt_properties properties ) +{ + // Initialise the codec if necessary + if ( !this->audio_codec[ index ] ) + { + // Get codec context + AVCodecContext *codec_context = this->audio_format->streams[index]->codec; + + // Find the codec + AVCodec *codec = avcodec_find_decoder( codec_context->codec_id ); + + // If we don't have a codec and we can't initialise it, we can't do much more... + avformat_lock( ); + if ( codec && avcodec_open( codec_context, codec ) >= 0 ) + { + // Now store the codec with its destructor + if ( this->audio_codec[ index ] ) + avcodec_close( this->audio_codec[ index ] ); + this->audio_codec[ index ] = codec_context; + } + else + { + // Remember that we can't use this later + this->audio_index = -1; + } + avformat_unlock( ); + + // Process properties as AVOptions + apply_properties( codec_context, properties, AV_OPT_FLAG_AUDIO_PARAM | AV_OPT_FLAG_DECODING_PARAM ); + } + return this->audio_codec[ index ] && this->audio_index > -1; +} + /** Set up audio handling. */ -static void producer_set_up_audio( mlt_producer this, mlt_frame frame ) +static void producer_set_up_audio( producer_avformat this, mlt_frame frame ) { + // Get the producer + mlt_producer producer = this->parent; + // Get the properties - mlt_properties properties = MLT_PRODUCER_PROPERTIES( this ); + mlt_properties properties = MLT_PRODUCER_PROPERTIES( producer ); - // Fetch the audio_context - AVFormatContext *context = mlt_properties_get_data( properties, "audio_context", NULL ); + // Fetch the audio format context + AVFormatContext *context = this->audio_format; + + mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame ); // Get the audio_index int index = mlt_properties_get_int( properties, "audio_index" ); + // Handle all audio tracks + if ( this->audio_index > -1 && + mlt_properties_get( properties, "audio_index" ) && + !strcmp( mlt_properties_get( properties, "audio_index" ), "all" ) ) + index = INT_MAX; + // Reopen the file if necessary - if ( !context && index > -1 ) + if ( !context && this->audio_index > -1 && index > -1 ) { - mlt_events_block( properties, this ); - producer_open( this, mlt_service_profile( MLT_PRODUCER_SERVICE(this) ), + mlt_events_block( properties, producer ); + producer_open( this, mlt_service_profile( MLT_PRODUCER_SERVICE(producer) ), mlt_properties_get( properties, "resource" ) ); - context = mlt_properties_get_data( properties, "audio_context", NULL ); - mlt_properties_set_data( properties, "dummy_context", NULL, 0, NULL, NULL ); - mlt_events_unblock( properties, this ); + context = this->audio_format; + if ( this->dummy_context ) + { + avformat_lock(); + av_close_input_file( this->dummy_context ); + avformat_unlock(); + } + this->dummy_context = NULL; + mlt_events_unblock( properties, producer ); + get_audio_streams_info( this ); } // Exception handling for audio_index - if ( context && index >= (int) context->nb_streams ) + if ( context && index >= (int) context->nb_streams && index < INT_MAX ) { - for ( index = context->nb_streams - 1; index >= 0 && context->streams[ index ]->codec->codec_type != CODEC_TYPE_AUDIO; --index ); + for ( index = context->nb_streams - 1; + index >= 0 && context->streams[ index ]->codec->codec_type != CODEC_TYPE_AUDIO; + index-- ); mlt_properties_set_int( properties, "audio_index", index ); } - if ( context && index > -1 && context->streams[ index ]->codec->codec_type != CODEC_TYPE_AUDIO ) + if ( context && index > -1 && index < INT_MAX && + context->streams[ index ]->codec->codec_type != CODEC_TYPE_AUDIO ) { - index = -1; + index = this->audio_index; mlt_properties_set_int( properties, "audio_index", index ); } // Update the audio properties if the index changed - if ( index > -1 && index != mlt_properties_get_int( properties, "_audio_index" ) ) + if ( context && index > -1 && index != this->audio_index ) { - mlt_properties_set_int( properties, "_audio_index", index ); - mlt_properties_set_data( properties, "audio_codec", NULL, 0, NULL, NULL ); + if ( this->audio_codec[ this->audio_index ] ) + { + avformat_lock(); + avcodec_close( this->audio_codec[ this->audio_index ] ); + avformat_unlock(); + } + this->audio_codec[ this->audio_index ] = NULL; } + if ( this->audio_index != -1 ) + this->audio_index = index; + else + index = -1; - // Deal with audio context - if ( context != NULL && index > -1 ) + // Get the codec(s) + if ( context && index == INT_MAX ) { - // Get the frame properties - mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame ); - - // Get the audio stream - AVStream *stream = context->streams[ index ]; - - // Get codec context - AVCodecContext *codec_context = stream->codec; - - // Get the codec - AVCodec *codec = mlt_properties_get_data( properties, "audio_codec", NULL ); - - // Initialise the codec if necessary - if ( codec == NULL ) + mlt_properties_set_int( frame_properties, "frequency", this->max_frequency ); + mlt_properties_set_int( frame_properties, "channels", this->total_channels ); + for ( index = 0; index < context->nb_streams; index++ ) { - // Find the codec - codec = avcodec_find_decoder( codec_context->codec_id ); - - // If we don't have a codec and we can't initialise it, we can't do much more... - avformat_lock( ); - if ( codec != NULL && avcodec_open( codec_context, codec ) >= 0 ) - { - // Now store the codec with its destructor - mlt_properties_set_data( properties, "audio_codec", codec_context, 0, producer_codec_close, NULL ); - - } - else - { - // Remember that we can't use this later - mlt_properties_set_int( properties, "audio_index", -1 ); - index = -1; - } - avformat_unlock( ); - - // Process properties as AVOptions - apply_properties( codec_context, properties, AV_OPT_FLAG_AUDIO_PARAM | AV_OPT_FLAG_DECODING_PARAM ); + if ( context->streams[ index ]->codec->codec_type == CODEC_TYPE_AUDIO ) + audio_codec_init( this, index, properties ); } - - // No codec, no show... - if ( codec && index > -1 ) + } + else if ( context && index > -1 && audio_codec_init( this, index, properties ) ) + { + // Set the frame properties + if ( index < INT_MAX ) { - mlt_frame_push_audio( frame, producer_get_audio ); - mlt_properties_set_data( frame_properties, "avformat_producer", this, 0, NULL, NULL ); - mlt_properties_set_int( frame_properties, "frequency", codec_context->sample_rate ); - mlt_properties_set_int( frame_properties, "channels", codec_context->channels ); + mlt_properties_set_int( frame_properties, "frequency", this->audio_codec[ index ]->sample_rate ); + mlt_properties_set_int( frame_properties, "channels", this->audio_codec[ index ]->channels ); } } + if ( context && index > -1 ) + { + // Add our audio operation + mlt_frame_push_audio( frame, this ); + mlt_frame_push_audio( frame, producer_get_audio ); + } } /** Our get frame implementation. */ -static int producer_get_frame( mlt_producer this, mlt_frame_ptr frame, int index ) +static int producer_get_frame( mlt_producer producer, mlt_frame_ptr frame, int index ) { + // Access the private data + mlt_service service = MLT_PRODUCER_SERVICE( producer ); + mlt_cache_item cache_item = mlt_service_cache_get( service, "producer_avformat" ); + producer_avformat this = mlt_cache_item_data( cache_item, NULL ); + + // If cache miss + if ( !this ) + { + this = calloc( 1, sizeof( struct producer_avformat_s ) ); + producer->child = this; + this->parent = producer; + mlt_service_cache_put( service, "producer_avformat", this, 0, (mlt_destructor) producer_avformat_close ); + cache_item = mlt_service_cache_get( service, "producer_avformat" ); + } + // Create an empty frame - *frame = mlt_frame_init( MLT_PRODUCER_SERVICE( this ) ); + *frame = mlt_frame_init( service); + + if ( *frame ) + { + mlt_properties_set_data( MLT_FRAME_PROPERTIES(*frame), "avformat_cache", cache_item, 0, (mlt_destructor) mlt_cache_item_close, NULL ); + } + else + { + mlt_cache_item_close( cache_item ); + return 1; + } // Update timecode on the frame we're creating - mlt_frame_set_position( *frame, mlt_producer_position( this ) ); + mlt_frame_set_position( *frame, mlt_producer_position( producer ) ); // Set the position of this producer - mlt_properties_set_position( MLT_FRAME_PROPERTIES( *frame ), "avformat_position", mlt_producer_frame( this ) ); - + mlt_properties_set_position( MLT_FRAME_PROPERTIES( *frame ), "avformat_position", mlt_producer_frame( producer ) ); + // Set up the video producer_set_up_video( this, *frame ); // Set up the audio producer_set_up_audio( this, *frame ); - // Set the aspect_ratio - mlt_properties_set_double( MLT_FRAME_PROPERTIES( *frame ), "aspect_ratio", mlt_properties_get_double( MLT_PRODUCER_PROPERTIES( this ), "aspect_ratio" ) ); - // Calculate the next timecode - mlt_producer_prepare_next( this ); + mlt_producer_prepare_next( producer ); return 0; } + +static void producer_avformat_close( producer_avformat this ) +{ + mlt_log_debug( NULL, "producer_avformat_close\n" ); + // Close the file + av_free( this->av_frame ); + avformat_lock(); + int i; + for ( i = 0; i < MAX_AUDIO_STREAMS; i++ ) + { + if ( this->audio_resample[i] ) + audio_resample_close( this->audio_resample[i] ); + mlt_pool_release( this->audio_buffer[i] ); + av_free( this->decode_buffer[i] ); + if ( this->audio_codec[i] ) + avcodec_close( this->audio_codec[i] ); + } + if ( this->video_codec ) + avcodec_close( this->video_codec ); + if ( this->dummy_context ) + av_close_input_file( this->dummy_context ); + if ( this->audio_format ) + av_close_input_file( this->audio_format ); + if ( this->video_format ) + av_close_input_file( this->video_format ); + avformat_unlock(); +#ifdef VDPAU + vdpau_producer_close( this ); +#endif + if ( this->image_cache ) + mlt_cache_close( this->image_cache ); + free( this ); +} + +static void producer_close( mlt_producer parent ) +{ + // Close the parent + parent->close = NULL; + mlt_producer_close( parent ); + + // Free the memory + free( parent ); +}