2 * producer_avformat.c -- avformat producer
3 * Copyright (C) 2003-2004 Ushodaya Enterprises Limited
4 * Author: Charles Yates <charles.yates@pandora.be>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software Foundation,
18 * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
22 #include "producer_avformat.h"
25 #include <framework/mlt_frame.h>
27 // ffmpeg Header files
28 #include <ffmpeg/avformat.h>
30 // System header files
35 // Forward references.
36 static int producer_open( mlt_producer this, char *file );
37 static int producer_get_frame( mlt_producer this, mlt_frame_ptr frame, int index );
39 // A static flag used to determine if avformat has been initialised
40 static int avformat_initialised = 0;
41 static pthread_mutex_t avformat_mutex;
43 /** Constructor for libavformat.
46 mlt_producer producer_avformat_init( char *file )
48 mlt_producer this = NULL;
50 // Check that we have a non-NULL argument
53 // Construct the producer
54 this = calloc( 1, sizeof( struct mlt_producer_s ) );
57 if ( mlt_producer_init( this, NULL ) == 0 )
60 mlt_properties properties = mlt_producer_properties( this );
62 // Set the resource property (required for all producers)
63 mlt_properties_set( properties, "resource", file );
65 // TEST: audio sync tweaking
66 mlt_properties_set_double( properties, "discrepancy", 1 );
68 // Register our get_frame implementation
69 this->get_frame = producer_get_frame;
71 // Initialise avformat if necessary
72 if ( avformat_initialised == 0 )
74 pthread_mutex_init( &avformat_mutex, NULL );
75 avformat_initialised = 1;
80 if ( producer_open( this, file ) != 0 )
83 mlt_producer_close( this );
92 /** Find the default streams.
95 static void find_default_streams( AVFormatContext *context, int *audio_index, int *video_index )
99 // Allow for multiple audio and video streams in the file and select first of each (if available)
100 for( i = 0; i < context->nb_streams; i++ )
102 // Get the codec context
103 AVCodecContext *codec_context = &context->streams[ i ]->codec;
105 // Determine the type and obtain the first index of each type
106 switch( codec_context->codec_type )
108 case CODEC_TYPE_VIDEO:
109 if ( *video_index < 0 )
112 case CODEC_TYPE_AUDIO:
113 if ( *audio_index < 0 )
122 /** Producer file destructor.
125 static void producer_file_close( void *context )
127 if ( context != NULL )
129 // Lock the mutex now
130 pthread_mutex_lock( &avformat_mutex );
133 av_close_input_file( context );
135 // Unlock the mutex now
136 pthread_mutex_unlock( &avformat_mutex );
140 /** Producer file destructor.
143 static void producer_codec_close( void *codec )
147 // Lock the mutex now
148 pthread_mutex_lock( &avformat_mutex );
151 avcodec_close( codec );
153 // Unlock the mutex now
154 pthread_mutex_unlock( &avformat_mutex );
160 NOTE: We need to have a valid [PAL or NTSC] frame rate before we can determine the
161 number of frames in the file. However, this is at odds with the way things work - the
162 constructor needs to provide in/out points before the user of the producer is able
163 to specify properties :-/. However, the PAL/NTSC distinction applies to all producers
164 and while we currently accept whatever the producer provides, this will not work in
165 the more general case. Plans are afoot... and this one will work without modification
166 (in theory anyway ;-)).
169 static int producer_open( mlt_producer this, char *file )
171 // Return an error code (0 == no error)
174 // Context for avformat
175 AVFormatContext *context = NULL;
177 // Get the properties
178 mlt_properties properties = mlt_producer_properties( this );
180 // We will treat everything with the producer fps
181 double fps = mlt_properties_get_double( properties, "fps" );
183 // Lock the mutex now
184 pthread_mutex_lock( &avformat_mutex );
186 // Now attempt to open the file
187 error = av_open_input_file( &context, file, NULL, 0, NULL );
188 // fprintf( stderr, "AVFORMAT: open %d %s\n", error, file );
191 // If successful, then try to get additional info
194 // Get the stream info
195 error = av_find_stream_info( context ) < 0;
197 // Continue if no error
200 // We will default to the first audio and video streams found
201 int audio_index = -1;
202 int video_index = -1;
204 // Now set properties where we can (use default unknowns if required)
205 if ( context->duration != AV_NOPTS_VALUE )
207 // This isn't going to be accurate for all formats
208 mlt_position frames = ( mlt_position )( ( ( double )context->duration / ( double )AV_TIME_BASE ) * fps );
209 mlt_properties_set_position( properties, "out", frames - 2 );
210 mlt_properties_set_position( properties, "length", frames - 1 );
213 // Find default audio and video streams
214 find_default_streams( context, &audio_index, &video_index );
216 // Store selected audio and video indexes on properties
217 mlt_properties_set_int( properties, "audio_index", audio_index );
218 mlt_properties_set_int( properties, "video_index", video_index );
220 // We're going to cheat here - for a/v files, we will have two contexts (reasoning will be clear later)
221 if ( audio_index != -1 && video_index != -1 )
223 // We'll use the open one as our video_context
224 mlt_properties_set_data( properties, "video_context", context, 0, producer_file_close, NULL );
226 // And open again for our audio context
227 av_open_input_file( &context, file, NULL, 0, NULL );
228 av_find_stream_info( context );
231 mlt_properties_set_data( properties, "audio_context", context, 0, producer_file_close, NULL );
233 else if ( video_index != -1 )
235 // We only have a video context
236 mlt_properties_set_data( properties, "video_context", context, 0, producer_file_close, NULL );
238 else if ( audio_index != -1 )
240 // We only have an audio context
241 mlt_properties_set_data( properties, "audio_context", context, 0, producer_file_close, NULL );
245 // Something has gone wrong
251 // Unlock the mutex now
252 pthread_mutex_unlock( &avformat_mutex );
257 /** Convert a frame position to a time code.
260 static double producer_time_of_frame( mlt_producer this, mlt_position position )
262 // Get the properties
263 mlt_properties properties = mlt_producer_properties( this );
266 double fps = mlt_properties_get_double( properties, "fps" );
269 return ( double )position / fps;
272 /** Get an image from a frame.
275 static int producer_get_image( mlt_frame frame, uint8_t **buffer, mlt_image_format *format, int *width, int *height, int writable )
277 // Get the properties from the frame
278 mlt_properties frame_properties = mlt_frame_properties( frame );
280 // Obtain the frame number of this frame
281 mlt_position position = mlt_properties_get_position( frame_properties, "avformat_position" );
284 mlt_producer this = mlt_properties_get_data( frame_properties, "avformat_producer", NULL );
286 // Get the producer properties
287 mlt_properties properties = mlt_producer_properties( this );
289 // Fetch the video_context
290 AVFormatContext *context = mlt_properties_get_data( properties, "video_context", NULL );
292 // Get the video_index
293 int index = mlt_properties_get_int( properties, "video_index" );
295 // Obtain the expected frame numer
296 mlt_position expected = mlt_properties_get_position( properties, "video_expected" );
298 // Calculate the real time code
299 double real_timecode = producer_time_of_frame( this, position );
301 // Get the video stream
302 AVStream *stream = context->streams[ index ];
305 AVCodecContext *codec_context = &stream->codec;
310 // Get the conversion frame
311 AVPicture *output = mlt_properties_get_data( properties, "video_output_frame", NULL );
313 // Special case pause handling flag
316 // Special case ffwd handling
319 // Current time calcs
320 double current_time = 0;
322 // Set the result arguments that we know here (only *buffer is now required)
323 *format = mlt_image_yuv422;
324 *width = codec_context->width;
325 *height = codec_context->height;
327 // Set this on the frame properties
328 mlt_properties_set_int( frame_properties, "width", *width );
329 mlt_properties_set_int( frame_properties, "height", *height );
331 // Lock the mutex now
332 pthread_mutex_lock( &avformat_mutex );
334 // Construct an AVFrame for YUV422 conversion
335 if ( output == NULL )
337 int size = avpicture_get_size( PIX_FMT_YUV422, *width, *height );
340 uint8_t *buf = malloc( size );
341 output = malloc( sizeof( AVPicture ) );
342 avpicture_fill( output, buf, PIX_FMT_YUV422, *width, *height );
343 mlt_properties_set_data( properties, "video_output_frame", output, 0, av_free, NULL );
344 mlt_properties_set_data( properties, "video_output_buffer", buf, 0, free, NULL );
348 if ( position != expected )
350 if ( position + 1 == expected )
352 // We're paused - use last image
355 else if ( position > expected && ( position - expected ) < 250 )
357 // Fast forward - seeking is inefficient for small distances - just ignore following frames
358 ignore = position - expected;
362 // Set to the real timecode
363 av_seek_frame( context, -1, real_timecode * 1000000.0 );
365 // Remove the cached info relating to the previous position
366 mlt_properties_set_double( properties, "current_time", 0 );
367 mlt_properties_set_data( properties, "current_image", NULL, 0, NULL, NULL );
371 // Duplicate the last image if necessary
372 if ( mlt_properties_get_data( properties, "current_image", NULL ) != NULL &&
373 ( paused || mlt_properties_get_double( properties, "current_time" ) > real_timecode ) )
375 // Get current image and size
377 uint8_t *image = mlt_properties_get_data( properties, "current_image", &size );
381 *buffer = malloc( size );
382 memcpy( *buffer, image, size );
384 // Set this on the frame properties
385 mlt_properties_set_data( frame_properties, "image", *buffer, size, free, NULL );
393 memset( &pkt, 0, sizeof( pkt ) );
394 memset( &frame, 0, sizeof( frame ) );
396 while( ret >= 0 && !got_picture )
399 ret = av_read_frame( context, &pkt );
401 // We only deal with video from the selected video_index
402 if ( ret >= 0 && pkt.stream_index == index && pkt.size > 0 )
405 // Wouldn't it be great if I could use this...
406 //if ( (float)pkt.pts / 1000000.0 >= real_timecode )
407 ret = avcodec_decode_video( codec_context, &frame, &got_picture, pkt.data, pkt.size );
410 if ( (float)pkt.pts / 1000000.0 < real_timecode )
415 else if ( (float)pkt.pts / 1000000.0 >= real_timecode )
419 else if ( got_picture && ignore -- )
424 current_time = ( double )pkt.pts / 1000000.0;
427 // We're finished with this packet regardless
428 av_free_packet( &pkt );
431 // Now handle the picture if we have one
434 // Get current image and size
436 uint8_t *image = mlt_properties_get_data( properties, "current_image", &size );
438 if ( image == NULL || size != *width * *height * 2 )
440 size = *width * ( *height + 1 ) * 2;
442 image = malloc( size );
443 mlt_properties_set_data( properties, "current_image", image, size, free, NULL );
447 *buffer = malloc( size );
448 img_convert( output, PIX_FMT_YUV422, (AVPicture *)&frame, codec_context->pix_fmt, *width, *height );
449 memcpy( image, output->data[ 0 ], size );
450 memcpy( *buffer, output->data[ 0 ], size );
451 mlt_properties_set_data( frame_properties, "image", *buffer, size, free, NULL );
452 mlt_properties_set_double( properties, "current_time", current_time );
456 // Regardless of speed, we expect to get the next frame (cos we ain't too bright)
457 mlt_properties_set_position( properties, "video_expected", position + 1 );
459 // Unlock the mutex now
460 pthread_mutex_unlock( &avformat_mutex );
465 /** Set up video handling.
468 static void producer_set_up_video( mlt_producer this, mlt_frame frame )
470 // Get the properties
471 mlt_properties properties = mlt_producer_properties( this );
473 // Fetch the video_context
474 AVFormatContext *context = mlt_properties_get_data( properties, "video_context", NULL );
476 // Get the video_index
477 int index = mlt_properties_get_int( properties, "video_index" );
479 // Get the frame properties
480 mlt_properties frame_properties = mlt_frame_properties( frame );
482 // Lock the mutex now
483 pthread_mutex_lock( &avformat_mutex );
485 if ( context != NULL && index != -1 )
487 // Get the video stream
488 AVStream *stream = context->streams[ index ];
491 AVCodecContext *codec_context = &stream->codec;
494 AVCodec *codec = mlt_properties_get_data( properties, "video_codec", NULL );
496 // Initialise the codec if necessary
500 codec = avcodec_find_decoder( codec_context->codec_id );
502 // If we don't have a codec and we can't initialise it, we can't do much more...
503 if ( codec != NULL && avcodec_open( codec_context, codec ) >= 0 )
505 double aspect_ratio = 0;
508 fprintf( stderr, "AVFORMAT: sample aspect %d\n", codec_context->sample_aspect_ratio.num );
509 if ( codec_context->sample_aspect_ratio.num == 0)
512 aspect_ratio = av_q2d( codec_context->sample_aspect_ratio ) * codec_context->width / codec_context->height;
514 if (aspect_ratio <= 0.0)
515 aspect_ratio = ( double )codec_context->width / ( double )codec_context->height;
517 mlt_properties_set_double( properties, "aspect_ratio", aspect_ratio );
519 // Now store the codec with its destructor
520 mlt_properties_set_data( properties, "video_codec", codec_context, 0, producer_codec_close, NULL );
522 // Set to the real timecode
523 av_seek_frame( context, -1, 0 );
527 // Remember that we can't use this later
528 mlt_properties_set_int( properties, "video_index", -1 );
532 // No codec, no show...
535 mlt_frame_push_get_image( frame, producer_get_image );
536 mlt_properties_set_data( frame_properties, "avformat_producer", this, 0, NULL, NULL );
540 mlt_properties_set_int( frame_properties, "test_image", 1 );
545 mlt_properties_set_int( frame_properties, "test_image", 1 );
548 // Unlock the mutex now
549 pthread_mutex_unlock( &avformat_mutex );
552 /** Get the audio from a frame.
555 static int producer_get_audio( mlt_frame frame, int16_t **buffer, mlt_audio_format *format, int *frequency, int *channels, int *samples )
557 // Get the properties from the frame
558 mlt_properties frame_properties = mlt_frame_properties( frame );
560 // Obtain the frame number of this frame
561 mlt_position position = mlt_properties_get_position( frame_properties, "avformat_position" );
564 mlt_producer this = mlt_properties_get_data( frame_properties, "avformat_producer", NULL );
566 // Get the producer properties
567 mlt_properties properties = mlt_producer_properties( this );
569 // Fetch the audio_context
570 AVFormatContext *context = mlt_properties_get_data( properties, "audio_context", NULL );
572 // Get the audio_index
573 int index = mlt_properties_get_int( properties, "audio_index" );
575 // Obtain the expected frame numer
576 mlt_position expected = mlt_properties_get_position( properties, "audio_expected" );
578 // Obtain the resample context if it exists (not always needed)
579 ReSampleContext *resample = mlt_properties_get_data( properties, "audio_resample", NULL );
581 // Obtain the audio buffer
582 int16_t *audio_buffer = mlt_properties_get_data( properties, "audio_buffer", NULL );
584 // Get amount of audio used
585 int audio_used = mlt_properties_get_int( properties, "audio_used" );
587 // Calculate the real time code
588 double real_timecode = producer_time_of_frame( this, position );
590 // Get the audio stream
591 AVStream *stream = context->streams[ index ];
594 AVCodecContext *codec_context = &stream->codec;
599 // Number of frames to ignore (for ffwd)
602 // Flag for paused (silence)
605 // Lock the mutex now
606 pthread_mutex_lock( &avformat_mutex );
608 // Check for resample and create if necessary
609 if ( resample == NULL )
611 // Create the resampler
612 resample = audio_resample_init( *channels, codec_context->channels, *frequency, codec_context->sample_rate );
614 // And store it on properties
615 mlt_properties_set_data( properties, "audio_resample", resample, 0, ( mlt_destructor )audio_resample_close, NULL );
618 // Check for audio buffer and create if necessary
619 if ( audio_buffer == NULL )
621 // Allocate the audio buffer
622 audio_buffer = malloc( AVCODEC_MAX_AUDIO_FRAME_SIZE * sizeof( int16_t ) );
624 // And store it on properties for reuse
625 mlt_properties_set_data( properties, "audio_buffer", audio_buffer, 0, free, NULL );
629 if ( position != expected )
631 if ( position + 1 == expected )
633 // We're paused - silence required
636 else if ( position > expected && ( position - expected ) < 250 )
638 // Fast forward - seeking is inefficient for small distances - just ignore following frames
639 ignore = position - expected;
643 // Set to the real timecode
644 av_seek_frame( context, -1, real_timecode * 1000000.0 );
646 // Clear the usage in the audio buffer
651 // Get the audio if required
656 int16_t temp[ AVCODEC_MAX_AUDIO_FRAME_SIZE / 2 ];
658 memset( &pkt, 0, sizeof( pkt ) );
660 while( ret >= 0 && !got_audio )
662 // Check if the buffer already contains the samples required
663 if ( audio_used >= *samples && ignore == 0 )
670 ret = av_read_frame( context, &pkt );
673 uint8_t *ptr = pkt.data;
676 if ( ptr == NULL || len == 0 )
679 // We only deal with video from the selected video_index
680 while ( ret >= 0 && pkt.stream_index == index && len > 0 )
683 ret = avcodec_decode_audio( codec_context, temp, &data_size, ptr, len );
693 int size_out = audio_resample( resample, &audio_buffer[ audio_used * *channels ], temp, data_size / ( codec_context->channels * sizeof( int16_t ) ) );
695 audio_used += size_out;
698 while ( ignore && audio_used > *samples )
701 audio_used -= *samples;
702 memmove( audio_buffer, &audio_buffer[ *samples * *channels ], audio_used * sizeof( int16_t ) );
706 // If we're behind, ignore this packet
707 float current_pts = (float)pkt.pts / 1000000.0;
708 double discrepancy = mlt_properties_get_double( properties, "discrepancy" );
709 if ( discrepancy * current_pts < real_timecode )
713 // We're finished with this packet regardless
714 av_free_packet( &pkt );
717 // Now handle the audio if we have enough
719 if ( audio_used >= *samples )
721 *buffer = malloc( *samples * *channels * sizeof( int16_t ) );
722 memcpy( *buffer, audio_buffer, *samples * *channels * sizeof( int16_t ) );
723 audio_used -= *samples;
724 memmove( audio_buffer, &audio_buffer[ *samples * *channels ], audio_used * *channels * sizeof( int16_t ) );
725 mlt_properties_set_data( frame_properties, "audio", *buffer, 0, free, NULL );
729 frame->get_audio = NULL;
730 mlt_frame_get_audio( frame, buffer, format, frequency, channels, samples );
734 // Store the number of audio samples still available
735 mlt_properties_set_int( properties, "audio_used", audio_used );
739 // Get silence and don't touch the context
740 frame->get_audio = NULL;
741 mlt_frame_get_audio( frame, buffer, format, frequency, channels, samples );
744 // Regardless of speed, we expect to get the next frame (cos we ain't too bright)
745 mlt_properties_set_position( properties, "audio_expected", position + 1 );
747 // Unlock the mutex now
748 pthread_mutex_unlock( &avformat_mutex );
753 /** Set up audio handling.
756 static void producer_set_up_audio( mlt_producer this, mlt_frame frame )
758 // Get the properties
759 mlt_properties properties = mlt_producer_properties( this );
761 // Fetch the audio_context
762 AVFormatContext *context = mlt_properties_get_data( properties, "audio_context", NULL );
764 // Get the audio_index
765 int index = mlt_properties_get_int( properties, "audio_index" );
767 // Lock the mutex now
768 pthread_mutex_lock( &avformat_mutex );
770 // Deal with audio context
771 if ( context != NULL && index != -1 )
773 // Get the frame properties
774 mlt_properties frame_properties = mlt_frame_properties( frame );
776 // Get the audio stream
777 AVStream *stream = context->streams[ index ];
780 AVCodecContext *codec_context = &stream->codec;
783 AVCodec *codec = mlt_properties_get_data( properties, "audio_codec", NULL );
785 // Initialise the codec if necessary
789 codec = avcodec_find_decoder( codec_context->codec_id );
791 // If we don't have a codec and we can't initialise it, we can't do much more...
792 if ( codec != NULL && avcodec_open( codec_context, codec ) >= 0 )
794 // Now store the codec with its destructor
795 mlt_properties_set_data( properties, "audio_codec", codec_context, 0, producer_codec_close, NULL );
799 // Remember that we can't use this later
800 mlt_properties_set_int( properties, "audio_index", -1 );
804 // No codec, no show...
807 frame->get_audio = producer_get_audio;
808 mlt_properties_set_data( frame_properties, "avformat_producer", this, 0, NULL, NULL );
812 // Unlock the mutex now
813 pthread_mutex_unlock( &avformat_mutex );
816 /** Our get frame implementation.
819 static int producer_get_frame( mlt_producer this, mlt_frame_ptr frame, int index )
821 // Create an empty frame
822 *frame = mlt_frame_init( );
824 // Update timecode on the frame we're creating
825 mlt_frame_set_position( *frame, mlt_producer_position( this ) );
827 // Set the position of this producer
828 mlt_properties_set_position( mlt_frame_properties( *frame ), "avformat_position", mlt_producer_get_in( this ) + mlt_producer_position( this ) );
831 producer_set_up_video( this, *frame );
834 producer_set_up_audio( this, *frame );
836 // Set the aspect_ratio
837 mlt_properties_set_double( mlt_frame_properties( *frame ), "aspect_ratio", mlt_properties_get_double( mlt_producer_properties( this ), "aspect_ratio" ) );
839 // Calculate the next timecode
840 mlt_producer_prepare_next( this );