3 * \brief interface for all frame classes
6 * Copyright (C) 2003-2009 Ushodaya Enterprises Limited
7 * \author Charles Yates <charles.yates@pandora.be>
9 * This library is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public
11 * License as published by the Free Software Foundation; either
12 * version 2.1 of the License, or (at your option) any later version.
14 * This library is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with this library; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
24 #include "mlt_frame.h"
25 #include "mlt_producer.h"
26 #include "mlt_factory.h"
27 #include "mlt_profile.h"
34 /** Constructor for a frame.
37 mlt_frame mlt_frame_init( mlt_service service )
40 mlt_frame this = calloc( sizeof( struct mlt_frame_s ), 1 );
44 mlt_profile profile = mlt_service_profile( service );
46 // Initialise the properties
47 mlt_properties properties = &this->parent;
48 mlt_properties_init( properties, this );
50 // Set default properties on the frame
51 mlt_properties_set_position( properties, "_position", 0.0 );
52 mlt_properties_set_data( properties, "image", NULL, 0, NULL, NULL );
53 mlt_properties_set_int( properties, "width", profile? profile->width : 720 );
54 mlt_properties_set_int( properties, "height", profile? profile->height : 576 );
55 mlt_properties_set_int( properties, "normalised_width", profile? profile->width : 720 );
56 mlt_properties_set_int( properties, "normalised_height", profile? profile->height : 576 );
57 mlt_properties_set_double( properties, "aspect_ratio", mlt_profile_sar( NULL ) );
58 mlt_properties_set_data( properties, "audio", NULL, 0, NULL, NULL );
59 mlt_properties_set_data( properties, "alpha", NULL, 0, NULL, NULL );
61 // Construct stacks for frames and methods
62 this->stack_image = mlt_deque_init( );
63 this->stack_audio = mlt_deque_init( );
64 this->stack_service = mlt_deque_init( );
70 /** Fetch the frames properties.
73 mlt_properties mlt_frame_properties( mlt_frame this )
75 return this != NULL ? &this->parent : NULL;
78 /** Check if we have a way to derive something other than a test card.
81 int mlt_frame_is_test_card( mlt_frame this )
83 return mlt_deque_count( this->stack_image ) == 0 || mlt_properties_get_int( MLT_FRAME_PROPERTIES( this ), "test_image" );
86 /** Check if we have a way to derive something other than test audio.
89 int mlt_frame_is_test_audio( mlt_frame this )
91 return mlt_deque_count( this->stack_audio ) == 0 || mlt_properties_get_int( MLT_FRAME_PROPERTIES( this ), "test_audio" );
94 /** Get the aspect ratio of the frame.
97 double mlt_frame_get_aspect_ratio( mlt_frame this )
99 return mlt_properties_get_double( MLT_FRAME_PROPERTIES( this ), "aspect_ratio" );
102 /** Set the aspect ratio of the frame.
105 int mlt_frame_set_aspect_ratio( mlt_frame this, double value )
107 return mlt_properties_set_double( MLT_FRAME_PROPERTIES( this ), "aspect_ratio", value );
110 /** Get the position of this frame.
113 mlt_position mlt_frame_get_position( mlt_frame this )
115 int pos = mlt_properties_get_position( MLT_FRAME_PROPERTIES( this ), "_position" );
116 return pos < 0 ? 0 : pos;
119 /** Set the position of this frame.
122 int mlt_frame_set_position( mlt_frame this, mlt_position value )
124 return mlt_properties_set_position( MLT_FRAME_PROPERTIES( this ), "_position", value );
127 /** Stack a get_image callback.
130 int mlt_frame_push_get_image( mlt_frame this, mlt_get_image get_image )
132 return mlt_deque_push_back( this->stack_image, get_image );
135 /** Pop a get_image callback.
138 mlt_get_image mlt_frame_pop_get_image( mlt_frame this )
140 return mlt_deque_pop_back( this->stack_image );
146 int mlt_frame_push_frame( mlt_frame this, mlt_frame that )
148 return mlt_deque_push_back( this->stack_image, that );
154 mlt_frame mlt_frame_pop_frame( mlt_frame this )
156 return mlt_deque_pop_back( this->stack_image );
162 int mlt_frame_push_service( mlt_frame this, void *that )
164 return mlt_deque_push_back( this->stack_image, that );
170 void *mlt_frame_pop_service( mlt_frame this )
172 return mlt_deque_pop_back( this->stack_image );
178 int mlt_frame_push_service_int( mlt_frame this, int that )
180 return mlt_deque_push_back_int( this->stack_image, that );
186 int mlt_frame_pop_service_int( mlt_frame this )
188 return mlt_deque_pop_back_int( this->stack_image );
191 /** Push an audio item on the stack.
194 int mlt_frame_push_audio( mlt_frame this, void *that )
196 return mlt_deque_push_back( this->stack_audio, that );
199 /** Pop an audio item from the stack
202 void *mlt_frame_pop_audio( mlt_frame this )
204 return mlt_deque_pop_back( this->stack_audio );
207 /** Return the service stack
210 mlt_deque mlt_frame_service_stack( mlt_frame this )
212 return this->stack_service;
215 /** Replace image stack with the information provided.
217 This might prove to be unreliable and restrictive - the idea is that a transition
218 which normally uses two images may decide to only use the b frame (ie: in the case
219 of a composite where the b frame completely obscures the a frame).
221 The image must be writable and the destructor for the image itself must be taken
222 care of on another frame and that frame cannot have a replace applied to it...
223 Further it assumes that no alpha mask is in use.
225 For these reasons, it can only be used in a specific situation - when you have
226 multiple tracks each with their own transition and these transitions are applied
227 in a strictly reversed order (ie: highest numbered [lowest track] is processed
230 More reliable approach - the cases should be detected during the process phase
231 and the upper tracks should simply not be invited to stack...
234 void mlt_frame_replace_image( mlt_frame this, uint8_t *image, mlt_image_format format, int width, int height )
236 // Remove all items from the stack
237 while( mlt_deque_pop_back( this->stack_image ) ) ;
239 // Update the information
240 mlt_properties_set_data( MLT_FRAME_PROPERTIES( this ), "image", image, 0, NULL, NULL );
241 mlt_properties_set_int( MLT_FRAME_PROPERTIES( this ), "width", width );
242 mlt_properties_set_int( MLT_FRAME_PROPERTIES( this ), "height", height );
243 mlt_properties_set_int( MLT_FRAME_PROPERTIES( this ), "format", format );
244 this->get_alpha_mask = NULL;
247 const char * mlt_image_format_name( mlt_image_format format )
251 case mlt_image_none: return "none";
252 case mlt_image_rgb24: return "rgb24";
253 case mlt_image_rgb24a: return "rgb24a";
254 case mlt_image_yuv422: return "yuv422";
255 case mlt_image_yuv420p: return "yuv420p";
256 case mlt_image_opengl: return "opengl";
261 /** Get the image associated to the frame.
264 int mlt_frame_get_image( mlt_frame this, uint8_t **buffer, mlt_image_format *format, int *width, int *height, int writable )
266 mlt_properties properties = MLT_FRAME_PROPERTIES( this );
267 mlt_get_image get_image = mlt_frame_pop_get_image( this );
268 mlt_producer producer = mlt_properties_get_data( properties, "test_card_producer", NULL );
269 mlt_image_format requested_format = *format;
274 mlt_properties_set_int( properties, "image_count", mlt_properties_get_int( properties, "image_count" ) - 1 );
275 mlt_position position = mlt_frame_get_position( this );
276 error = get_image( this, buffer, format, width, height, writable );
277 mlt_properties_set_int( properties, "width", *width );
278 mlt_properties_set_int( properties, "height", *height );
279 mlt_properties_set_int( properties, "format", *format );
280 mlt_frame_set_position( this, position );
281 if ( this->convert_image )
282 this->convert_image( this, buffer, format, requested_format );
284 else if ( mlt_properties_get_data( properties, "image", NULL ) )
286 *format = mlt_properties_get_int( properties, "format" );
287 *buffer = mlt_properties_get_data( properties, "image", NULL );
288 *width = mlt_properties_get_int( properties, "width" );
289 *height = mlt_properties_get_int( properties, "height" );
290 if ( this->convert_image )
291 this->convert_image( this, buffer, format, requested_format );
295 mlt_frame test_frame = NULL;
296 mlt_service_get_frame( MLT_PRODUCER_SERVICE( producer ), &test_frame, 0 );
299 mlt_properties test_properties = MLT_FRAME_PROPERTIES( test_frame );
300 mlt_properties_set_double( test_properties, "consumer_aspect_ratio", mlt_properties_get_double( properties, "consumer_aspect_ratio" ) );
301 mlt_properties_set( test_properties, "rescale.interp", mlt_properties_get( properties, "rescale.interp" ) );
302 mlt_frame_get_image( test_frame, buffer, format, width, height, writable );
303 mlt_properties_set_data( properties, "test_card_frame", test_frame, 0, ( mlt_destructor )mlt_frame_close, NULL );
304 mlt_properties_set_double( properties, "aspect_ratio", mlt_frame_get_aspect_ratio( test_frame ) );
305 // mlt_properties_set_data( properties, "image", *buffer, *width * *height * 2, NULL, NULL );
306 // mlt_properties_set_int( properties, "width", *width );
307 // mlt_properties_set_int( properties, "height", *height );
308 // mlt_properties_set_int( properties, "format", *format );
312 mlt_properties_set_data( properties, "test_card_producer", NULL, 0, NULL, NULL );
313 mlt_frame_get_image( this, buffer, format, width, height, writable );
322 *width = *width == 0 ? 720 : *width;
323 *height = *height == 0 ? 576 : *height;
324 size = *width * *height;
326 mlt_properties_set_int( properties, "format", *format );
327 mlt_properties_set_int( properties, "width", *width );
328 mlt_properties_set_int( properties, "height", *height );
329 mlt_properties_set_int( properties, "aspect_ratio", 0 );
337 case mlt_image_rgb24:
340 *buffer = mlt_pool_alloc( size );
342 memset( *buffer, 255, size );
344 case mlt_image_rgb24a:
345 case mlt_image_opengl:
348 *buffer = mlt_pool_alloc( size );
350 memset( *buffer, 255, size );
352 case mlt_image_yuv422:
355 *buffer = mlt_pool_alloc( size );
358 while ( p != NULL && p != q )
364 case mlt_image_yuv420p:
366 *buffer = mlt_pool_alloc( size );
368 memset( *buffer, 255, size );
372 mlt_properties_set_data( properties, "image", *buffer, size, ( mlt_destructor )mlt_pool_release, NULL );
373 mlt_properties_set_int( properties, "test_image", 1 );
376 mlt_properties_set_int( properties, "scaled_width", *width );
377 mlt_properties_set_int( properties, "scaled_height", *height );
382 uint8_t *mlt_frame_get_alpha_mask( mlt_frame this )
384 uint8_t *alpha = NULL;
387 if ( this->get_alpha_mask != NULL )
388 alpha = this->get_alpha_mask( this );
390 alpha = mlt_properties_get_data( &this->parent, "alpha", NULL );
393 int size = mlt_properties_get_int( &this->parent, "scaled_width" ) * mlt_properties_get_int( &this->parent, "scaled_height" );
394 alpha = mlt_pool_alloc( size );
395 memset( alpha, 255, size );
396 mlt_properties_set_data( &this->parent, "alpha", alpha, size, mlt_pool_release, NULL );
402 const char * mlt_audio_format_name( mlt_audio_format format )
406 case mlt_audio_none: return "none";
407 case mlt_audio_s16: return "s16";
408 case mlt_audio_s32: return "s32";
409 case mlt_audio_float: return "float";
414 int mlt_frame_get_audio( mlt_frame this, void **buffer, mlt_audio_format *format, int *frequency, int *channels, int *samples )
416 mlt_get_audio get_audio = mlt_frame_pop_audio( this );
417 mlt_properties properties = MLT_FRAME_PROPERTIES( this );
418 int hide = mlt_properties_get_int( properties, "test_audio" );
419 mlt_audio_format requested_format = *format;
421 if ( hide == 0 && get_audio != NULL )
423 mlt_position position = mlt_frame_get_position( this );
424 get_audio( this, buffer, format, frequency, channels, samples );
425 mlt_frame_set_position( this, position );
426 mlt_properties_set_int( properties, "audio_frequency", *frequency );
427 mlt_properties_set_int( properties, "audio_channels", *channels );
428 mlt_properties_set_int( properties, "audio_samples", *samples );
429 mlt_properties_set_int( properties, "audio_format", *format );
430 if ( this->convert_audio )
431 this->convert_audio( this, buffer, format, requested_format );
433 else if ( mlt_properties_get_data( properties, "audio", NULL ) )
435 *buffer = mlt_properties_get_data( properties, "audio", NULL );
436 *format = mlt_properties_get_int( properties, "audio_format" );
437 *frequency = mlt_properties_get_int( properties, "audio_frequency" );
438 *channels = mlt_properties_get_int( properties, "audio_channels" );
439 *samples = mlt_properties_get_int( properties, "audio_samples" );
440 if ( this->convert_audio )
441 this->convert_audio( this, buffer, format, requested_format );
446 *samples = *samples <= 0 ? 1920 : *samples;
447 *channels = *channels <= 0 ? 2 : *channels;
448 *frequency = *frequency <= 0 ? 48000 : *frequency;
449 mlt_properties_set_int( properties, "audio_frequency", *frequency );
450 mlt_properties_set_int( properties, "audio_channels", *channels );
451 mlt_properties_set_int( properties, "audio_samples", *samples );
452 mlt_properties_set_int( properties, "audio_format", *format );
461 size = *samples * *channels * sizeof( int16_t );
464 size = *samples * *channels * sizeof( int32_t );
466 case mlt_audio_float:
467 size = *samples * *channels * sizeof( float );
471 *buffer = mlt_pool_alloc( size );
473 memset( *buffer, 0, size );
474 mlt_properties_set_data( properties, "audio", *buffer, size, ( mlt_destructor )mlt_pool_release, NULL );
475 mlt_properties_set_int( properties, "test_audio", 1 );
478 // TODO: This does not belong here
479 if ( *format == mlt_audio_s16 && mlt_properties_get( properties, "meta.volume" ) )
481 double value = mlt_properties_get_double( properties, "meta.volume" );
485 memset( *buffer, 0, *samples * *channels * 2 );
487 else if ( value != 1.0 )
489 int total = *samples * *channels;
490 int16_t *p = *buffer;
498 mlt_properties_set( properties, "meta.volume", NULL );
504 int mlt_frame_set_audio( mlt_frame this, void *buffer, mlt_audio_format format, int size, mlt_destructor destructor )
506 mlt_properties_set_int( MLT_FRAME_PROPERTIES( this ), "audio_format", format );
507 return mlt_properties_set_data( MLT_FRAME_PROPERTIES( this ), "audio", buffer, size, destructor, NULL );
510 unsigned char *mlt_frame_get_waveform( mlt_frame this, int w, int h )
513 mlt_properties properties = MLT_FRAME_PROPERTIES( this );
514 mlt_audio_format format = mlt_audio_s16;
515 int frequency = 32000; // lower frequency available?
517 double fps = mlt_profile_fps( NULL );
518 int samples = mlt_sample_calculator( fps, frequency, mlt_frame_get_position( this ) );
521 mlt_frame_get_audio( this, (void**)&pcm, &format, &frequency, &channels, &samples );
523 // Make an 8-bit buffer large enough to hold rendering
525 unsigned char *bitmap = ( unsigned char* )mlt_pool_alloc( size );
526 if ( bitmap != NULL )
527 memset( bitmap, 0, size );
528 mlt_properties_set_data( properties, "waveform", bitmap, size, ( mlt_destructor )mlt_pool_release, NULL );
530 // Render vertical lines
531 int16_t *ubound = pcm + samples * channels;
532 int skip = samples / w - 1;
535 // Iterate sample stream and along x coordinate
536 for ( i = 0; i < w && pcm < ubound; i++ )
538 // pcm data has channels interleaved
539 for ( j = 0; j < channels; j++ )
541 // Determine sample's magnitude from 2s complement;
542 int pcm_magnitude = *pcm < 0 ? ~(*pcm) + 1 : *pcm;
543 // The height of a line is the ratio of the magnitude multiplied by
544 // half the vertical resolution
545 int height = ( int )( ( double )( pcm_magnitude ) / 32768 * h / 2 );
546 // Determine the starting y coordinate - left channel above center,
547 // right channel below - currently assumes 2 channels
548 int displacement = ( h / 2 ) - ( 1 - j ) * height;
549 // Position buffer pointer using y coordinate, stride, and x coordinate
550 unsigned char *p = &bitmap[ i + displacement * w ];
552 // Draw vertical line
553 for ( k = 0; k < height; k++ )
558 pcm += skip * channels;
564 mlt_producer mlt_frame_get_original_producer( mlt_frame this )
567 return mlt_properties_get_data( MLT_FRAME_PROPERTIES( this ), "_producer", NULL );
571 void mlt_frame_close( mlt_frame this )
573 if ( this != NULL && mlt_properties_dec_ref( MLT_FRAME_PROPERTIES( this ) ) <= 0 )
575 mlt_deque_close( this->stack_image );
576 mlt_deque_close( this->stack_audio );
577 while( mlt_deque_peek_back( this->stack_service ) )
578 mlt_service_close( mlt_deque_pop_back( this->stack_service ) );
579 mlt_deque_close( this->stack_service );
580 mlt_properties_close( &this->parent );
585 /***** convenience functions *****/
587 /* Will this break when mlt_position is converted to double? -Zach */
588 int mlt_sample_calculator( float fps, int frequency, int64_t position )
594 /* Compute the cumulative number of samples until the start of this frame and the
595 cumulative number of samples until the start of the next frame. Round each to the
596 nearest integer and take the difference to determine the number of samples in
599 This approach should prevent rounding errors that can accumulate over a large number
600 of frames causing A/V sync problems. */
602 int64_t samples_at_this =
603 (int64_t)( (double) position * (double) frequency / (double) fps +
604 ( position < 0 ? -0.5 : 0.5 ) );
605 int64_t samples_at_next =
606 (int64_t)( (double) (position + 1) * (double) frequency / (double) fps +
607 ( position < 0 ? -0.5 : 0.5 ) );
608 samples = (int)( samples_at_next - samples_at_this );
614 int64_t mlt_sample_calculator_to_now( float fps, int frequency, int64_t frame )
620 samples = (int64_t)( (double) frame * (double) frequency / (double) fps +
621 ( frame < 0 ? -0.5 : 0.5 ) );