3 * \brief interface for all frame classes
6 * Copyright (C) 2003-2009 Ushodaya Enterprises Limited
7 * \author Charles Yates <charles.yates@pandora.be>
9 * This library is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public
11 * License as published by the Free Software Foundation; either
12 * version 2.1 of the License, or (at your option) any later version.
14 * This library is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with this library; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
24 #include "mlt_frame.h"
25 #include "mlt_producer.h"
26 #include "mlt_factory.h"
27 #include "mlt_profile.h"
34 /** Constructor for a frame.
37 mlt_frame mlt_frame_init( mlt_service service )
40 mlt_frame this = calloc( sizeof( struct mlt_frame_s ), 1 );
44 mlt_profile profile = mlt_service_profile( service );
46 // Initialise the properties
47 mlt_properties properties = &this->parent;
48 mlt_properties_init( properties, this );
50 // Set default properties on the frame
51 mlt_properties_set_position( properties, "_position", 0.0 );
52 mlt_properties_set_data( properties, "image", NULL, 0, NULL, NULL );
53 mlt_properties_set_int( properties, "width", profile? profile->width : 720 );
54 mlt_properties_set_int( properties, "height", profile? profile->height : 576 );
55 mlt_properties_set_int( properties, "normalised_width", profile? profile->width : 720 );
56 mlt_properties_set_int( properties, "normalised_height", profile? profile->height : 576 );
57 mlt_properties_set_double( properties, "aspect_ratio", mlt_profile_sar( NULL ) );
58 mlt_properties_set_data( properties, "audio", NULL, 0, NULL, NULL );
59 mlt_properties_set_data( properties, "alpha", NULL, 0, NULL, NULL );
61 // Construct stacks for frames and methods
62 this->stack_image = mlt_deque_init( );
63 this->stack_audio = mlt_deque_init( );
64 this->stack_service = mlt_deque_init( );
70 /** Fetch the frames properties.
73 mlt_properties mlt_frame_properties( mlt_frame this )
75 return this != NULL ? &this->parent : NULL;
78 /** Check if we have a way to derive something other than a test card.
81 int mlt_frame_is_test_card( mlt_frame this )
83 return mlt_deque_count( this->stack_image ) == 0 || mlt_properties_get_int( MLT_FRAME_PROPERTIES( this ), "test_image" );
86 /** Check if we have a way to derive something other than test audio.
89 int mlt_frame_is_test_audio( mlt_frame this )
91 return mlt_deque_count( this->stack_audio ) == 0 || mlt_properties_get_int( MLT_FRAME_PROPERTIES( this ), "test_audio" );
94 /** Get the aspect ratio of the frame.
97 double mlt_frame_get_aspect_ratio( mlt_frame this )
99 return mlt_properties_get_double( MLT_FRAME_PROPERTIES( this ), "aspect_ratio" );
102 /** Set the aspect ratio of the frame.
105 int mlt_frame_set_aspect_ratio( mlt_frame this, double value )
107 return mlt_properties_set_double( MLT_FRAME_PROPERTIES( this ), "aspect_ratio", value );
110 /** Get the position of this frame.
113 mlt_position mlt_frame_get_position( mlt_frame this )
115 int pos = mlt_properties_get_position( MLT_FRAME_PROPERTIES( this ), "_position" );
116 return pos < 0 ? 0 : pos;
119 /** Set the position of this frame.
122 int mlt_frame_set_position( mlt_frame this, mlt_position value )
124 return mlt_properties_set_position( MLT_FRAME_PROPERTIES( this ), "_position", value );
127 /** Stack a get_image callback.
130 int mlt_frame_push_get_image( mlt_frame this, mlt_get_image get_image )
132 return mlt_deque_push_back( this->stack_image, get_image );
135 /** Pop a get_image callback.
138 mlt_get_image mlt_frame_pop_get_image( mlt_frame this )
140 return mlt_deque_pop_back( this->stack_image );
146 int mlt_frame_push_frame( mlt_frame this, mlt_frame that )
148 return mlt_deque_push_back( this->stack_image, that );
154 mlt_frame mlt_frame_pop_frame( mlt_frame this )
156 return mlt_deque_pop_back( this->stack_image );
162 int mlt_frame_push_service( mlt_frame this, void *that )
164 return mlt_deque_push_back( this->stack_image, that );
170 void *mlt_frame_pop_service( mlt_frame this )
172 return mlt_deque_pop_back( this->stack_image );
178 int mlt_frame_push_service_int( mlt_frame this, int that )
180 return mlt_deque_push_back_int( this->stack_image, that );
186 int mlt_frame_pop_service_int( mlt_frame this )
188 return mlt_deque_pop_back_int( this->stack_image );
191 /** Push an audio item on the stack.
194 int mlt_frame_push_audio( mlt_frame this, void *that )
196 return mlt_deque_push_back( this->stack_audio, that );
199 /** Pop an audio item from the stack
202 void *mlt_frame_pop_audio( mlt_frame this )
204 return mlt_deque_pop_back( this->stack_audio );
207 /** Return the service stack
210 mlt_deque mlt_frame_service_stack( mlt_frame this )
212 return this->stack_service;
215 /** Replace image stack with the information provided.
217 This might prove to be unreliable and restrictive - the idea is that a transition
218 which normally uses two images may decide to only use the b frame (ie: in the case
219 of a composite where the b frame completely obscures the a frame).
221 The image must be writable and the destructor for the image itself must be taken
222 care of on another frame and that frame cannot have a replace applied to it...
223 Further it assumes that no alpha mask is in use.
225 For these reasons, it can only be used in a specific situation - when you have
226 multiple tracks each with their own transition and these transitions are applied
227 in a strictly reversed order (ie: highest numbered [lowest track] is processed
230 More reliable approach - the cases should be detected during the process phase
231 and the upper tracks should simply not be invited to stack...
234 void mlt_frame_replace_image( mlt_frame this, uint8_t *image, mlt_image_format format, int width, int height )
236 // Remove all items from the stack
237 while( mlt_deque_pop_back( this->stack_image ) ) ;
239 // Update the information
240 mlt_properties_set_data( MLT_FRAME_PROPERTIES( this ), "image", image, 0, NULL, NULL );
241 mlt_properties_set_int( MLT_FRAME_PROPERTIES( this ), "width", width );
242 mlt_properties_set_int( MLT_FRAME_PROPERTIES( this ), "height", height );
243 mlt_properties_set_int( MLT_FRAME_PROPERTIES( this ), "format", format );
244 this->get_alpha_mask = NULL;
247 const char * mlt_image_format_name( mlt_image_format format )
251 case mlt_image_none: return "none";
252 case mlt_image_rgb24: return "rgb24";
253 case mlt_image_rgb24a: return "rgb24a";
254 case mlt_image_yuv422: return "yuv422";
255 case mlt_image_yuv420p: return "yuv420p";
256 case mlt_image_opengl: return "opengl";
261 /** Get the image associated to the frame.
264 int mlt_frame_get_image( mlt_frame this, uint8_t **buffer, mlt_image_format *format, int *width, int *height, int writable )
266 mlt_properties properties = MLT_FRAME_PROPERTIES( this );
267 mlt_get_image get_image = mlt_frame_pop_get_image( this );
268 mlt_producer producer = mlt_properties_get_data( properties, "test_card_producer", NULL );
269 mlt_image_format requested_format = *format;
274 mlt_properties_set_int( properties, "image_count", mlt_properties_get_int( properties, "image_count" ) - 1 );
275 error = get_image( this, buffer, format, width, height, writable );
276 mlt_properties_set_int( properties, "width", *width );
277 mlt_properties_set_int( properties, "height", *height );
278 mlt_properties_set_int( properties, "format", *format );
279 if ( this->convert_image )
280 this->convert_image( this, buffer, format, requested_format );
282 else if ( mlt_properties_get_data( properties, "image", NULL ) )
284 *format = mlt_properties_get_int( properties, "format" );
285 *buffer = mlt_properties_get_data( properties, "image", NULL );
286 *width = mlt_properties_get_int( properties, "width" );
287 *height = mlt_properties_get_int( properties, "height" );
288 if ( this->convert_image )
289 this->convert_image( this, buffer, format, requested_format );
293 mlt_frame test_frame = NULL;
294 mlt_service_get_frame( MLT_PRODUCER_SERVICE( producer ), &test_frame, 0 );
297 mlt_properties test_properties = MLT_FRAME_PROPERTIES( test_frame );
298 mlt_properties_set_double( test_properties, "consumer_aspect_ratio", mlt_properties_get_double( properties, "consumer_aspect_ratio" ) );
299 mlt_properties_set( test_properties, "rescale.interp", mlt_properties_get( properties, "rescale.interp" ) );
300 mlt_frame_get_image( test_frame, buffer, format, width, height, writable );
301 mlt_properties_set_data( properties, "test_card_frame", test_frame, 0, ( mlt_destructor )mlt_frame_close, NULL );
302 mlt_properties_set_double( properties, "aspect_ratio", mlt_frame_get_aspect_ratio( test_frame ) );
303 // mlt_properties_set_data( properties, "image", *buffer, *width * *height * 2, NULL, NULL );
304 // mlt_properties_set_int( properties, "width", *width );
305 // mlt_properties_set_int( properties, "height", *height );
306 // mlt_properties_set_int( properties, "format", *format );
310 mlt_properties_set_data( properties, "test_card_producer", NULL, 0, NULL, NULL );
311 mlt_frame_get_image( this, buffer, format, width, height, writable );
320 *width = *width == 0 ? 720 : *width;
321 *height = *height == 0 ? 576 : *height;
322 size = *width * *height;
324 mlt_properties_set_int( properties, "format", *format );
325 mlt_properties_set_int( properties, "width", *width );
326 mlt_properties_set_int( properties, "height", *height );
327 mlt_properties_set_int( properties, "aspect_ratio", 0 );
335 case mlt_image_rgb24:
338 *buffer = mlt_pool_alloc( size );
340 memset( *buffer, 255, size );
342 case mlt_image_rgb24a:
343 case mlt_image_opengl:
346 *buffer = mlt_pool_alloc( size );
348 memset( *buffer, 255, size );
350 case mlt_image_yuv422:
353 *buffer = mlt_pool_alloc( size );
356 while ( p != NULL && p != q )
362 case mlt_image_yuv420p:
364 *buffer = mlt_pool_alloc( size );
366 memset( *buffer, 255, size );
370 mlt_properties_set_data( properties, "image", *buffer, size, ( mlt_destructor )mlt_pool_release, NULL );
371 mlt_properties_set_int( properties, "test_image", 1 );
374 mlt_properties_set_int( properties, "scaled_width", *width );
375 mlt_properties_set_int( properties, "scaled_height", *height );
380 uint8_t *mlt_frame_get_alpha_mask( mlt_frame this )
382 uint8_t *alpha = NULL;
385 if ( this->get_alpha_mask != NULL )
386 alpha = this->get_alpha_mask( this );
388 alpha = mlt_properties_get_data( &this->parent, "alpha", NULL );
391 int size = mlt_properties_get_int( &this->parent, "scaled_width" ) * mlt_properties_get_int( &this->parent, "scaled_height" );
392 alpha = mlt_pool_alloc( size );
393 memset( alpha, 255, size );
394 mlt_properties_set_data( &this->parent, "alpha", alpha, size, mlt_pool_release, NULL );
400 const char * mlt_audio_format_name( mlt_audio_format format )
404 case mlt_audio_none: return "none";
405 case mlt_audio_s16: return "s16";
406 case mlt_audio_s32: return "s32";
407 case mlt_audio_float: return "float";
412 int mlt_frame_get_audio( mlt_frame this, void **buffer, mlt_audio_format *format, int *frequency, int *channels, int *samples )
414 mlt_get_audio get_audio = mlt_frame_pop_audio( this );
415 mlt_properties properties = MLT_FRAME_PROPERTIES( this );
416 int hide = mlt_properties_get_int( properties, "test_audio" );
417 mlt_audio_format requested_format = *format;
419 if ( hide == 0 && get_audio != NULL )
421 get_audio( this, buffer, format, frequency, channels, samples );
422 mlt_properties_set_int( properties, "audio_frequency", *frequency );
423 mlt_properties_set_int( properties, "audio_channels", *channels );
424 mlt_properties_set_int( properties, "audio_samples", *samples );
425 mlt_properties_set_int( properties, "audio_format", *format );
426 if ( this->convert_audio )
427 this->convert_audio( this, buffer, format, requested_format );
429 else if ( mlt_properties_get_data( properties, "audio", NULL ) )
431 *buffer = mlt_properties_get_data( properties, "audio", NULL );
432 *format = mlt_properties_get_int( properties, "audio_format" );
433 *frequency = mlt_properties_get_int( properties, "audio_frequency" );
434 *channels = mlt_properties_get_int( properties, "audio_channels" );
435 *samples = mlt_properties_get_int( properties, "audio_samples" );
436 if ( this->convert_audio )
437 this->convert_audio( this, buffer, format, requested_format );
442 *samples = *samples <= 0 ? 1920 : *samples;
443 *channels = *channels <= 0 ? 2 : *channels;
444 *frequency = *frequency <= 0 ? 48000 : *frequency;
445 mlt_properties_set_int( properties, "audio_frequency", *frequency );
446 mlt_properties_set_int( properties, "audio_channels", *channels );
447 mlt_properties_set_int( properties, "audio_samples", *samples );
448 mlt_properties_set_int( properties, "audio_format", *format );
457 size = *samples * *channels * sizeof( int16_t );
460 size = *samples * *channels * sizeof( int32_t );
462 case mlt_audio_float:
463 size = *samples * *channels * sizeof( float );
467 *buffer = mlt_pool_alloc( size );
469 memset( *buffer, 0, size );
470 mlt_properties_set_data( properties, "audio", *buffer, size, ( mlt_destructor )mlt_pool_release, NULL );
471 mlt_properties_set_int( properties, "test_audio", 1 );
474 // TODO: This does not belong here
475 if ( *format == mlt_audio_s16 && mlt_properties_get( properties, "meta.volume" ) )
477 double value = mlt_properties_get_double( properties, "meta.volume" );
481 memset( *buffer, 0, *samples * *channels * 2 );
483 else if ( value != 1.0 )
485 int total = *samples * *channels;
486 int16_t *p = *buffer;
494 mlt_properties_set( properties, "meta.volume", NULL );
500 int mlt_frame_set_audio( mlt_frame this, void *buffer, mlt_audio_format format, int size, mlt_destructor destructor )
502 mlt_properties_set_int( MLT_FRAME_PROPERTIES( this ), "audio_format", format );
503 return mlt_properties_set_data( MLT_FRAME_PROPERTIES( this ), "audio", buffer, size, destructor, NULL );
506 unsigned char *mlt_frame_get_waveform( mlt_frame this, int w, int h )
509 mlt_properties properties = MLT_FRAME_PROPERTIES( this );
510 mlt_audio_format format = mlt_audio_s16;
511 int frequency = 32000; // lower frequency available?
513 double fps = mlt_profile_fps( NULL );
514 int samples = mlt_sample_calculator( fps, frequency, mlt_frame_get_position( this ) );
517 mlt_frame_get_audio( this, (void**)&pcm, &format, &frequency, &channels, &samples );
519 // Make an 8-bit buffer large enough to hold rendering
521 unsigned char *bitmap = ( unsigned char* )mlt_pool_alloc( size );
522 if ( bitmap != NULL )
523 memset( bitmap, 0, size );
524 mlt_properties_set_data( properties, "waveform", bitmap, size, ( mlt_destructor )mlt_pool_release, NULL );
526 // Render vertical lines
527 int16_t *ubound = pcm + samples * channels;
528 int skip = samples / w - 1;
531 // Iterate sample stream and along x coordinate
532 for ( i = 0; i < w && pcm < ubound; i++ )
534 // pcm data has channels interleaved
535 for ( j = 0; j < channels; j++ )
537 // Determine sample's magnitude from 2s complement;
538 int pcm_magnitude = *pcm < 0 ? ~(*pcm) + 1 : *pcm;
539 // The height of a line is the ratio of the magnitude multiplied by
540 // half the vertical resolution
541 int height = ( int )( ( double )( pcm_magnitude ) / 32768 * h / 2 );
542 // Determine the starting y coordinate - left channel above center,
543 // right channel below - currently assumes 2 channels
544 int displacement = ( h / 2 ) - ( 1 - j ) * height;
545 // Position buffer pointer using y coordinate, stride, and x coordinate
546 unsigned char *p = &bitmap[ i + displacement * w ];
548 // Draw vertical line
549 for ( k = 0; k < height; k++ )
554 pcm += skip * channels;
560 mlt_producer mlt_frame_get_original_producer( mlt_frame this )
563 return mlt_properties_get_data( MLT_FRAME_PROPERTIES( this ), "_producer", NULL );
567 void mlt_frame_close( mlt_frame this )
569 if ( this != NULL && mlt_properties_dec_ref( MLT_FRAME_PROPERTIES( this ) ) <= 0 )
571 mlt_deque_close( this->stack_image );
572 mlt_deque_close( this->stack_audio );
573 while( mlt_deque_peek_back( this->stack_service ) )
574 mlt_service_close( mlt_deque_pop_back( this->stack_service ) );
575 mlt_deque_close( this->stack_service );
576 mlt_properties_close( &this->parent );
581 /***** convenience functions *****/
583 /* Will this break when mlt_position is converted to double? -Zach */
584 int mlt_sample_calculator( float fps, int frequency, int64_t position )
590 /* Compute the cumulative number of samples until the start of this frame and the
591 cumulative number of samples until the start of the next frame. Round each to the
592 nearest integer and take the difference to determine the number of samples in
595 This approach should prevent rounding errors that can accumulate over a large number
596 of frames causing A/V sync problems. */
598 int64_t samples_at_this =
599 (int64_t)( (double) position * (double) frequency / (double) fps +
600 ( position < 0 ? -0.5 : 0.5 ) );
601 int64_t samples_at_next =
602 (int64_t)( (double) (position + 1) * (double) frequency / (double) fps +
603 ( position < 0 ? -0.5 : 0.5 ) );
604 samples = (int)( samples_at_next - samples_at_this );
610 int64_t mlt_sample_calculator_to_now( float fps, int frequency, int64_t frame )
616 samples = (int64_t)( (double) frame * (double) frequency / (double) fps +
617 ( frame < 0 ? -0.5 : 0.5 ) );