3 * \brief interface for all frame classes
6 * Copyright (C) 2003-2009 Ushodaya Enterprises Limited
7 * \author Charles Yates <charles.yates@pandora.be>
9 * This library is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public
11 * License as published by the Free Software Foundation; either
12 * version 2.1 of the License, or (at your option) any later version.
14 * This library is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with this library; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
24 #include "mlt_frame.h"
25 #include "mlt_producer.h"
26 #include "mlt_factory.h"
27 #include "mlt_profile.h"
34 /** Construct a frame object.
36 * \public \memberof mlt_frame_s
37 * \param service the pointer to any service that can provide access to the profile
38 * \return a frame object on success or NULL if there was an allocation error
41 mlt_frame mlt_frame_init( mlt_service service )
44 mlt_frame this = calloc( sizeof( struct mlt_frame_s ), 1 );
48 mlt_profile profile = mlt_service_profile( service );
50 // Initialise the properties
51 mlt_properties properties = &this->parent;
52 mlt_properties_init( properties, this );
54 // Set default properties on the frame
55 mlt_properties_set_position( properties, "_position", 0.0 );
56 mlt_properties_set_data( properties, "image", NULL, 0, NULL, NULL );
57 mlt_properties_set_int( properties, "width", profile? profile->width : 720 );
58 mlt_properties_set_int( properties, "height", profile? profile->height : 576 );
59 mlt_properties_set_int( properties, "normalised_width", profile? profile->width : 720 );
60 mlt_properties_set_int( properties, "normalised_height", profile? profile->height : 576 );
61 mlt_properties_set_double( properties, "aspect_ratio", mlt_profile_sar( NULL ) );
62 mlt_properties_set_data( properties, "audio", NULL, 0, NULL, NULL );
63 mlt_properties_set_data( properties, "alpha", NULL, 0, NULL, NULL );
65 // Construct stacks for frames and methods
66 this->stack_image = mlt_deque_init( );
67 this->stack_audio = mlt_deque_init( );
68 this->stack_service = mlt_deque_init( );
74 /** Get a frame's properties.
76 * \public \memberof mlt_frame_s
78 * \return the frame's properties or NULL if an invalid frame is supplied
81 mlt_properties mlt_frame_properties( mlt_frame this )
83 return this != NULL ? &this->parent : NULL;
86 /** Determine if the frame will produce a test card image.
88 * \public \memberof mlt_frame_s
90 * \return true (non-zero) if this will produce from a test card
93 int mlt_frame_is_test_card( mlt_frame this )
95 return mlt_deque_count( this->stack_image ) == 0 || mlt_properties_get_int( MLT_FRAME_PROPERTIES( this ), "test_image" );
98 /** Determine if the frame will produce audio from a test card.
100 * \public \memberof mlt_frame_s
101 * \param this a frame
102 * \return true (non-zero) if this will produce from a test card
105 int mlt_frame_is_test_audio( mlt_frame this )
107 return mlt_deque_count( this->stack_audio ) == 0 || mlt_properties_get_int( MLT_FRAME_PROPERTIES( this ), "test_audio" );
110 /** Get the sample aspect ratio of the frame.
112 * \public \memberof mlt_frame_s
113 * \param this a frame
114 * \return the aspect ratio
117 double mlt_frame_get_aspect_ratio( mlt_frame this )
119 return mlt_properties_get_double( MLT_FRAME_PROPERTIES( this ), "aspect_ratio" );
122 /** Set the sample aspect ratio of the frame.
124 * \public \memberof mlt_frame_s
125 * \param this a frame
126 * \param value the new image sample aspect ratio
127 * \return true if error
130 int mlt_frame_set_aspect_ratio( mlt_frame this, double value )
132 return mlt_properties_set_double( MLT_FRAME_PROPERTIES( this ), "aspect_ratio", value );
135 /** Get the time position of this frame.
137 * \public \memberof mlt_frame_s
138 * \param this a frame
139 * \return the position
142 mlt_position mlt_frame_get_position( mlt_frame this )
144 int pos = mlt_properties_get_position( MLT_FRAME_PROPERTIES( this ), "_position" );
145 return pos < 0 ? 0 : pos;
148 /** Set the time position of this frame.
150 * \public \memberof mlt_frame_s
151 * \param this a frame
152 * \param value the position
153 * \return true if error
156 int mlt_frame_set_position( mlt_frame this, mlt_position value )
158 return mlt_properties_set_position( MLT_FRAME_PROPERTIES( this ), "_position", value );
161 /** Stack a get_image callback.
163 * \public \memberof mlt_frame_s
164 * \param this a frame
165 * \param the get_image callback
166 * \return true if error
169 int mlt_frame_push_get_image( mlt_frame this, mlt_get_image get_image )
171 return mlt_deque_push_back( this->stack_image, get_image );
174 /** Pop a get_image callback.
176 * \public \memberof mlt_frame_s
177 * \param this a frame
178 * \return the get_image callback
181 mlt_get_image mlt_frame_pop_get_image( mlt_frame this )
183 return mlt_deque_pop_back( this->stack_image );
188 * \public \memberof mlt_frame_s
189 * \param this a frame
190 * \param that the frame to push onto \p this
191 * \return true if error
194 int mlt_frame_push_frame( mlt_frame this, mlt_frame that )
196 return mlt_deque_push_back( this->stack_image, that );
201 * \public \memberof mlt_frame_s
202 * \param this a frame
203 * \return a frame that was previously pushed
206 mlt_frame mlt_frame_pop_frame( mlt_frame this )
208 return mlt_deque_pop_back( this->stack_image );
213 * \public \memberof mlt_frame_s
214 * \param this a frame
215 * \param that an opaque pointer
216 * \return true if error
219 int mlt_frame_push_service( mlt_frame this, void *that )
221 return mlt_deque_push_back( this->stack_image, that );
226 * \public \memberof mlt_frame_s
227 * \param this a frame
228 * \return an opaque pointer to something previously pushed
231 void *mlt_frame_pop_service( mlt_frame this )
233 return mlt_deque_pop_back( this->stack_image );
238 * \public \memberof mlt_frame_s
239 * \param this a frame
240 * \param that an integer
241 * \return true if error
244 int mlt_frame_push_service_int( mlt_frame this, int that )
246 return mlt_deque_push_back_int( this->stack_image, that );
251 * \public \memberof mlt_frame_s
252 * \param this a frame
253 * \return an integer that was previously pushed
256 int mlt_frame_pop_service_int( mlt_frame this )
258 return mlt_deque_pop_back_int( this->stack_image );
261 /** Push an audio item on the stack.
263 * \public \memberof mlt_frame_s
264 * \param this a frame
265 * \param that an opaque pointer
266 * \return true if error
269 int mlt_frame_push_audio( mlt_frame this, void *that )
271 return mlt_deque_push_back( this->stack_audio, that );
274 /** Pop an audio item from the stack
276 * \public \memberof mlt_frame_s
277 * \param this a frame
278 * \return an opaque pointer to something that was pushed onto the frame's audio stack
281 void *mlt_frame_pop_audio( mlt_frame this )
283 return mlt_deque_pop_back( this->stack_audio );
286 /** Return the service stack
288 * \public \memberof mlt_frame_s
289 * \param this a frame
290 * \return the service stack
293 mlt_deque mlt_frame_service_stack( mlt_frame this )
295 return this->stack_service;
298 /** Replace image stack with the information provided.
300 * This might prove to be unreliable and restrictive - the idea is that a transition
301 * which normally uses two images may decide to only use the b frame (ie: in the case
302 * of a composite where the b frame completely obscures the a frame).
304 * The image must be writable and the destructor for the image itself must be taken
305 * care of on another frame and that frame cannot have a replace applied to it...
306 * Further it assumes that no alpha mask is in use.
308 * For these reasons, it can only be used in a specific situation - when you have
309 * multiple tracks each with their own transition and these transitions are applied
310 * in a strictly reversed order (ie: highest numbered [lowest track] is processed
313 * More reliable approach - the cases should be detected during the process phase
314 * and the upper tracks should simply not be invited to stack...
316 * \public \memberof mlt_frame_s
317 * \param this a frame
318 * \param image a new image
319 * \param format the image format
320 * \param width the width of the new image
321 * \param height the height of the new image
324 void mlt_frame_replace_image( mlt_frame this, uint8_t *image, mlt_image_format format, int width, int height )
326 // Remove all items from the stack
327 while( mlt_deque_pop_back( this->stack_image ) ) ;
329 // Update the information
330 mlt_properties_set_data( MLT_FRAME_PROPERTIES( this ), "image", image, 0, NULL, NULL );
331 mlt_properties_set_int( MLT_FRAME_PROPERTIES( this ), "width", width );
332 mlt_properties_set_int( MLT_FRAME_PROPERTIES( this ), "height", height );
333 mlt_properties_set_int( MLT_FRAME_PROPERTIES( this ), "format", format );
334 this->get_alpha_mask = NULL;
337 /** Get the short name for an image format.
339 * \public \memberof mlt_frame_s
340 * \param format the image format
344 const char * mlt_image_format_name( mlt_image_format format )
348 case mlt_image_none: return "none";
349 case mlt_image_rgb24: return "rgb24";
350 case mlt_image_rgb24a: return "rgb24a";
351 case mlt_image_yuv422: return "yuv422";
352 case mlt_image_yuv420p: return "yuv420p";
353 case mlt_image_opengl: return "opengl";
358 /** Get the image associated to the frame.
360 * You should express the desired format, width, and height as inputs. As long
361 * as the loader producer was used to generate this or the imageconvert filter
362 * was attached, then you will get the image back in the format you desire.
363 * However, you do not always get the width and height you request depending
364 * on properties and filters. You do not need to supply a pre-allocated
365 * buffer, but you should always supply the desired image format.
367 * \public \memberof mlt_frame_s
368 * \param this a frame
369 * \param[out] buffer an image buffer
370 * \param[in,out] format the image format
371 * \param[in,out] width the horizontal size in pixels
372 * \param[in,out] height the vertical size in pixels
373 * \param writable whether or not you will need to be able to write to the memory returned in \p buffer
374 * \return true if error
375 * \todo Better describe the width and height as inputs.
378 int mlt_frame_get_image( mlt_frame this, uint8_t **buffer, mlt_image_format *format, int *width, int *height, int writable )
380 mlt_properties properties = MLT_FRAME_PROPERTIES( this );
381 mlt_get_image get_image = mlt_frame_pop_get_image( this );
382 mlt_producer producer = mlt_properties_get_data( properties, "test_card_producer", NULL );
383 mlt_image_format requested_format = *format;
388 mlt_properties_set_int( properties, "image_count", mlt_properties_get_int( properties, "image_count" ) - 1 );
389 error = get_image( this, buffer, format, width, height, writable );
392 mlt_properties_set_int( properties, "width", *width );
393 mlt_properties_set_int( properties, "height", *height );
394 mlt_properties_set_int( properties, "format", *format );
395 if ( this->convert_image )
396 this->convert_image( this, buffer, format, requested_format );
400 // Cause the image to be loaded from test card or fallback (white) below.
401 mlt_frame_get_image( this, buffer, format, width, height, writable );
404 else if ( mlt_properties_get_data( properties, "image", NULL ) )
406 *format = mlt_properties_get_int( properties, "format" );
407 *buffer = mlt_properties_get_data( properties, "image", NULL );
408 *width = mlt_properties_get_int( properties, "width" );
409 *height = mlt_properties_get_int( properties, "height" );
410 if ( this->convert_image )
411 this->convert_image( this, buffer, format, requested_format );
415 mlt_frame test_frame = NULL;
416 mlt_service_get_frame( MLT_PRODUCER_SERVICE( producer ), &test_frame, 0 );
419 mlt_properties test_properties = MLT_FRAME_PROPERTIES( test_frame );
420 mlt_properties_set_double( test_properties, "consumer_aspect_ratio", mlt_properties_get_double( properties, "consumer_aspect_ratio" ) );
421 mlt_properties_set( test_properties, "rescale.interp", mlt_properties_get( properties, "rescale.interp" ) );
422 mlt_frame_get_image( test_frame, buffer, format, width, height, writable );
423 mlt_properties_set_data( properties, "test_card_frame", test_frame, 0, ( mlt_destructor )mlt_frame_close, NULL );
424 mlt_properties_set_double( properties, "aspect_ratio", mlt_frame_get_aspect_ratio( test_frame ) );
425 // mlt_properties_set_data( properties, "image", *buffer, *width * *height * 2, NULL, NULL );
426 // mlt_properties_set_int( properties, "width", *width );
427 // mlt_properties_set_int( properties, "height", *height );
428 // mlt_properties_set_int( properties, "format", *format );
432 mlt_properties_set_data( properties, "test_card_producer", NULL, 0, NULL, NULL );
433 mlt_frame_get_image( this, buffer, format, width, height, writable );
442 *width = *width == 0 ? 720 : *width;
443 *height = *height == 0 ? 576 : *height;
444 size = *width * *height;
446 mlt_properties_set_int( properties, "format", *format );
447 mlt_properties_set_int( properties, "width", *width );
448 mlt_properties_set_int( properties, "height", *height );
449 mlt_properties_set_int( properties, "aspect_ratio", 0 );
457 case mlt_image_rgb24:
460 *buffer = mlt_pool_alloc( size );
462 memset( *buffer, 255, size );
464 case mlt_image_rgb24a:
465 case mlt_image_opengl:
468 *buffer = mlt_pool_alloc( size );
470 memset( *buffer, 255, size );
472 case mlt_image_yuv422:
475 *buffer = mlt_pool_alloc( size );
478 while ( p != NULL && p != q )
484 case mlt_image_yuv420p:
486 *buffer = mlt_pool_alloc( size );
488 memset( *buffer, 255, size );
492 mlt_properties_set_data( properties, "image", *buffer, size, ( mlt_destructor )mlt_pool_release, NULL );
493 mlt_properties_set_int( properties, "test_image", 1 );
496 mlt_properties_set_int( properties, "scaled_width", *width );
497 mlt_properties_set_int( properties, "scaled_height", *height );
502 /** Get the alpha channel associated to the frame.
504 * \public \memberof mlt_frame_s
505 * \param this a frame
506 * \return the alpha channel
509 uint8_t *mlt_frame_get_alpha_mask( mlt_frame this )
511 uint8_t *alpha = NULL;
514 if ( this->get_alpha_mask != NULL )
515 alpha = this->get_alpha_mask( this );
517 alpha = mlt_properties_get_data( &this->parent, "alpha", NULL );
520 int size = mlt_properties_get_int( &this->parent, "scaled_width" ) * mlt_properties_get_int( &this->parent, "scaled_height" );
521 alpha = mlt_pool_alloc( size );
522 memset( alpha, 255, size );
523 mlt_properties_set_data( &this->parent, "alpha", alpha, size, mlt_pool_release, NULL );
529 /** Get the short name for an audio format.
531 * You do not need to deallocate the returned string.
532 * \public \memberof mlt_frame_s
533 * \param this a frame
534 * \param format an image format enum
535 * \return a string for the name of the image format
538 const char * mlt_audio_format_name( mlt_audio_format format )
542 case mlt_audio_none: return "none";
543 case mlt_audio_s16: return "s16";
544 case mlt_audio_s32: return "s32";
545 case mlt_audio_float: return "float";
550 /** Get the audio associated to the frame.
552 * You should express the desired format, frequency, channels, and samples as inputs. As long
553 * as the loader producer was used to generate this or the audioconvert filter
554 * was attached, then you will get the audio back in the format you desire.
555 * However, you do not always get the channels and samples you request depending
556 * on properties and filters. You do not need to supply a pre-allocated
557 * buffer, but you should always supply the desired audio format.
558 * The audio is always in interleaved format.
559 * You should use the \p mlt_sample_calculator to determine the number of samples you want.
561 * \public \memberof mlt_frame_s
562 * \param this a frame
563 * \param[out] buffer an audio buffer
564 * \param[in,out] format the audio format
565 * \param[in,out] frequency the sample rate
566 * \param[in,out] channels
567 * \param[in,out] samples the number of samples per frame
568 * \return true if error
571 int mlt_frame_get_audio( mlt_frame this, void **buffer, mlt_audio_format *format, int *frequency, int *channels, int *samples )
573 mlt_get_audio get_audio = mlt_frame_pop_audio( this );
574 mlt_properties properties = MLT_FRAME_PROPERTIES( this );
575 int hide = mlt_properties_get_int( properties, "test_audio" );
576 mlt_audio_format requested_format = *format;
578 if ( hide == 0 && get_audio != NULL )
580 get_audio( this, buffer, format, frequency, channels, samples );
581 mlt_properties_set_int( properties, "audio_frequency", *frequency );
582 mlt_properties_set_int( properties, "audio_channels", *channels );
583 mlt_properties_set_int( properties, "audio_samples", *samples );
584 mlt_properties_set_int( properties, "audio_format", *format );
585 if ( this->convert_audio )
586 this->convert_audio( this, buffer, format, requested_format );
588 else if ( mlt_properties_get_data( properties, "audio", NULL ) )
590 *buffer = mlt_properties_get_data( properties, "audio", NULL );
591 *format = mlt_properties_get_int( properties, "audio_format" );
592 *frequency = mlt_properties_get_int( properties, "audio_frequency" );
593 *channels = mlt_properties_get_int( properties, "audio_channels" );
594 *samples = mlt_properties_get_int( properties, "audio_samples" );
595 if ( this->convert_audio )
596 this->convert_audio( this, buffer, format, requested_format );
601 *samples = *samples <= 0 ? 1920 : *samples;
602 *channels = *channels <= 0 ? 2 : *channels;
603 *frequency = *frequency <= 0 ? 48000 : *frequency;
604 mlt_properties_set_int( properties, "audio_frequency", *frequency );
605 mlt_properties_set_int( properties, "audio_channels", *channels );
606 mlt_properties_set_int( properties, "audio_samples", *samples );
607 mlt_properties_set_int( properties, "audio_format", *format );
616 size = *samples * *channels * sizeof( int16_t );
619 size = *samples * *channels * sizeof( int32_t );
621 case mlt_audio_float:
622 size = *samples * *channels * sizeof( float );
626 *buffer = mlt_pool_alloc( size );
628 memset( *buffer, 0, size );
629 mlt_properties_set_data( properties, "audio", *buffer, size, ( mlt_destructor )mlt_pool_release, NULL );
630 mlt_properties_set_int( properties, "test_audio", 1 );
633 // TODO: This does not belong here
634 if ( *format == mlt_audio_s16 && mlt_properties_get( properties, "meta.volume" ) )
636 double value = mlt_properties_get_double( properties, "meta.volume" );
640 memset( *buffer, 0, *samples * *channels * 2 );
642 else if ( value != 1.0 )
644 int total = *samples * *channels;
645 int16_t *p = *buffer;
653 mlt_properties_set( properties, "meta.volume", NULL );
659 /** Set the audio on a frame.
661 * \public \memberof mlt_frame_s
662 * \param this a frame
663 * \param buffer an buffer containing audio samples
664 * \param format the format of the audio in the \p buffer
665 * \param size the total size of the buffer (optional)
666 * \param destructor a function that releases or deallocates the \p buffer
667 * \return true if error
670 int mlt_frame_set_audio( mlt_frame this, void *buffer, mlt_audio_format format, int size, mlt_destructor destructor )
672 mlt_properties_set_int( MLT_FRAME_PROPERTIES( this ), "audio_format", format );
673 return mlt_properties_set_data( MLT_FRAME_PROPERTIES( this ), "audio", buffer, size, destructor, NULL );
676 /** Get audio on a frame as a waveform image.
678 * This generates an 8-bit grayscale image representation of the audio in a
679 * frame. Currently, this only really works for 2 channels.
680 * This allocates the bitmap using mlt_pool so you should release the return
681 * value with \p mlt_pool_release.
683 * \public \memberof mlt_frame_s
684 * \param this a frame
685 * \param w the width of the image
686 * \param h the height of the image to create
687 * \return a pointer to a new bitmap
690 unsigned char *mlt_frame_get_waveform( mlt_frame this, int w, int h )
693 mlt_properties properties = MLT_FRAME_PROPERTIES( this );
694 mlt_audio_format format = mlt_audio_s16;
695 int frequency = 32000; // lower frequency available?
697 double fps = mlt_profile_fps( NULL );
698 int samples = mlt_sample_calculator( fps, frequency, mlt_frame_get_position( this ) );
701 mlt_frame_get_audio( this, (void**)&pcm, &format, &frequency, &channels, &samples );
703 // Make an 8-bit buffer large enough to hold rendering
705 unsigned char *bitmap = ( unsigned char* )mlt_pool_alloc( size );
706 if ( bitmap != NULL )
707 memset( bitmap, 0, size );
708 mlt_properties_set_data( properties, "waveform", bitmap, size, ( mlt_destructor )mlt_pool_release, NULL );
710 // Render vertical lines
711 int16_t *ubound = pcm + samples * channels;
712 int skip = samples / w - 1;
715 // Iterate sample stream and along x coordinate
716 for ( i = 0; i < w && pcm < ubound; i++ )
718 // pcm data has channels interleaved
719 for ( j = 0; j < channels; j++ )
721 // Determine sample's magnitude from 2s complement;
722 int pcm_magnitude = *pcm < 0 ? ~(*pcm) + 1 : *pcm;
723 // The height of a line is the ratio of the magnitude multiplied by
724 // half the vertical resolution
725 int height = ( int )( ( double )( pcm_magnitude ) / 32768 * h / 2 );
726 // Determine the starting y coordinate - left channel above center,
727 // right channel below - currently assumes 2 channels
728 int displacement = ( h / 2 ) - ( 1 - j ) * height;
729 // Position buffer pointer using y coordinate, stride, and x coordinate
730 unsigned char *p = &bitmap[ i + displacement * w ];
732 // Draw vertical line
733 for ( k = 0; k < height; k++ )
738 pcm += skip * channels;
744 /** Get the end service that produced this frame.
746 * This fetches the first producer of the frame and not any producers that
749 * \public \memberof mlt_frame_s
750 * \param this a frame
754 mlt_producer mlt_frame_get_original_producer( mlt_frame this )
757 return mlt_properties_get_data( MLT_FRAME_PROPERTIES( this ), "_producer", NULL );
761 /** Destroy the frame.
763 * \public \memberof mlt_frame_s
764 * \param this a frame
767 void mlt_frame_close( mlt_frame this )
769 if ( this != NULL && mlt_properties_dec_ref( MLT_FRAME_PROPERTIES( this ) ) <= 0 )
771 mlt_deque_close( this->stack_image );
772 mlt_deque_close( this->stack_audio );
773 while( mlt_deque_peek_back( this->stack_service ) )
774 mlt_service_close( mlt_deque_pop_back( this->stack_service ) );
775 mlt_deque_close( this->stack_service );
776 mlt_properties_close( &this->parent );
781 /***** convenience functions *****/
783 /** Determine the number of samples that belong in a frame at a time position.
785 * \public \memberof mlt_frame_s
786 * \param fps the frame rate
787 * \param frequency the sample rate
788 * \param position the time position
789 * \return the number of samples per channel
792 int mlt_sample_calculator( float fps, int frequency, int64_t position )
798 /* Compute the cumulative number of samples until the start of this frame and the
799 cumulative number of samples until the start of the next frame. Round each to the
800 nearest integer and take the difference to determine the number of samples in
803 This approach should prevent rounding errors that can accumulate over a large number
804 of frames causing A/V sync problems. */
806 int64_t samples_at_this =
807 (int64_t)( (double) position * (double) frequency / (double) fps +
808 ( position < 0 ? -0.5 : 0.5 ) );
809 int64_t samples_at_next =
810 (int64_t)( (double) (position + 1) * (double) frequency / (double) fps +
811 ( position < 0 ? -0.5 : 0.5 ) );
812 samples = (int)( samples_at_next - samples_at_this );
818 /** Determine the number of samples that belong before a time position.
820 * \public \memberof mlt_frame_s
821 * \param fps the frame rate
822 * \param frequency the sample rate
823 * \param position the time position
824 * \return the number of samples per channel
825 * \bug Will this break when mlt_position is converted to double?
828 int64_t mlt_sample_calculator_to_now( float fps, int frequency, int64_t position )
834 samples = (int64_t)( (double) position * (double) frequency / (double) fps +
835 ( position < 0 ? -0.5 : 0.5 ) );