2 * filter_movit_convert.cpp
3 * Copyright (C) 2013 Dan Dennedy <dan@dennedy.org>
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software Foundation,
17 * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
20 #include <framework/mlt.h>
26 #include "filter_glsl_manager.h"
27 #include <movit/effect_chain.h>
28 #include <movit/util.h>
29 #include "mlt_movit_input.h"
30 #include <mlt++/MltProducer.h>
31 #include "mlt_flip_effect.h"
33 using namespace movit;
35 static void set_movit_parameters( GlslChain *chain, mlt_service service, mlt_frame frame );
37 static void yuv422_to_yuv422p( uint8_t *yuv422, uint8_t *yuv422p, int width, int height )
40 uint8_t *U = Y + width * height;
41 uint8_t *V = U + width * height / 2;
42 int n = width * height / 2 + 1;
51 static int convert_on_cpu( mlt_frame frame, uint8_t **image, mlt_image_format *format, mlt_image_format output_format )
54 mlt_filter cpu_csc = (mlt_filter) mlt_properties_get_data( MLT_FRAME_PROPERTIES( frame ), "cpu_csc", NULL );
56 int (* save_fp )( mlt_frame self, uint8_t **image, mlt_image_format *input, mlt_image_format output )
57 = frame->convert_image;
58 frame->convert_image = NULL;
59 mlt_filter_process( cpu_csc, frame );
60 error = frame->convert_image( frame, image, format, output_format );
61 frame->convert_image = save_fp;
68 static void delete_chain( EffectChain* chain )
73 static void get_format_from_properties( mlt_properties properties, ImageFormat* image_format, YCbCrFormat* ycbcr_format )
75 switch ( mlt_properties_get_int( properties, "colorspace" ) ) {
77 ycbcr_format->luma_coefficients = YCBCR_REC_601;
81 ycbcr_format->luma_coefficients = YCBCR_REC_709;
85 switch ( mlt_properties_get_int( properties, "color_primaries" ) ) {
87 image_format->color_space = COLORSPACE_REC_601_625;
90 image_format->color_space = COLORSPACE_REC_601_525;
94 image_format->color_space = COLORSPACE_REC_709;
98 image_format->gamma_curve = GAMMA_REC_709;
100 if ( mlt_properties_get_int( properties, "force_full_luma" ) ) {
101 ycbcr_format->full_range = true;
103 ycbcr_format->full_range = ( mlt_properties_get_int( properties, "full_luma" ) == 1 );
106 // TODO: make new frame properties set by producers
107 ycbcr_format->cb_x_position = ycbcr_format->cr_x_position = 0.0f;
108 ycbcr_format->cb_y_position = ycbcr_format->cr_y_position = 0.5f;
111 static void build_fingerprint( mlt_service service, mlt_frame frame, std::string *fingerprint )
113 if ( service == (mlt_service) -1 ) {
114 fingerprint->append( "input" );
118 Effect* effect = GlslManager::get_effect( service, frame );
120 mlt_service input_a = GlslManager::get_effect_input( service, frame );
121 fingerprint->push_back( '(' );
122 build_fingerprint( input_a, frame, fingerprint );
123 fingerprint->push_back( ')' );
127 GlslManager::get_effect_secondary_input( service, frame, &input_b, &frame_b );
129 fingerprint->push_back( '(' );
130 build_fingerprint( input_b, frame_b, fingerprint );
131 fingerprint->push_back( ')' );
134 GlslManager::get_effect_third_input( service, frame, &input_b, &frame_b );
136 fingerprint->push_back( '(' );
137 build_fingerprint( input_b, frame_b, fingerprint );
138 fingerprint->push_back( ')' );
141 fingerprint->push_back( '(' );
142 fingerprint->append( mlt_properties_get( MLT_SERVICE_PROPERTIES( service ), "_unique_id" ) );
144 const char* effect_fingerprint = mlt_properties_get( MLT_SERVICE_PROPERTIES( service ), "_movit fingerprint" );
145 if ( effect_fingerprint ) {
146 fingerprint->push_back( '[' );
147 fingerprint->append( effect_fingerprint );
148 fingerprint->push_back( ']' );
151 bool disable = mlt_properties_get_int( MLT_SERVICE_PROPERTIES( service ), "_movit.parms.int.disable" );
153 fingerprint->push_back( 'd' );
155 fingerprint->push_back( ')' );
158 static Effect* build_movit_chain( mlt_service service, mlt_frame frame, GlslChain *chain )
160 if ( service == (mlt_service) -1 ) {
161 mlt_producer producer = mlt_producer_cut_parent( mlt_frame_get_original_producer( frame ) );
162 MltInput* input = GlslManager::get_input( producer, frame );
163 GlslManager::set_input( producer, frame, NULL );
164 chain->effect_chain->add_input( input->get_input() );
165 chain->inputs.insert(std::make_pair( producer, input ) );
166 return input->get_input();
169 Effect* effect = GlslManager::get_effect( service, frame );
171 GlslManager::set_effect( service, frame, NULL );
173 mlt_service input_a = GlslManager::get_effect_input( service, frame );
174 mlt_service input_b, input_c;
175 mlt_frame frame_b, frame_c;
176 GlslManager::get_effect_secondary_input( service, frame, &input_b, &frame_b );
177 GlslManager::get_effect_third_input( service, frame, &input_c, &frame_c );
178 Effect *effect_a = build_movit_chain( input_a, frame, chain );
180 if ( input_c && input_b ) {
181 Effect *effect_b = build_movit_chain( input_b, frame_b, chain );
182 Effect *effect_c = build_movit_chain( input_c, frame_c, chain );
183 chain->effect_chain->add_effect( effect, effect_a, effect_b, effect_c );
184 } else if ( input_b ) {
185 Effect *effect_b = build_movit_chain( input_b, frame_b, chain );
186 chain->effect_chain->add_effect( effect, effect_a, effect_b );
188 chain->effect_chain->add_effect( effect, effect_a );
191 chain->effects.insert(std::make_pair( service, effect ) );
195 static void dispose_movit_effects( mlt_service service, mlt_frame frame )
197 if ( service == (mlt_service) -1 ) {
198 mlt_producer producer = mlt_producer_cut_parent( mlt_frame_get_original_producer( frame ) );
199 delete GlslManager::get_input( producer, frame );
200 GlslManager::set_input( producer, frame, NULL );
204 delete GlslManager::get_effect( service, frame );
205 GlslManager::set_effect( service, frame, NULL );
207 mlt_service input_a = GlslManager::get_effect_input( service, frame );
210 GlslManager::get_effect_secondary_input( service, frame, &input_b, &frame_b );
211 dispose_movit_effects( input_a, frame );
214 dispose_movit_effects( input_b, frame_b );
216 GlslManager::get_effect_third_input( service, frame, &input_b, &frame_b );
218 dispose_movit_effects( input_b, frame_b );
222 static void finalize_movit_chain( mlt_service leaf_service, mlt_frame frame )
224 GlslChain* chain = GlslManager::get_chain( leaf_service );
226 std::string new_fingerprint;
227 build_fingerprint( leaf_service, frame, &new_fingerprint );
229 // Build the chain if needed.
230 if ( !chain || new_fingerprint != chain->fingerprint ) {
231 mlt_log_debug( leaf_service, "=== CREATING NEW CHAIN (old chain=%p, leaf=%p, fingerprint=%s) ===\n", chain, leaf_service, new_fingerprint.c_str() );
232 mlt_profile profile = mlt_service_profile( leaf_service );
233 chain = new GlslChain;
234 chain->effect_chain = new EffectChain(
235 profile->display_aspect_num,
236 profile->display_aspect_den,
237 GlslManager::get_instance()->get_resource_pool()
239 chain->fingerprint = new_fingerprint;
241 build_movit_chain( leaf_service, frame, chain );
242 set_movit_parameters( chain, leaf_service, frame );
243 chain->effect_chain->add_effect( new Mlt::VerticalFlip );
245 ImageFormat output_format;
246 output_format.color_space = COLORSPACE_sRGB;
247 output_format.gamma_curve = GAMMA_sRGB;
248 chain->effect_chain->add_output(output_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED);
249 chain->effect_chain->set_dither_bits(8);
250 chain->effect_chain->finalize();
252 GlslManager::set_chain( leaf_service, chain );
254 // Delete all the created Effect instances to avoid memory leaks.
255 dispose_movit_effects( leaf_service, frame );
259 static void set_movit_parameters( GlslChain *chain, mlt_service service, mlt_frame frame )
261 if ( service == (mlt_service) -1 ) {
262 mlt_producer producer = mlt_producer_cut_parent( mlt_frame_get_original_producer( frame ) );
263 MltInput* input = chain->inputs[ producer ];
264 input->set_pixel_data( GlslManager::get_input_pixel_pointer( producer, frame ) );
268 Effect* effect = chain->effects[ service ];
269 mlt_service input_a = GlslManager::get_effect_input( service, frame );
270 set_movit_parameters( chain, input_a, frame );
274 GlslManager::get_effect_secondary_input( service, frame, &input_b, &frame_b );
276 set_movit_parameters( chain, input_b, frame_b );
278 GlslManager::get_effect_third_input( service, frame, &input_b, &frame_b );
280 set_movit_parameters( chain, input_b, frame_b );
283 mlt_properties properties = MLT_SERVICE_PROPERTIES( service );
284 int count = mlt_properties_count( properties );
285 for (int i = 0; i < count; ++i) {
286 const char *name = mlt_properties_get_name( properties, i );
287 if (strncmp(name, "_movit.parms.float.", strlen("_movit.parms.float.")) == 0 &&
288 mlt_properties_get_value( properties, i )) {
289 bool ok = effect->set_float(name + strlen("_movit.parms.float."),
290 mlt_properties_get_double( properties, name ));
293 if (strncmp(name, "_movit.parms.int.", strlen("_movit.parms.int.")) == 0 &&
294 mlt_properties_get_value( properties, i )) {
295 bool ok = effect->set_int(name + strlen("_movit.parms.int."),
296 mlt_properties_get_int( properties, name ));
299 if (strncmp(name, "_movit.parms.vec3.", strlen("_movit.parms.vec3.")) == 0 &&
300 strcmp(name + strlen(name) - 3, "[0]") == 0 &&
301 mlt_properties_get_value( properties, i )) {
303 char *name_copy = strdup(name);
304 char *index_char = name_copy + strlen(name_copy) - 2;
305 val[0] = mlt_properties_get_double( properties, name_copy );
307 val[1] = mlt_properties_get_double( properties, name_copy );
309 val[2] = mlt_properties_get_double( properties, name_copy );
310 index_char[-1] = '\0';
311 bool ok = effect->set_vec3(name_copy + strlen("_movit.parms.vec3."), val);
315 if (strncmp(name, "_movit.parms.vec4.", strlen("_movit.parms.vec4.")) == 0 &&
316 strcmp(name + strlen(name) - 3, "[0]") == 0 &&
317 mlt_properties_get_value( properties, i )) {
319 char *name_copy = strdup(name);
320 char *index_char = name_copy + strlen(name_copy) - 2;
321 val[0] = mlt_properties_get_double( properties, name_copy );
323 val[1] = mlt_properties_get_double( properties, name_copy );
325 val[2] = mlt_properties_get_double( properties, name_copy );
327 val[3] = mlt_properties_get_double( properties, name_copy );
328 index_char[-1] = '\0';
329 bool ok = effect->set_vec4(name_copy + strlen("_movit.parms.vec4."), val);
336 static void dispose_pixel_pointers( GlslChain *chain, mlt_service service, mlt_frame frame )
338 if ( service == (mlt_service) -1 ) {
339 mlt_producer producer = mlt_producer_cut_parent( mlt_frame_get_original_producer( frame ) );
340 MltInput* input = chain->inputs[ producer ];
341 input->invalidate_pixel_data();
342 mlt_pool_release( GlslManager::get_input_pixel_pointer( producer, frame ) );
346 mlt_service input_a = GlslManager::get_effect_input( service, frame );
347 dispose_pixel_pointers( chain, input_a, frame );
351 GlslManager::get_effect_secondary_input( service, frame, &input_b, &frame_b );
353 dispose_pixel_pointers( chain, input_b, frame_b );
355 GlslManager::get_effect_third_input( service, frame, &input_b, &frame_b );
357 dispose_pixel_pointers( chain, input_b, frame_b );
361 static int movit_render( EffectChain *chain, mlt_frame frame, mlt_image_format *format, mlt_image_format output_format, int width, int height, uint8_t **image )
363 GlslManager* glsl = GlslManager::get_instance();
365 if ( output_format == mlt_image_glsl_texture ) {
366 error = glsl->render_frame_texture( chain, frame, width, height, image );
369 error = glsl->render_frame_rgba( chain, frame, width, height, image );
370 if ( !error && output_format != mlt_image_rgb24a ) {
371 *format = mlt_image_rgb24a;
372 error = convert_on_cpu( frame, image, format, output_format );
378 // Create an MltInput for an image with the given format and dimensions.
379 static MltInput* create_input( mlt_properties properties, mlt_image_format format, int aspect_width, int aspect_height, int width, int height )
381 MltInput* input = new MltInput();
382 if ( format == mlt_image_rgb24a || format == mlt_image_opengl ) {
383 // TODO: Get the color space if available.
384 input->useFlatInput( FORMAT_RGBA_POSTMULTIPLIED_ALPHA, width, height );
386 else if ( format == mlt_image_rgb24 ) {
387 // TODO: Get the color space if available.
388 input->useFlatInput( FORMAT_RGB, width, height );
390 else if ( format == mlt_image_yuv420p ) {
391 ImageFormat image_format;
392 YCbCrFormat ycbcr_format;
393 get_format_from_properties( properties, &image_format, &ycbcr_format );
394 ycbcr_format.chroma_subsampling_x = ycbcr_format.chroma_subsampling_y = 2;
395 input->useYCbCrInput( image_format, ycbcr_format, width, height );
397 else if ( format == mlt_image_yuv422 ) {
398 ImageFormat image_format;
399 YCbCrFormat ycbcr_format;
400 get_format_from_properties( properties, &image_format, &ycbcr_format );
401 ycbcr_format.chroma_subsampling_x = 2;
402 ycbcr_format.chroma_subsampling_y = 1;
403 input->useYCbCrInput( image_format, ycbcr_format, width, height );
408 // Make a copy of the given image (allocated using mlt_pool_alloc) suitable
409 // to pass as pixel pointer to an MltInput (created using create_input
410 // with the same parameters), and return that pointer.
411 static uint8_t* make_input_copy( mlt_image_format format, uint8_t *image, int width, int height )
413 int img_size = mlt_image_format_size( format, width, height, NULL );
414 uint8_t* img_copy = (uint8_t*) mlt_pool_alloc( img_size );
415 if ( format == mlt_image_yuv422 ) {
416 yuv422_to_yuv422p( image, img_copy, width, height );
418 memcpy( img_copy, image, img_size );
423 static int convert_image( mlt_frame frame, uint8_t **image, mlt_image_format *format, mlt_image_format output_format )
426 if ( *format == output_format )
429 mlt_properties properties = MLT_FRAME_PROPERTIES( frame );
431 mlt_log_debug( NULL, "filter_movit_convert: %s -> %s (%d)\n",
432 mlt_image_format_name( *format ), mlt_image_format_name( output_format ),
433 mlt_frame_get_position( frame ) );
435 // Use CPU if glsl not initialized or not supported.
436 GlslManager* glsl = GlslManager::get_instance();
437 if ( !glsl || !glsl->get_int("glsl_supported" ) )
438 return convert_on_cpu( frame, image, format, output_format );
440 // Do non-GL image conversions on a CPU-based image converter.
441 if ( *format != mlt_image_glsl && output_format != mlt_image_glsl && output_format != mlt_image_glsl_texture )
442 return convert_on_cpu( frame, image, format, output_format );
445 int width = mlt_properties_get_int( properties, "width" );
446 int height = mlt_properties_get_int( properties, "height" );
447 GlslManager::get_instance()->lock_service( frame );
449 // If we're at the beginning of a series of Movit effects, store the input
450 // sent into the chain.
451 if ( output_format == mlt_image_glsl ) {
452 mlt_producer producer = mlt_producer_cut_parent( mlt_frame_get_original_producer( frame ) );
453 mlt_profile profile = mlt_service_profile( MLT_PRODUCER_SERVICE( producer ) );
454 MltInput *input = create_input( properties, *format, profile->width, profile->height, width, height );
455 GlslManager::set_input( producer, frame, input );
456 uint8_t *img_copy = make_input_copy( *format, *image, width, height );
457 GlslManager::set_input_pixel_pointer( producer, frame, img_copy );
459 *image = (uint8_t *) -1;
460 mlt_frame_set_image( frame, *image, 0, NULL );
463 // If we're at the _end_ of a series of Movit effects, render the chain.
464 if ( *format == mlt_image_glsl ) {
465 mlt_service leaf_service = (mlt_service) *image;
467 // Construct the chain unless we already have a good one.
468 finalize_movit_chain( leaf_service, frame );
470 // Set per-frame parameters now that we know which Effect instances to set them on.
471 // (finalize_movit_chain may already have done this, though, but twice doesn't hurt.)
472 GlslChain *chain = GlslManager::get_chain( leaf_service );
473 set_movit_parameters( chain, leaf_service, frame );
475 error = movit_render( chain->effect_chain, frame, format, output_format, width, height, image );
477 dispose_pixel_pointers( chain, leaf_service, frame );
480 // If we've been asked to render some frame directly to a texture (without any
481 // effects in-between), we create a new mini-chain to do so.
482 if ( *format != mlt_image_glsl && output_format == mlt_image_glsl_texture ) {
483 // We might already have a texture from a previous conversion from mlt_image_glsl.
484 glsl_texture texture = (glsl_texture) mlt_properties_get_data( properties, "movit.convert.texture", NULL );
485 // XXX: requires a special property set on the frame by the app for now
486 // because we do not have reliable way to clear the texture property
487 // when a downstream filter has changed image.
488 if ( texture && mlt_properties_get_int( properties, "movit.convert.use_texture") ) {
489 *image = (uint8_t*) &texture->texture;
490 mlt_frame_set_image( frame, *image, 0, NULL );
492 // Use a separate chain to convert image in RAM to OpenGL texture.
493 // Use cached chain if available and compatible.
494 Mlt::Producer producer( mlt_producer_cut_parent( mlt_frame_get_original_producer( frame ) ) );
495 EffectChain *chain = (EffectChain*) producer.get_data( "movit.convert.chain" );
496 MltInput *input = (MltInput*) producer.get_data( "movit.convert.input" );
497 int w = producer.get_int( "movit.convert.width" );
498 int h = producer.get_int( "movit.convert.height" );
499 mlt_image_format f = (mlt_image_format) producer.get_int( "movit.convert.format" );
500 if ( !chain || !input || width != w || height != h || *format != f ) {
501 chain = new EffectChain( width, height, GlslManager::get_instance()->get_resource_pool() );
502 input = create_input( properties, *format, width, height, width, height );
503 chain->add_input( input->get_input() );
504 chain->add_effect( new Mlt::VerticalFlip() );
505 ImageFormat movit_output_format;
506 movit_output_format.color_space = COLORSPACE_sRGB;
507 movit_output_format.gamma_curve = GAMMA_sRGB;
508 chain->add_output(movit_output_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED);
509 chain->set_dither_bits(8);
511 producer.set( "movit.convert.chain", chain, 0, (mlt_destructor) delete_chain );
512 producer.set( "movit.convert.input", input, 0, NULL );
513 producer.set( "movit.convert.width", width );
514 producer.set( "movit.convert.height", height );
515 producer.set( "movit.convert.format", *format );
518 if ( *format == mlt_image_yuv422 ) {
519 // We need to convert to planar, which make_input_copy() will do for us.
520 uint8_t *planar = make_input_copy( *format, *image, width, height );
521 input->set_pixel_data( planar );
522 error = movit_render( chain, frame, format, output_format, width, height, image );
523 mlt_pool_release( planar );
525 input->set_pixel_data( *image );
526 error = movit_render( chain, frame, format, output_format, width, height, image );
531 GlslManager::get_instance()->unlock_service( frame );
533 mlt_properties_set_int( properties, "format", output_format );
534 *format = output_format;
539 static mlt_frame process( mlt_filter filter, mlt_frame frame )
541 // Set a default colorspace on the frame if not yet set by the producer.
542 // The producer may still change it during get_image.
543 // This way we do not have to modify each producer to set a valid colorspace.
544 mlt_properties properties = MLT_FRAME_PROPERTIES(frame);
545 if ( mlt_properties_get_int( properties, "colorspace" ) <= 0 )
546 mlt_properties_set_int( properties, "colorspace", mlt_service_profile( MLT_FILTER_SERVICE(filter) )->colorspace );
548 frame->convert_image = convert_image;
550 mlt_filter cpu_csc = (mlt_filter) mlt_properties_get_data( MLT_FILTER_PROPERTIES( filter ), "cpu_csc", NULL );
551 mlt_properties_inc_ref( MLT_FILTER_PROPERTIES(cpu_csc) );
552 mlt_properties_set_data( properties, "cpu_csc", cpu_csc, 0,
553 (mlt_destructor) mlt_filter_close, NULL );
558 static mlt_filter create_filter( mlt_profile profile, const char *effect )
561 char *id = strdup( effect );
562 char *arg = strchr( id, ':' );
566 // The swscale and avcolor_space filters require resolution as arg to test compatibility
567 if ( !strcmp( effect, "avcolor_space" ) )
568 filter = mlt_factory_filter( profile, id, &profile->width );
570 filter = mlt_factory_filter( profile, id, arg );
572 mlt_properties_set_int( MLT_FILTER_PROPERTIES( filter ), "_loader", 1 );
579 mlt_filter filter_movit_convert_init( mlt_profile profile, mlt_service_type type, const char *id, char *arg )
581 mlt_filter filter = NULL;
582 GlslManager* glsl = GlslManager::get_instance();
584 if ( glsl && ( filter = mlt_filter_new() ) )
586 mlt_properties properties = MLT_FILTER_PROPERTIES( filter );
587 glsl->add_ref( properties );
589 // XXX avcolor_space is crashing on Windows in this context!
590 mlt_filter cpu_csc = NULL;
592 mlt_filter cpu_csc = create_filter( profile, "avcolor_space" );
595 cpu_csc = create_filter( profile, "imageconvert" );
597 mlt_properties_set_data( MLT_FILTER_PROPERTIES( filter ), "cpu_csc", cpu_csc, 0,
598 (mlt_destructor) mlt_filter_close, NULL );
599 filter->process = process;