From: Brian Matherly Date: Sat, 18 Jan 2014 04:21:10 +0000 (-0600) Subject: Add support for more image formats to vid.stab X-Git-Url: https://git.sesse.net/?a=commitdiff_plain;ds=sidebyside;h=868437163988b81b721567ed571ff37d0c64d717;p=mlt Add support for more image formats to vid.stab --- diff --git a/src/framework/mlt_frame.h b/src/framework/mlt_frame.h index ae45e7be..9d006023 100644 --- a/src/framework/mlt_frame.h +++ b/src/framework/mlt_frame.h @@ -157,4 +157,13 @@ extern void mlt_frame_write_ppm( mlt_frame frame ); u = ((-152*r - 300*g + 450*b) >> 10) + 128;\ v = ((450*r - 377*g - 73*b) >> 10) + 128; +/** This macro scales YUV up into the full gamut of the RGB color space. */ +#define YUV2RGB_601_SCALED( y, u, v, r, g, b ) \ + r = ((1192 * ( y - 16 ) + 1634 * ( v - 128 ) ) >> 10 ); \ + g = ((1192 * ( y - 16 ) - 832 * ( v - 128 ) - 401 * ( u - 128 ) ) >> 10 ); \ + b = ((1192 * ( y - 16 ) + 2066 * ( u - 128 ) ) >> 10 ); \ + r = r < 0 ? 0 : r > 255 ? 255 : r; \ + g = g < 0 ? 0 : g > 255 ? 255 : g; \ + b = b < 0 ? 0 : b > 255 ? 255 : b; + #endif diff --git a/src/modules/core/filter_imageconvert.c b/src/modules/core/filter_imageconvert.c index 33c93880..20b13288 100644 --- a/src/modules/core/filter_imageconvert.c +++ b/src/modules/core/filter_imageconvert.c @@ -37,15 +37,6 @@ u = u > 240 ? 240 : u;\ v = v > 240 ? 240 : v -/** This macro scales YUV up into the full gamut of the RGB color space. */ -#define YUV2RGB_601_SCALED( y, u, v, r, g, b ) \ - r = ((1192 * ( y - 16 ) + 1634 * ( v - 128 ) ) >> 10 ); \ - g = ((1192 * ( y - 16 ) - 832 * ( v - 128 ) - 401 * ( u - 128 ) ) >> 10 ); \ - b = ((1192 * ( y - 16 ) + 2066 * ( u - 128 ) ) >> 10 ); \ - r = r < 0 ? 0 : r > 255 ? 255 : r; \ - g = g < 0 ? 0 : g > 255 ? 255 : g; \ - b = b < 0 ? 0 : b > 255 ? 255 : b; - /** This macro converts a YUV value to the RGB color space. */ #define YUV2RGB_601_UNSCALED( y, u, v, r, g, b ) \ r = ((1024 * y + 1404 * ( v - 128 ) ) >> 10 ); \ diff --git a/src/modules/vid.stab/Makefile b/src/modules/vid.stab/Makefile index f9991b23..7f829f84 100644 --- a/src/modules/vid.stab/Makefile +++ b/src/modules/vid.stab/Makefile @@ -6,7 +6,8 @@ include ../../../config.mak TARGET = ../libmltvidstab$(LIBSUF) -OBJS = factory.o +OBJS = factory.o \ + common.o CPPOBJS = filter_deshake.o CPPOBJS += filter_vidstab.o diff --git a/src/modules/vid.stab/common.c b/src/modules/vid.stab/common.c new file mode 100644 index 00000000..13acc67f --- /dev/null +++ b/src/modules/vid.stab/common.c @@ -0,0 +1,156 @@ +/* + * common.c + * Copyright (C) 2014 Brian Matherly + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +#include +#include +#include "common.h" + +mlt_image_format validate_format( mlt_image_format format ) +{ + switch( format ) + { + case mlt_image_rgb24a: + case mlt_image_rgb24: + return mlt_image_rgb24; + case mlt_image_yuv420p: + return mlt_image_yuv420p; + default: + case mlt_image_none: + case mlt_image_yuv422: + case mlt_image_opengl: + case mlt_image_glsl: + case mlt_image_glsl_texture: + return mlt_image_yuv422; + } +} + +VSPixelFormat mltimage_to_vsimage( mlt_image_format mlt_format, int width, int height, uint8_t* mlt_img, uint8_t** vs_img ) +{ + switch( mlt_format ) + { + case mlt_image_rgb24: + // Convert RGB24 to YUV444 because it is the only planar + // format with comparable bit depth. + { + *vs_img = mlt_pool_alloc( width * height * 3 ); + int y, u, v, r, g, b; + int total = width * height + 1; + uint8_t* yp = *vs_img; + uint8_t* up = yp + ( width * height ); + uint8_t* vp = up + ( width * height ); + + while( --total ) + { + r = *mlt_img++; + g = *mlt_img++; + b = *mlt_img++; + RGB2YUV_601_SCALED(r, g, b, y, u, v); + *yp++ = y; + *up++ = u; + *vp++ = v; + } + + return PF_YUV444P; + } + case mlt_image_yuv420p: + // This format maps with no conversion + { + *vs_img = mlt_img; + return PF_YUV420P; + } + case mlt_image_yuv422: + // Convert packed to planar + { + *vs_img = mlt_pool_alloc( width * height * 2 ); + uint8_t* yp = *vs_img; + uint8_t* up = yp + ( width * height ); + uint8_t* vp = up + ( width * height / 2 ); + int total = ( width * height / 2 ) + 1; + + while( --total ) + { + *yp++ = *mlt_img++; + *up++ = *mlt_img++; + *yp++ = *mlt_img++; + *vp++ = *mlt_img++; + } + + return PF_YUV422P; + } + default: + return PF_NONE; + } +} + +void vsimage_to_mltimage( uint8_t* vs_img, uint8_t* mlt_img, mlt_image_format mlt_format, int width, int height ) +{ + switch( mlt_format ) + { + case mlt_image_rgb24: + // Convert YUV444 to RGB24. + { + int y, u, v, r, g, b; + int total = width * height + 1; + uint8_t* yp = vs_img; + uint8_t* up = yp + ( width * height ); + uint8_t* vp = up + ( width * height ); + + while( --total ) + { + y = *yp++; + u = *up++; + v = *vp++; + YUV2RGB_601_SCALED( y, u, v, r, g, b ); + *mlt_img++ = r; + *mlt_img++ = g; + *mlt_img++ = b; + } + } + break; + case mlt_image_yuv420p: + // This format was never converted + break; + case mlt_image_yuv422: + // Convert planar to packed + { + uint8_t* yp = vs_img; + uint8_t* up = yp + ( width * height ); + uint8_t* vp = up + ( width * height / 2 ); + int total = ( width * height / 2 ) + 1; + + while( --total ) + { + *mlt_img++ = *yp++; + *mlt_img++ = *up++; + *mlt_img++ = *yp++; + *mlt_img++ = *vp++; + } + } + break; + default: + return PF_NONE; + } +} + +void free_vsimage( uint8_t* vs_img, VSPixelFormat format ) +{ + if( format != PF_YUV420P ) + { + mlt_pool_release( vs_img ); + } +} diff --git a/src/modules/vid.stab/common.h b/src/modules/vid.stab/common.h index c9a060bf..de081f43 100644 --- a/src/modules/vid.stab/common.h +++ b/src/modules/vid.stab/common.h @@ -1,6 +1,7 @@ /* * common.h * Copyright (C) 2013 Jakub Ksiezniak + * Copyright (C) 2014 Brian Matherly * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -20,22 +21,12 @@ #ifndef VIDSTAB_COMMON_H_ #define VIDSTAB_COMMON_H_ -extern "C" { #include #include -} -inline VSPixelFormat convertImageFormat(mlt_image_format &format) { - switch (format) { - case mlt_image_rgb24: - return PF_RGB24; - case mlt_image_rgb24a: - return PF_RGBA; - case mlt_image_yuv420p: - return PF_YUV420P; - default: - return PF_NONE; - } -} +mlt_image_format validate_format( mlt_image_format format ); +VSPixelFormat mltimage_to_vsimage( mlt_image_format mlt_format, int width, int height, uint8_t* mlt_img, uint8_t** vs_img ); +void vsimage_to_mltimage( uint8_t* vs_img, uint8_t* mlt_img, mlt_image_format mlt_format, int width, int height ); +void free_vsimage( uint8_t* vs_img, VSPixelFormat format ); #endif /* VIDSTAB_COMMON_H_ */ diff --git a/src/modules/vid.stab/filter_deshake.cpp b/src/modules/vid.stab/filter_deshake.cpp index 0dd13cc2..c643a877 100644 --- a/src/modules/vid.stab/filter_deshake.cpp +++ b/src/modules/vid.stab/filter_deshake.cpp @@ -2,6 +2,7 @@ * filter_deshake.cpp * Copyright (C) 2013 Marco Gittler * Copyright (C) 2013 Jakub Ksiezniak + * Copyright (C) 2014 Brian Matherly * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -21,12 +22,12 @@ extern "C" { #include +#include "common.h" } #include #include #include -#include "common.h" #define FILTER_NAME "vid.stab.deshake" @@ -41,12 +42,11 @@ typedef struct _deshake_data } DeshakeData; int init_deshake(DeshakeData *data, mlt_properties properties, - mlt_image_format *format, int *width, int *height, char* interps) + VSPixelFormat vs_format, int *width, int *height, char* interps) { - VSPixelFormat pf = convertImageFormat(*format); VSFrameInfo fiIn, fiOut; - vsFrameInfoInit(&fiIn, *width, *height, pf); - vsFrameInfoInit(&fiOut, *width, *height, pf); + vsFrameInfoInit(&fiIn, *width, *height, vs_format); + vsFrameInfoInit(&fiOut, *width, *height, vs_format); VSMotionDetectConfig conf = vsMotionDetectGetDefaultConfig(FILTER_NAME); conf.shakiness = mlt_properties_get_int(properties, "shakiness"); @@ -96,12 +96,24 @@ static int get_image(mlt_frame frame, uint8_t **image, mlt_image_format *format, { mlt_filter filter = (mlt_filter) mlt_frame_pop_service(frame); mlt_properties properties = MLT_FILTER_PROPERTIES(filter); + uint8_t* vs_image = NULL; + VSPixelFormat vs_format = PF_NONE; + + // VS only works on progressive frames + mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "consumer_deinterlace", 1 ); - *format = mlt_image_yuv420p; + *format = validate_format( *format ); DeshakeData *data = static_cast(filter->child); int error = mlt_frame_get_image(frame, image, format, width, height, 1); - if (!error) + + // Convert the received image to a format vid.stab can handle + if ( !error ) + { + vs_format = mltimage_to_vsimage( *format, *width, *height, *image, &vs_image ); + } + + if ( vs_image ) { // Service locks are for concurrency control mlt_service_lock(MLT_FILTER_SERVICE(filter)); @@ -125,7 +137,7 @@ static int get_image(mlt_frame frame, uint8_t **image, mlt_image_format *format, if (!data->initialized) { char *interps = mlt_properties_get(MLT_FRAME_PROPERTIES(frame), "rescale.interp"); - init_deshake(data, properties, format, width, height, + init_deshake(data, properties, vs_format, width, height, interps); data->initialized = true; } @@ -136,7 +148,7 @@ static int get_image(mlt_frame frame, uint8_t **image, mlt_image_format *format, VSTransform motion; VSFrame vsFrame; - vsFrameFillFromBuffer(&vsFrame, *image, &md->fi); + vsFrameFillFromBuffer(&vsFrame, vs_image, &md->fi); vsMotionDetection(md, &localmotions, &vsFrame); motion = vsSimpleMotionsToTransform(md->fi, FILTER_NAME, &localmotions); @@ -151,7 +163,11 @@ static int get_image(mlt_frame frame, uint8_t **image, mlt_image_format *format, vsDoTransform(td, t); vsTransformFinish(td); + vsimage_to_mltimage( vs_image, *image, *format, *width, *height ); + mlt_service_unlock(MLT_FILTER_SERVICE(filter)); + + free_vsimage( vs_image, vs_format ); } return error; diff --git a/src/modules/vid.stab/filter_vidstab.cpp b/src/modules/vid.stab/filter_vidstab.cpp index 4b4fe46f..a6d82943 100644 --- a/src/modules/vid.stab/filter_vidstab.cpp +++ b/src/modules/vid.stab/filter_vidstab.cpp @@ -2,6 +2,7 @@ * filter_vidstab.cpp * Copyright (C) 2013 Marco Gittler * Copyright (C) 2013 Jakub Ksiezniak + * Copyright (C) 2014 Brian Matherly * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -23,12 +24,12 @@ extern "C" #include #include #include +#include "common.h" } #include #include #include -#include "common.h" typedef struct { @@ -56,7 +57,11 @@ static void free_manylocalmotions( VSManyLocalMotions* mlms ) for( int i = 0; i < vs_vector_size( mlms ); i++ ) { LocalMotions* lms = (LocalMotions*)vs_vector_get( mlms, i ); - vs_vector_del( lms ); + + if( lms ) + { + vs_vector_del( lms ); + } } vs_vector_del( mlms ); } @@ -159,12 +164,11 @@ static void read_manylocalmotions( mlt_properties properties, VSManyLocalMotions mlt_animation_close( animation ); } -static vs_apply* init_apply_data( mlt_filter filter, mlt_frame frame, int width, int height, mlt_image_format format ) +static vs_apply* init_apply_data( mlt_filter filter, mlt_frame frame, VSPixelFormat vs_format, int width, int height ) { mlt_properties properties = MLT_FILTER_PROPERTIES( filter ); vs_apply* apply_data = (vs_apply*)calloc( 1, sizeof(vs_apply) ); memset( apply_data, 0, sizeof( vs_apply ) ); - VSPixelFormat pf = convertImageFormat( format ); const char* filterName = mlt_properties_get( properties, "mlt_service" ); VSTransformConfig conf = vsTransformGetDefaultConfig( filterName ); @@ -201,8 +205,8 @@ static vs_apply* init_apply_data( mlt_filter filter, mlt_frame frame, int width, VSTransformData* td = &apply_data->td; VSTransformations* trans = &apply_data->trans; VSFrameInfo fi_src, fi_dst; - vsFrameInfoInit( &fi_src, width, height, pf ); - vsFrameInfoInit( &fi_dst, width, height, pf ); + vsFrameInfoInit( &fi_src, width, height, vs_format ); + vsFrameInfoInit( &fi_dst, width, height, vs_format ); vsTransformDataInit( td, &conf, &fi_src, &fi_dst ); vsTransformationsInit( trans ); vsLocalmotions2Transforms( td, &mlms, trans ); @@ -223,7 +227,7 @@ static void destory_apply_data( vs_apply* apply_data ) } } -static vs_analyze* init_analyze_data( mlt_filter filter, mlt_frame frame, mlt_image_format format, int width, int height ) +static vs_analyze* init_analyze_data( mlt_filter filter, mlt_frame frame, VSPixelFormat vs_format, int width, int height ) { mlt_properties properties = MLT_FILTER_PROPERTIES( filter ); vs_analyze* analyze_data = (vs_analyze*)calloc( 1, sizeof(vs_analyze) ); @@ -234,9 +238,8 @@ static vs_analyze* init_analyze_data( mlt_filter filter, mlt_frame frame, mlt_im vs_vector_init( &analyze_data->mlms, mlt_filter_get_length2( filter, frame ) ); // Initialize a VSFrameInfo to be used below - VSPixelFormat pf = convertImageFormat( format ); VSFrameInfo fi; - vsFrameInfoInit( &fi, width, height, pf ); + vsFrameInfoInit( &fi, width, height, vs_format ); // Initialize a VSMotionDetect const char* filterName = mlt_properties_get( properties, "mlt_service" ); @@ -263,106 +266,112 @@ void destory_analyze_data( vs_analyze* analyze_data ) } } -static int get_image_and_apply( mlt_filter filter, mlt_frame frame, uint8_t **image, mlt_image_format *format, int *width, int *height, int writable ) +static int transform_image( mlt_filter filter, mlt_frame frame, uint8_t* vs_image, VSPixelFormat vs_format, int width, int height ) { int error = 0; mlt_properties properties = MLT_FILTER_PROPERTIES( filter ); vs_data* data = (vs_data*)filter->child; - *format = mlt_image_yuv420p; - - error = mlt_frame_get_image( frame, image, format, width, height, 1 ); - if ( !error ) + // Handle signal from app to re-init data + if ( mlt_properties_get_int(properties, "refresh") ) { - mlt_service_lock( MLT_FILTER_SERVICE( filter ) ); - - // Handle signal from app to re-init data - if ( mlt_properties_get_int(properties, "refresh") ) - { - mlt_properties_set(properties, "refresh", NULL); - destory_apply_data( data->apply_data ); - data->apply_data = NULL; - } - - // Init transform data if necessary (first time) - if ( !data->apply_data ) - { - data->apply_data = init_apply_data( filter, frame, *width, *height, *format ); - } + mlt_properties_set(properties, "refresh", NULL); + destory_apply_data( data->apply_data ); + data->apply_data = NULL; + } - // Apply transformations to this image - VSTransformData* td = &data->apply_data->td; - VSTransformations* trans = &data->apply_data->trans; - VSFrame vsFrame; - vsFrameFillFromBuffer( &vsFrame, *image, vsTransformGetSrcFrameInfo( td ) ); - trans->current = mlt_filter_get_position( filter, frame ); - vsTransformPrepare( td, &vsFrame, &vsFrame ); - VSTransform t = vsGetNextTransform( td, trans ); - vsDoTransform( td, t ); - vsTransformFinish( td ); - - mlt_service_unlock( MLT_FILTER_SERVICE( filter ) ); + // Init transform data if necessary (first time) + if ( !data->apply_data ) + { + data->apply_data = init_apply_data( filter, frame, vs_format, width, height ); } + // Apply transformations to this image + VSTransformData* td = &data->apply_data->td; + VSTransformations* trans = &data->apply_data->trans; + VSFrame vsFrame; + vsFrameFillFromBuffer( &vsFrame, vs_image, vsTransformGetSrcFrameInfo( td ) ); + trans->current = mlt_filter_get_position( filter, frame ); + vsTransformPrepare( td, &vsFrame, &vsFrame ); + VSTransform t = vsGetNextTransform( td, trans ); + vsDoTransform( td, t ); + vsTransformFinish( td ); + return error; } - -static int get_image_and_analyze( mlt_filter filter, mlt_frame frame, uint8_t **image, mlt_image_format *format, int *width, int *height, int writable ) +static void analyze_image( mlt_filter filter, mlt_frame frame, uint8_t* vs_image, VSPixelFormat vs_format, int width, int height ) { mlt_properties properties = MLT_FILTER_PROPERTIES( filter ); vs_data* data = (vs_data*)filter->child; mlt_position pos = mlt_filter_get_position( filter, frame ); - *format = mlt_image_yuv420p; - - writable = writable || mlt_properties_get_int( properties, "show" ) ? 1 : 0; - - int error = mlt_frame_get_image( frame, image, format, width, height, writable ); - if ( !error ) + if ( !data->analyze_data ) { - // Service locks are for concurrency control - mlt_service_lock( MLT_FILTER_SERVICE(filter) ); - - if ( !data->analyze_data ) - { - data->analyze_data = init_analyze_data( filter, frame, *format, *width, *height ); - } - - // Initialize the VSFrame to be analyzed. - VSMotionDetect* md = &data->analyze_data->md; - LocalMotions localmotions; - VSFrame vsFrame; - vsFrameFillFromBuffer( &vsFrame, *image, &md->fi ); + data->analyze_data = init_analyze_data( filter, frame, vs_format, width, height ); + } - // Detect and save motions. - vsMotionDetection( md, &localmotions, &vsFrame ); - vs_vector_set_dup( &data->analyze_data->mlms, pos, &localmotions, sizeof(LocalMotions) ); + // Initialize the VSFrame to be analyzed. + VSMotionDetect* md = &data->analyze_data->md; + LocalMotions localmotions; + VSFrame vsFrame; + vsFrameFillFromBuffer( &vsFrame, vs_image, &md->fi ); - // Publish the motions if this is the last frame. - if ( pos + 1 == mlt_filter_get_length2( filter, frame ) ) - { - publish_manylocalmotions( properties, &data->analyze_data->mlms ); - } + // Detect and save motions. + vsMotionDetection( md, &localmotions, &vsFrame ); + vs_vector_set_dup( &data->analyze_data->mlms, pos, &localmotions, sizeof(LocalMotions) ); - mlt_service_unlock( MLT_FILTER_SERVICE(filter) ); + // Publish the motions if this is the last frame. + if ( pos + 1 == mlt_filter_get_length2( filter, frame ) ) + { + publish_manylocalmotions( properties, &data->analyze_data->mlms ); } - return error; } static int get_image( mlt_frame frame, uint8_t **image, mlt_image_format *format, int *width, int *height, int writable ) { mlt_filter filter = (mlt_filter)mlt_frame_pop_service( frame ); mlt_properties properties = MLT_FILTER_PROPERTIES( filter ); + uint8_t* vs_image = NULL; + VSPixelFormat vs_format = PF_NONE; + + // VS only works on progressive frames + mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "consumer_deinterlace", 1 ); - if( mlt_properties_get( properties, "results" ) ) + *format = validate_format( *format ); + + int error = mlt_frame_get_image( frame, image, format, width, height, 1 ); + + // Convert the received image to a format vid.stab can handle + if ( !error ) { - return get_image_and_apply( filter, frame, image, format, width, height, writable ); + vs_format = mltimage_to_vsimage( *format, *width, *height, *image, &vs_image ); } - else + + if( vs_image ) { - return get_image_and_analyze( filter, frame, image, format, width, height, writable ); + mlt_service_lock( MLT_FILTER_SERVICE(filter) ); + + if( mlt_properties_get( properties, "results" ) ) + { + transform_image( filter, frame, vs_image, vs_format, *width, *height ); + vsimage_to_mltimage( vs_image, *image, *format, *width, *height ); + } + else + { + analyze_image( filter, frame, vs_image, vs_format, *width, *height ); + if( mlt_properties_get_int( properties, "show" ) == 1 ) + { + vsimage_to_mltimage( vs_image, *image, *format, *width, *height ); + } + } + + mlt_service_unlock( MLT_FILTER_SERVICE(filter) ); + + free_vsimage( vs_image, vs_format ); } + + return error; } static mlt_frame process_filter( mlt_filter filter, mlt_frame frame )