git.sesse.net Git - mlt/blob - src/modules/vid.stab/filter_vidstab.cpp

   1 /*
   2  * filter_vidstab.cpp
   3  * Copyright (C) 2013 Marco Gittler <g.marco@freenet.de>
   4  * Copyright (C) 2013 Jakub Ksiezniak <j.ksiezniak@gmail.com>
   5  * Copyright (C) 2014 Brian Matherly <pez4brian@yahoo.com>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software Foundation,
  19  * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  20  */
  21
  22 extern "C"
  23 {
  24 #include <vid.stab/libvidstab.h>
  25 #include <framework/mlt.h>
  26 #include <framework/mlt_animation.h>
  27 #include "common.h"
  28 }
  29
  30 #include <sstream>
  31 #include <string.h>
  32 #include <assert.h>
  33
  34 typedef struct
  35 {
  36         VSMotionDetect md;
  37         FILE* results;
  38         mlt_position last_position;
  39 } vs_analyze;
  40
  41 typedef struct
  42 {
  43         VSTransformData td;
  44         VSTransformConfig conf;
  45         VSTransformations trans;
  46 } vs_apply;
  47
  48 typedef struct
  49 {
  50         vs_analyze* analyze_data;
  51         vs_apply* apply_data;
  52 } vs_data;
  53
  54 static void get_transform_config( VSTransformConfig* conf, mlt_filter filter, mlt_frame frame )
  55 {
  56         mlt_properties properties = MLT_FILTER_PROPERTIES( filter );
  57         const char* filterName = mlt_properties_get( properties, "mlt_service" );
  58
  59         *conf = vsTransformGetDefaultConfig( filterName );
  60         conf->smoothing = mlt_properties_get_int( properties, "smoothing" );
  61         conf->maxShift = mlt_properties_get_int( properties, "maxshift" );
  62         conf->maxAngle = mlt_properties_get_double( properties, "maxangle" );
  63         conf->crop = (VSBorderType)mlt_properties_get_int( properties, "crop" );
  64         conf->zoom = mlt_properties_get_int( properties, "zoom" );
  65         conf->optZoom = mlt_properties_get_int( properties, "optzoom" );
  66         conf->zoomSpeed = mlt_properties_get_double( properties, "zoomspeed" );
  67         conf->relative = mlt_properties_get_int( properties, "relative" );
  68         conf->invert = mlt_properties_get_int( properties, "invert" );
  69         if ( mlt_properties_get_int( properties, "tripod" ) != 0 )
  70         {
  71                 // Virtual tripod mode: relative=False, smoothing=0
  72                 conf->relative = 0;
  73                 conf->smoothing = 0;
  74         }
  75
  76         // by default a bicubic interpolation is selected
  77         const char *interps = mlt_properties_get( MLT_FRAME_PROPERTIES( frame ), "rescale.interp" );
  78         conf->interpolType = VS_BiCubic;
  79         if ( strcmp( interps, "nearest" ) == 0 || strcmp( interps, "neighbor" ) == 0 )
  80                 conf->interpolType = VS_Zero;
  81         else if ( strcmp( interps, "tiles" ) == 0 || strcmp( interps, "fast_bilinear" ) == 0 )
  82                 conf->interpolType = VS_Linear;
  83         else if ( strcmp( interps, "bilinear" ) == 0 )
  84                 conf->interpolType = VS_BiLinear;
  85 }
  86
  87 static int check_apply_config( mlt_filter filter, mlt_frame frame )
  88 {
  89         vs_apply* apply_data = ((vs_data*)filter->child)->apply_data;
  90
  91         if( apply_data )
  92         {
  93                 VSTransformConfig new_conf;
  94                 memset( &new_conf, 0, sizeof(VSTransformConfig) );
  95                 get_transform_config( &new_conf, filter, frame );
  96                 return compare_transform_config( &apply_data->conf, &new_conf );
  97         }
  98
  99         return 0;
 100 }
 101
 102 static void destory_apply_data( vs_apply* apply_data )
 103 {
 104         if ( apply_data )
 105         {
 106                 vsTransformDataCleanup( &apply_data->td );
 107                 vsTransformationsCleanup( &apply_data->trans );
 108                 free( apply_data );
 109         }
 110 }
 111
 112 static void init_apply_data( mlt_filter filter, mlt_frame frame, VSPixelFormat vs_format, int width, int height )
 113 {
 114         mlt_properties properties = MLT_FILTER_PROPERTIES( filter );
 115         vs_data* data = (vs_data*)filter->child;
 116         vs_apply* apply_data = (vs_apply*)calloc( 1, sizeof(vs_apply) );
 117         char* filename = mlt_properties_get( properties, "results" );
 118         memset( apply_data, 0, sizeof( vs_apply ) );
 119
 120         mlt_log_info( MLT_FILTER_SERVICE(filter), "Load results from %s\n", filename );
 121
 122         // Initialize the VSTransformConfig
 123         get_transform_config( &apply_data->conf, filter, frame );
 124
 125         // Initialize VSTransformData
 126         VSFrameInfo fi_src, fi_dst;
 127         vsFrameInfoInit( &fi_src, width, height, vs_format );
 128         vsFrameInfoInit( &fi_dst, width, height, vs_format );
 129         vsTransformDataInit( &apply_data->td, &apply_data->conf, &fi_src, &fi_dst );
 130
 131         // Initialize VSTransformations
 132         vsTransformationsInit( &apply_data->trans );
 133
 134         // Load the motions from the analyze step and convert them to VSTransformations
 135
 136         FILE* f = fopen( filename, "r" );
 137         VSManyLocalMotions mlms;
 138
 139         if( vsReadLocalMotionsFile( f, &mlms ) == VS_OK )
 140         {
 141                 int i = 0;
 142                 mlt_log_info( MLT_FILTER_SERVICE(filter), "Successfully loaded %d motions\n", vs_vector_size( &mlms ) );
 143                 vsLocalmotions2Transforms( &apply_data->td, &mlms, &apply_data->trans );
 144                 vsPreprocessTransforms( &apply_data->td, &apply_data->trans );
 145
 146                 // Free the MultipleLocalMotions
 147                 for( i = 0; i < vs_vector_size( &mlms ); i++ )
 148                 {
 149                         LocalMotions* lms = (LocalMotions*)vs_vector_get( &mlms, i );
 150                         if( lms )
 151                         {
 152                                 vs_vector_del( lms );
 153                         }
 154                 }
 155                 vs_vector_del( &mlms );
 156
 157                 data->apply_data = apply_data;
 158         }
 159         else
 160         {
 161                 mlt_log_error( MLT_FILTER_SERVICE(filter), "Can not read results file: %s\n", filename );
 162                 destory_apply_data( apply_data );
 163                 data->apply_data = NULL;
 164         }
 165
 166         if( f )
 167         {
 168                 fclose( f );
 169         }
 170 }
 171
 172 void destory_analyze_data( vs_analyze* analyze_data )
 173 {
 174         if ( analyze_data )
 175         {
 176                 vsMotionDetectionCleanup( &analyze_data->md );
 177                 if( analyze_data->results )
 178                 {
 179                         fclose( analyze_data->results );
 180                         analyze_data->results = NULL;
 181                 }
 182                 free( analyze_data );
 183         }
 184 }
 185
 186 static void init_analyze_data( mlt_filter filter, mlt_frame frame, VSPixelFormat vs_format, int width, int height )
 187 {
 188         mlt_properties properties = MLT_FILTER_PROPERTIES( filter );
 189         vs_data* data = (vs_data*)filter->child;
 190         vs_analyze* analyze_data = (vs_analyze*)calloc( 1, sizeof(vs_analyze) );
 191         memset( analyze_data, 0, sizeof(vs_analyze) );
 192
 193         // Initialize a VSMotionDetectConfig
 194         const char* filterName = mlt_properties_get( properties, "mlt_service" );
 195         VSMotionDetectConfig conf = vsMotionDetectGetDefaultConfig( filterName );
 196         conf.shakiness = mlt_properties_get_int( properties, "shakiness" );
 197         conf.accuracy = mlt_properties_get_int( properties, "accuracy" );
 198         conf.stepSize = mlt_properties_get_int( properties, "stepsize" );
 199         conf.contrastThreshold = mlt_properties_get_double( properties, "mincontrast" );
 200         conf.show = mlt_properties_get_int( properties, "show" );
 201         conf.virtualTripod = mlt_properties_get_int( properties, "tripod" );
 202
 203         // Initialize a VSFrameInfo
 204         VSFrameInfo fi;
 205         vsFrameInfoInit( &fi, width, height, vs_format );
 206
 207         // Initialize the saved VSMotionDetect
 208         vsMotionDetectInit( &analyze_data->md, &conf, &fi );
 209
 210         // Initialize the file to save results to
 211         char* filename = mlt_properties_get( properties, "filename" );
 212         analyze_data->results = fopen( filename, "w" );
 213         if ( vsPrepareFile( &analyze_data->md, analyze_data->results ) != VS_OK )
 214         {
 215                 mlt_log_error( MLT_FILTER_SERVICE(filter), "Can not write to results file: %s\n", filename );
 216                 destory_analyze_data( analyze_data );
 217                 data->analyze_data = NULL;
 218         }
 219         else
 220         {
 221                 data->analyze_data = analyze_data;
 222         }
 223 }
 224
 225 static int apply_results( mlt_filter filter, mlt_frame frame, uint8_t* vs_image, VSPixelFormat vs_format, int width, int height )
 226 {
 227         int error = 0;
 228         mlt_properties properties = MLT_FILTER_PROPERTIES( filter );
 229         vs_data* data = (vs_data*)filter->child;
 230
 231         if ( check_apply_config( filter, frame ) ||
 232                 mlt_properties_get_int( properties, "reload" ) )
 233         {
 234                 mlt_properties_set_int( properties, "reload", 0 );
 235                 destory_apply_data( data->apply_data );
 236                 data->apply_data = NULL;
 237         }
 238
 239         // Init transform data if necessary (first time)
 240         if ( !data->apply_data )
 241         {
 242                 init_apply_data( filter, frame, vs_format, width, height );
 243         }
 244
 245         if( data->apply_data )
 246         {
 247                 // Apply transformations to this image
 248                 VSTransformData* td = &data->apply_data->td;
 249                 VSTransformations* trans = &data->apply_data->trans;
 250                 VSFrame vsFrame;
 251                 vsFrameFillFromBuffer( &vsFrame, vs_image, vsTransformGetSrcFrameInfo( td ) );
 252                 trans->current = mlt_filter_get_position( filter, frame );
 253                 vsTransformPrepare( td, &vsFrame, &vsFrame );
 254                 VSTransform t = vsGetNextTransform( td, trans );
 255                 vsDoTransform( td, t );
 256                 vsTransformFinish( td );
 257         }
 258
 259         return error;
 260 }
 261
 262 static void analyze_image( mlt_filter filter, mlt_frame frame, uint8_t* vs_image, VSPixelFormat vs_format, int width, int height )
 263 {
 264         mlt_properties properties = MLT_FILTER_PROPERTIES( filter );
 265         vs_data* data = (vs_data*)filter->child;
 266         mlt_position pos = mlt_filter_get_position( filter, frame );
 267
 268         // If any frames are skipped, analysis data will be incomplete.
 269         if( data->analyze_data && pos != data->analyze_data->last_position + 1 )
 270         {
 271                 mlt_log_error( MLT_FILTER_SERVICE(filter), "Bad frame sequence\n" );
 272                 destory_analyze_data( data->analyze_data );
 273                 data->analyze_data = NULL;
 274         }
 275
 276         if ( !data->analyze_data && pos == 0 )
 277         {
 278                 // Analysis must start on the first frame
 279                 init_analyze_data( filter, frame, vs_format, width, height );
 280         }
 281
 282         if( data->analyze_data )
 283         {
 284                 // Initialize the VSFrame to be analyzed.
 285                 VSMotionDetect* md = &data->analyze_data->md;
 286                 LocalMotions localmotions;
 287                 VSFrame vsFrame;
 288                 vsFrameFillFromBuffer( &vsFrame, vs_image, &md->fi );
 289
 290                 // Detect and save motions.
 291                 if( vsMotionDetection( md, &localmotions, &vsFrame ) == VS_OK )
 292                 {
 293                         vsWriteToFile( md, data->analyze_data->results, &localmotions);
 294                         vs_vector_del( &localmotions );
 295                 }
 296                 else
 297                 {
 298                         mlt_log_error( MLT_FILTER_SERVICE(filter), "Motion detection failed\n" );
 299                         destory_analyze_data( data->analyze_data );
 300                         data->analyze_data = NULL;
 301                 }
 302
 303                 // Publish the motions if this is the last frame.
 304                 if ( pos + 1 == mlt_filter_get_length2( filter, frame ) )
 305                 {
 306                         mlt_log_info( MLT_FILTER_SERVICE(filter), "Analysis complete\n" );
 307                         destory_analyze_data( data->analyze_data );
 308                         data->analyze_data = NULL;
 309                         mlt_properties_set( properties, "results", mlt_properties_get( properties, "filename" ) );
 310                 }
 311                 else
 312                 {
 313                         data->analyze_data->last_position = pos;
 314                 }
 315         }
 316 }
 317
 318 static int get_image( mlt_frame frame, uint8_t **image, mlt_image_format *format, int *width, int *height, int writable )
 319 {
 320         mlt_filter filter = (mlt_filter)mlt_frame_pop_service( frame );
 321         mlt_properties properties = MLT_FILTER_PROPERTIES( filter );
 322         uint8_t* vs_image = NULL;
 323         VSPixelFormat vs_format = PF_NONE;
 324
 325         // VS only works on progressive frames
 326         mlt_properties_set_int( MLT_FRAME_PROPERTIES( frame ), "consumer_deinterlace", 1 );
 327
 328         *format = validate_format( *format );
 329
 330         int error = mlt_frame_get_image( frame, image, format, width, height, 1 );
 331
 332         // Convert the received image to a format vid.stab can handle
 333         if ( !error )
 334         {
 335                 vs_format = mltimage_to_vsimage( *format, *width, *height, *image, &vs_image );
 336         }
 337
 338         if( vs_image )
 339         {
 340                 mlt_service_lock( MLT_FILTER_SERVICE(filter) );
 341
 342                 char* results = mlt_properties_get( properties, "results" );
 343                 if( results && strcmp( results, "" ) )
 344                 {
 345                         apply_results( filter, frame, vs_image, vs_format, *width, *height );
 346                         vsimage_to_mltimage( vs_image, *image, *format, *width, *height );
 347                 }
 348                 else
 349                 {
 350                         analyze_image( filter, frame, vs_image, vs_format, *width, *height );
 351                         if( mlt_properties_get_int( properties, "show" ) == 1 )
 352                         {
 353                                 vsimage_to_mltimage( vs_image, *image, *format, *width, *height );
 354                         }
 355                 }
 356
 357                 mlt_service_unlock( MLT_FILTER_SERVICE(filter) );
 358
 359                 free_vsimage( vs_image, vs_format );
 360         }
 361
 362         return error;
 363 }
 364
 365 static mlt_frame process_filter( mlt_filter filter, mlt_frame frame )
 366 {
 367         mlt_frame_push_service( frame, filter );
 368         mlt_frame_push_get_image( frame, get_image );
 369         return frame;
 370 }
 371
 372 static void filter_close( mlt_filter filter )
 373 {
 374         vs_data* data = (vs_data*)filter->child;
 375         if ( data )
 376         {
 377                 if ( data->analyze_data ) destory_analyze_data( data->analyze_data );
 378                 if ( data->apply_data ) destory_apply_data( data->apply_data );
 379                 free( data );
 380         }
 381         filter->close = NULL;
 382         filter->child = NULL;
 383         filter->parent.close = NULL;
 384         mlt_service_close( &filter->parent );
 385 }
 386
 387 extern "C"
 388 {
 389
 390 mlt_filter filter_vidstab_init( mlt_profile profile, mlt_service_type type, const char *id, char *arg )
 391 {
 392         mlt_filter filter = mlt_filter_new();
 393         vs_data* data = (vs_data*)calloc( 1, sizeof(vs_data) );
 394
 395         if ( filter && data )
 396         {
 397                 data->analyze_data = NULL;
 398                 data->apply_data = NULL;
 399
 400                 filter->close = filter_close;
 401                 filter->child = data;
 402                 filter->process = process_filter;
 403
 404                 mlt_properties properties = MLT_FILTER_PROPERTIES(filter);
 405
 406                 //properties for analyze
 407                 mlt_properties_set( properties, "filename", "vidstab.trf" );
 408                 mlt_properties_set( properties, "shakiness", "4" );
 409                 mlt_properties_set( properties, "accuracy", "4" );
 410                 mlt_properties_set( properties, "stepsize", "6" );
 411                 mlt_properties_set( properties, "algo", "1" );
 412                 mlt_properties_set( properties, "mincontrast", "0.3" );
 413                 mlt_properties_set( properties, "show", "0" );
 414                 mlt_properties_set( properties, "tripod", "0" );
 415
 416                 // properties for apply
 417                 mlt_properties_set( properties, "smoothing", "15" );
 418                 mlt_properties_set( properties, "maxshift", "-1" );
 419                 mlt_properties_set( properties, "maxangle", "-1" );
 420                 mlt_properties_set( properties, "crop", "0" );
 421                 mlt_properties_set( properties, "invert", "0" );
 422                 mlt_properties_set( properties, "relative", "1" );
 423                 mlt_properties_set( properties, "zoom", "0" );
 424                 mlt_properties_set( properties, "optzoom", "1" );
 425                 mlt_properties_set( properties, "zoomspeed", "0.25" );
 426                 mlt_properties_set( properties, "reload", "0" );
 427
 428                 mlt_properties_set( properties, "vid.stab.version", LIBVIDSTAB_VERSION );
 429
 430                 init_vslog();
 431         }
 432         else
 433         {
 434                 if( filter )
 435                 {
 436                         mlt_filter_close( filter );
 437                 }
 438
 439                 if( data )
 440                 {
 441                         free( data );
 442                 }
 443
 444                 filter = NULL;
 445         }
 446         return filter;
 447 }
 448
 449 }