git.sesse.net Git - mlt/blob - src/modules/vid.stab/filter_vidstab.cpp

   1 /*
   2  * filter_vidstab.cpp
   3  * Copyright (C) 2013 Marco Gittler <g.marco@freenet.de>
   4  * Copyright (C) 2013 Jakub Ksiezniak <j.ksiezniak@gmail.com>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software Foundation,
  18  * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  19  */
  20
  21 extern "C"
  22 {
  23 #include <vid.stab/libvidstab.h>
  24 #include <framework/mlt.h>
  25 #include <framework/mlt_animation.h>
  26 }
  27
  28 #include <sstream>
  29 #include <string.h>
  30 #include <assert.h>
  31 #include "common.h"
  32
  33
  34 typedef struct _stab_data
  35 {
  36         VSMotionDetect md;
  37         mlt_animation animation;
  38 } StabData;
  39
  40 typedef struct
  41 {
  42         VSTransformData td;
  43         VSTransformations trans;
  44 } TransformData;
  45
  46
  47 char* vectors_serializer(mlt_animation animation, int length)
  48 {
  49         return mlt_animation_serialize(animation);
  50 }
  51
  52 char* lm_serializer(LocalMotions *lms, int length)
  53 {
  54         std::ostringstream oss;
  55         int size = vs_vector_size(lms);
  56         for (int i = 0; i < size; ++i)
  57         {
  58                 LocalMotion* lm = (LocalMotion*) vs_vector_get(lms, i);
  59                 oss << lm->v.x << ' ';
  60                 oss << lm->v.y << ' ';
  61                 oss << lm->f.x << ' ';
  62                 oss << lm->f.y << ' ';
  63                 oss << lm->f.size << ' ';
  64                 oss << lm->contrast << ' ';
  65                 oss << lm->match << ' ';
  66         }
  67         return strdup(oss.str().c_str());
  68 }
  69
  70 int lm_deserialize(LocalMotions *lms, mlt_property property)
  71 {
  72         std::istringstream iss(mlt_property_get_string(property));
  73         vs_vector_init(lms, 0);
  74
  75         while (iss.good())
  76         {
  77                 LocalMotion lm;
  78                 iss >> lm.v.x >> lm.v.y >> lm.f.x >> lm.f.y >> lm.f.size >> lm.contrast >> lm.match;
  79                 if (iss.fail())
  80                 {
  81                         break;
  82                 }
  83                 vs_vector_append_dup(lms, &lm, sizeof(lm));
  84         }
  85         return 0;
  86 }
  87
  88 void lm_destructor(void *lms)
  89 {
  90         vs_vector_del(static_cast<VSVector*>(lms));
  91 }
  92
  93 static void serialize_localmotions(StabData* data, LocalMotions &vectors, mlt_position pos)
  94 {
  95         mlt_animation_item_s item;
  96
  97         // Initialize animation item
  98         item.is_key = 1;
  99         item.frame = data->md.frameNum;
 100         item.keyframe_type = mlt_keyframe_discrete;
 101         item.property = mlt_property_init();
 102
 103         mlt_property_set_data(item.property, &vectors, 1, lm_destructor, (mlt_serialiser) lm_serializer);
 104         mlt_animation_insert(data->animation, &item);
 105         mlt_property_close(item.property);
 106 }
 107
 108 int vectors_deserialize(mlt_animation anim, VSManyLocalMotions *mlms)
 109 {
 110         int error = 0;
 111         mlt_animation_item_s item;
 112         item.property = mlt_property_init();
 113
 114         vs_vector_init(mlms, 1024); // initial number of frames, but it will be increased
 115
 116         int length = mlt_animation_get_length(anim);
 117         for (int i = 0; i < length; ++i)
 118         {
 119                 LocalMotions lms;
 120
 121                 // read lms
 122                 mlt_animation_get_item(anim, &item, i + 1);
 123                 if ((error = lm_deserialize(&lms, item.property)))
 124                 {
 125                         break;
 126                 }
 127
 128                 vs_vector_set_dup(mlms, i, &lms, sizeof(LocalMotions));
 129         }
 130
 131         mlt_property_close(item.property);
 132         return error;
 133 }
 134
 135 void destroy_transforms(TransformData *data)
 136 {
 137         if (data)
 138         {
 139                 vsTransformDataCleanup(&data->td);
 140                 vsTransformationsCleanup(&data->trans);
 141                 delete data;
 142         }
 143 }
 144
 145 TransformData* initialize_transforms(int *width, int *height, mlt_image_format *format,
 146                 mlt_properties properties, const char* interps)
 147 {
 148         TransformData *data = new TransformData;
 149         memset(data, 0, sizeof(TransformData));
 150
 151         VSPixelFormat pf = convertImageFormat(*format);
 152         VSFrameInfo fi_src, fi_dst;
 153         vsFrameInfoInit(&fi_src, *width, *height, pf);
 154         vsFrameInfoInit(&fi_dst, *width, *height, pf);
 155
 156         const char* filterName = mlt_properties_get(properties, "mlt_service");
 157
 158         VSTransformConfig conf = vsTransformGetDefaultConfig(filterName);
 159         conf.smoothing = mlt_properties_get_int(properties, "smoothing");
 160         conf.maxShift = mlt_properties_get_int(properties, "maxshift");
 161         conf.maxAngle = mlt_properties_get_double(properties, "maxangle");
 162         conf.crop = (VSBorderType) mlt_properties_get_int(properties, "crop");
 163         conf.zoom = mlt_properties_get_int(properties, "zoom");
 164         conf.optZoom = mlt_properties_get_int(properties, "optzoom");
 165         conf.zoomSpeed = mlt_properties_get_double(properties, "zoomspeed");
 166         conf.relative = mlt_properties_get_int(properties, "relative");
 167         conf.invert = mlt_properties_get_int(properties, "invert");
 168         if (mlt_properties_get_int(properties, "tripod") != 0)
 169         {
 170                 // Virtual tripod mode: relative=False, smoothing=0
 171                 conf.relative = 0;
 172                 conf.smoothing = 0;
 173         }
 174
 175         // by default a bilinear interpolation is selected
 176         conf.interpolType = VS_BiLinear;
 177         if (strcmp(interps, "nearest") == 0 || strcmp(interps, "neighbor") == 0)
 178                 conf.interpolType = VS_Zero;
 179         else if (strcmp(interps, "tiles") == 0 || strcmp(interps, "fast_bilinear") == 0)
 180                 conf.interpolType = VS_Linear;
 181
 182         vsTransformDataInit(&data->td, &conf, &fi_src, &fi_dst);
 183         vsTransformationsInit(&data->trans);
 184
 185         // load transformations
 186         mlt_animation animation = mlt_animation_new();
 187         char* strAnim = mlt_properties_get(properties, "vectors");
 188         if (mlt_animation_parse(animation, strAnim, 0, 0, NULL))
 189         {
 190                 mlt_log_warning(NULL, "parse failed\n");
 191                 mlt_animation_close(animation);
 192                 destroy_transforms(data);
 193                 return NULL;
 194         }
 195
 196         VSManyLocalMotions mlms;
 197         if (vectors_deserialize(animation, &mlms))
 198         {
 199                 mlt_animation_close(animation);
 200                 destroy_transforms(data);
 201                 return NULL;
 202         }
 203
 204         mlt_animation_close(animation);
 205
 206         vsLocalmotions2Transforms(&data->td, &mlms, &data->trans);
 207         vsPreprocessTransforms(&data->td, &data->trans);
 208         return data;
 209 }
 210
 211 int get_image_and_transform(mlt_frame frame, uint8_t **image, mlt_image_format *format, int *width, int *height, int writable)
 212 {
 213         int error = 0;
 214         mlt_filter filter = (mlt_filter) mlt_frame_pop_service(frame);
 215         mlt_properties properties = MLT_FILTER_PROPERTIES(filter);
 216
 217         *format = mlt_image_yuv420p;
 218
 219         error = mlt_frame_get_image(frame, image, format, width, height, 1);
 220         if (!error)
 221         {
 222                 // Service locks are for concurrency control
 223                 mlt_service_lock(MLT_FILTER_SERVICE(filter));
 224
 225                 TransformData *data = static_cast<TransformData*>(mlt_properties_get_data(properties, "_transform_data", NULL));
 226
 227                 // Handle signal from app to re-init data
 228                 if (mlt_properties_get_int(properties, "refresh"))
 229                 {
 230                         mlt_properties_set(properties, "refresh", NULL);
 231                         destroy_transforms(data);
 232                         data = NULL;
 233                 }
 234
 235                 if (!data)
 236                 {
 237                         const char *interps = mlt_properties_get(MLT_FRAME_PROPERTIES(frame), "rescale.interp");
 238                         data = initialize_transforms(width, height, format, properties, interps);
 239                         if(!data) {
 240                                 mlt_service_unlock(MLT_FILTER_SERVICE(filter));
 241                                 return 1; // return error code
 242                         }
 243                         mlt_properties_set_data(properties, "_transform_data", data, 0, (mlt_destructor) destroy_transforms, NULL);
 244                 }
 245
 246                 VSTransformData* td = &data->td;
 247                 VSFrame vsFrame;
 248                 vsFrameFillFromBuffer(&vsFrame, *image, vsTransformGetSrcFrameInfo(td));
 249
 250                 // transform frame
 251                 data->trans.current = mlt_filter_get_position(filter, frame);
 252                 vsTransformPrepare(td, &vsFrame, &vsFrame);
 253                 VSTransform t = vsGetNextTransform(td, &data->trans);
 254                 vsDoTransform(td, t);
 255                 vsTransformFinish(td);
 256
 257                 mlt_service_unlock(MLT_FILTER_SERVICE(filter));
 258         }
 259
 260         return error;
 261 }
 262
 263 static StabData* init_detect(mlt_properties properties, mlt_image_format *format, int *width, int *height)
 264 {
 265         StabData *data = new StabData;
 266         memset(data, 0, sizeof(StabData));
 267         data->animation = mlt_animation_new();
 268
 269         VSPixelFormat pf = convertImageFormat(*format);
 270         VSFrameInfo fi;
 271         vsFrameInfoInit(&fi, *width, *height, pf);
 272
 273         const char* filterName = mlt_properties_get(properties, "mlt_service");
 274
 275         VSMotionDetectConfig conf = vsMotionDetectGetDefaultConfig(filterName);
 276         conf.shakiness = mlt_properties_get_int(properties, "shakiness");
 277         conf.accuracy = mlt_properties_get_int(properties, "accuracy");
 278         conf.stepSize = mlt_properties_get_int(properties, "stepsize");
 279         conf.algo = mlt_properties_get_int(properties, "algo");
 280         conf.contrastThreshold = mlt_properties_get_double(properties, "mincontrast");
 281         conf.show = mlt_properties_get_int(properties, "show");
 282         conf.virtualTripod = mlt_properties_get_int(properties, "tripod");
 283         vsMotionDetectInit(&data->md, &conf, &fi);
 284
 285         // add vectors to properties
 286         mlt_properties_set_data(properties, "vectors", data->animation, 1, (mlt_destructor) mlt_animation_close,
 287                                         (mlt_serialiser) vectors_serializer);
 288         return data;
 289 }
 290
 291 void destroy_detect(StabData *data)
 292 {
 293         if (data)
 294         {
 295                 vsMotionDetectionCleanup(&data->md);
 296                 delete data;
 297         }
 298 }
 299
 300 int get_image_and_detect(mlt_frame frame, uint8_t **image, mlt_image_format *format, int *width, int *height, int writable)
 301 {
 302         mlt_filter filter = (mlt_filter) mlt_frame_pop_service(frame);
 303         mlt_properties properties = MLT_FILTER_PROPERTIES(filter);
 304
 305         *format = mlt_image_yuv420p;
 306
 307         writable = writable || mlt_properties_get_int(properties, "show") ? 1 : 0;
 308
 309         int error = mlt_frame_get_image(frame, image, format, width, height, writable);
 310         if (!error)
 311         {
 312                 // Service locks are for concurrency control
 313                 mlt_service_lock(MLT_FILTER_SERVICE(filter));
 314
 315                 StabData *data = static_cast<StabData*>(mlt_properties_get_data(properties, "_stab_data", NULL));
 316                 if (!data)
 317                 {
 318                         data = init_detect(properties, format, width, height);
 319                         mlt_properties_set_data(properties, "_stab_data", data, 0, (mlt_destructor) destroy_detect, NULL);
 320                 }
 321
 322                 VSMotionDetect* md = &data->md;
 323                 LocalMotions localmotions;
 324                 VSFrame vsFrame;
 325                 vsFrameFillFromBuffer(&vsFrame, *image, &md->fi);
 326
 327                 // detect and save motions
 328                 vsMotionDetection(md, &localmotions, &vsFrame);
 329                 mlt_position pos = mlt_filter_get_position( filter, frame );
 330                 serialize_localmotions(data, localmotions, pos);
 331
 332                 mlt_service_unlock(MLT_FILTER_SERVICE(filter));
 333         }
 334
 335         return error;
 336 }
 337
 338 static mlt_frame process_filter(mlt_filter filter, mlt_frame frame)
 339 {
 340         mlt_properties properties = MLT_FILTER_PROPERTIES(filter);
 341         mlt_get_image vidstab_get_image = (mlt_get_image) mlt_properties_get_data( properties, "_vidstab_get_image", NULL );
 342
 343 #if 1
 344         mlt_position pos = mlt_filter_get_position(filter, frame);
 345         mlt_position length = mlt_filter_get_length2(filter, frame) - 1;
 346         if(pos >= length)
 347         {
 348                 mlt_properties_set_data(properties, "_vidstab_get_image", NULL, 0, NULL, NULL);
 349         }
 350 #endif
 351
 352         if(vidstab_get_image == NULL)
 353         {
 354                 if(mlt_properties_get(properties, "vectors") == NULL)
 355                 {
 356                         // vectors are NULL, so use a detect filter
 357                         vidstab_get_image = get_image_and_detect;
 358                 } else {
 359                         // found vectors, so use a transform filter
 360                         vidstab_get_image = get_image_and_transform;
 361                 }
 362
 363                 mlt_properties_set_data( properties, "_vidstab_get_image", (void*)vidstab_get_image, 0, NULL, NULL );
 364         }
 365
 366         mlt_frame_push_service(frame, filter);
 367         mlt_frame_push_get_image(frame, vidstab_get_image);
 368         return frame;
 369 }
 370
 371 extern "C"
 372 {
 373
 374 mlt_filter filter_vidstab_init(mlt_profile profile, mlt_service_type type,
 375                 const char *id, char *arg)
 376 {
 377         mlt_filter filter = mlt_filter_new();
 378
 379         if( filter )
 380         {
 381                 filter->process = process_filter;
 382
 383                 mlt_properties properties = MLT_FILTER_PROPERTIES(filter);
 384
 385                 //properties for stabilize
 386                 mlt_properties_set(properties, "shakiness", "4");
 387                 mlt_properties_set(properties, "accuracy", "4");
 388                 mlt_properties_set(properties, "stepsize", "6");
 389                 mlt_properties_set(properties, "algo", "1");
 390                 mlt_properties_set(properties, "mincontrast", "0.3");
 391                 mlt_properties_set(properties, "show", "0");
 392                 mlt_properties_set(properties, "tripod", "0");
 393
 394                 // properties for transform
 395                 mlt_properties_set(properties, "smoothing", "15");
 396                 mlt_properties_set(properties, "maxshift", "-1");
 397                 mlt_properties_set(properties, "maxangle", "-1");
 398                 mlt_properties_set(properties, "crop", "0");
 399                 mlt_properties_set(properties, "invert", "0");
 400                 mlt_properties_set(properties, "relative", "1");
 401                 mlt_properties_set(properties, "zoom", "0");
 402                 mlt_properties_set(properties, "optzoom", "1");
 403                 mlt_properties_set(properties, "zoomspeed", "0.25");
 404
 405                 mlt_properties_set(properties, "vid.stab.version", LIBVIDSTAB_VERSION);
 406         }
 407
 408         return filter;
 409 }
 410
 411 }