git.sesse.net Git - mlt/blob - src/modules/motion_est/producer_slowmotion.c

   1 /*
   2  * producer_slowmotion.c -- create subspeed frames
   3  * Author: Zachary Drew
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software Foundation,
  17  * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18  */
  19
  20 #include "filter_motion_est.h"
  21 #include <framework/mlt.h>
  22
  23 #include <stdio.h>
  24 #include <stdlib.h>
  25 #include <math.h>
  26 #include <string.h>
  27 #include <sys/time.h>
  28 #include <assert.h>
  29 #define SHIFT 8
  30 #define ABS(a) ((a) >= 0 ? (a) : (-(a)))
  31
  32 // This is used to constrains pixel operations between two blocks to be within the image boundry
  33 inline static int constrain(    int *x, int *y, int *w, int *h,
  34                                 const int dx, const int dy,
  35                                 const int left, const int right,
  36                                 const int top, const int bottom)
  37 {
  38         uint32_t penalty = 1 << SHIFT;                  // Retain a few extra bits of precision
  39         int x2 = *x + dx;
  40         int y2 = *y + dy;
  41         int w_remains = *w;
  42         int h_remains = *h;
  43
  44         // Origin of macroblock moves left of image boundy
  45         if( *x < left || x2 < left ) {
  46                 w_remains = *w - left + ((*x < x2) ?  *x : x2);
  47                 *x += *w - w_remains;
  48         }
  49         // Portion of macroblock moves right of image boundry
  50         else if( *x + *w > right || x2 + *w > right )
  51                 w_remains = right - ((*x > x2) ? *x : x2);
  52
  53         // Origin of macroblock moves above image boundy
  54         if( *y < top || y2 < top ) {
  55                 h_remains = *h - top + ((*y < y2) ? *y : y2);
  56                 *y += *h - h_remains;
  57         }
  58         // Portion of macroblock moves bellow image boundry
  59         else if( *y + *h > bottom || y2 + *h > bottom )
  60                 h_remains = bottom - ((*y > y2) ?  *y : y2);
  61
  62         if( w_remains == *w && h_remains == *h ) return penalty;
  63         if( w_remains <= 0 || h_remains <= 0) return 0; // Block is clipped out of existance
  64         penalty = (*w * *h * penalty)
  65                 / ( w_remains * h_remains);             // Recipricol of the fraction of the block that remains
  66
  67         *w = w_remains;                                 // Update the width and height
  68         *h = h_remains;
  69
  70         return penalty;
  71 }
  72
  73 static void motion_interpolate( uint8_t *first_image, uint8_t *second_image, uint8_t *output,
  74                                 int top_mb, int bottom_mb, int left_mb, int right_mb,
  75                                 int mb_w, int mb_h,
  76                                 int width, int height,
  77                                 int xstride, int ystride,
  78                                 double scale,
  79                                 motion_vector *vectors )
  80 {
  81         assert ( scale >= 0.0 && scale <= 1.0 );
  82
  83         int i, j;
  84         int x,y,w,h;
  85         int dx, dy;
  86         int scaled_dx, scaled_dy;
  87         int tx,ty;
  88         uint8_t *f,*s,*r;
  89         motion_vector *here;
  90         int mv_width = width / mb_w;
  91
  92         for( j = top_mb; j <= bottom_mb; j++ ){
  93          for( i = left_mb; i <= right_mb; i++ ){
  94
  95                 here = vectors + j*mv_width + i;
  96                 scaled_dx = (1.0 - scale) * (double)here->dx;
  97                 scaled_dy = (1.0 - scale) * (double)here->dy;
  98                 dx = here->dx;
  99                 dy = here->dy;
 100                 w = mb_w; h = mb_h;
 101                 x = i * w; y = j * h;
 102
 103                 // Denoise function caused some blocks to be completely clipped, ignore them
 104                 if (constrain( &x, &y, &w, &h, dx, dy, 0, width, 0, height) == 0 )
 105                         continue;
 106
 107                 for( ty = y; ty < y + h ; ty++ ){
 108                  for( tx = x; tx < x + w ; tx++ ){
 109
 110                         f = first_image  + (tx + dx     )*xstride + (ty + dy     )*ystride;
 111                         s = second_image + (tx          )*xstride + (ty    )*ystride;
 112                         r = output + (tx+scaled_dx)*xstride + (ty+scaled_dy)*ystride;
 113 /*
 114                         if( ABS(f[0] - s[0]) > 3 * here->msad / (mb_w * mb_h * 2) )
 115                         {
 116                                 r[0] = f[0];
 117                                 r[1] = f[1];
 118                         }
 119
 120                         else
 121                         {
 122
 123 */
 124                                 r[0] = ( 1.0 - scale ) * (double)f[0] + scale * (double)s[0];
 125
 126                                 if( dx % 2 == 0 )
 127                                 {
 128                                         if( scaled_dx % 2 == 0 )
 129                                                 r[1] =  ( 1.0 - scale ) * (double)f[1] + scale * (double) s[1];
 130                                         else
 131                                                 *(r-1) =  ( 1.0 - scale ) * (double)f[1] + scale * (double) s[1];
 132                                 }
 133                                 else
 134                                 {
 135                                         if( scaled_dx %2 == 0 )
 136                                                 // FIXME: may exceed boundies
 137                                                 r[1] =  ( 1.0 - scale ) * ( (double)(*(f-1) + (double)f[3]) / 2.0 ) + scale * (double) s[1];
 138                                         else
 139                                                 // FIXME: may exceed boundies
 140                                                 *(r-1) =  ( 1.0 - scale ) * ( (double)(*(f-1) + (double)f[3]) / 2.0 ) + scale * (double) s[1];
 141                                 }
 142 //                       }
 143                  }
 144                 }
 145          }
 146         }
 147 }
 148
 149 // Image stack(able) method
 150 static int slowmotion_get_image( mlt_frame this, uint8_t **image, mlt_image_format *format, int *width, int *height, int writable )
 151 {
 152
 153         // Get the filter object and properties
 154         mlt_producer producer = mlt_frame_pop_service( this );
 155         mlt_frame second_frame = mlt_frame_pop_service( this );
 156         mlt_frame first_frame = mlt_frame_pop_service( this );
 157
 158         mlt_properties producer_properties = MLT_PRODUCER_PROPERTIES( producer );
 159
 160         // Frame properties objects
 161         mlt_properties frame_properties = MLT_FRAME_PROPERTIES( this );
 162         mlt_properties first_frame_properties = MLT_FRAME_PROPERTIES( first_frame );
 163         mlt_properties second_frame_properties = MLT_FRAME_PROPERTIES( second_frame );
 164
 165         // image stride
 166         int size, xstride, ystride;
 167         switch( *format ){
 168                 case mlt_image_yuv422:
 169                         size = *width * *height * 2;
 170                         xstride = 2;
 171                         ystride = 2 * *width;
 172                         break;
 173                 default:
 174                         fprintf(stderr, "Unsupported image format\n");
 175                         return -1;
 176         }
 177
 178         uint8_t *output = mlt_properties_get_data( producer_properties, "output_buffer", 0 );
 179         if( output == NULL )
 180         {
 181                 output = mlt_pool_alloc( size );
 182
 183                 // Let someone else clean up
 184                 mlt_properties_set_data( producer_properties, "output_buffer", output, size, mlt_pool_release, NULL );
 185         }
 186
 187         uint8_t *first_image = mlt_properties_get_data( first_frame_properties, "image", NULL );
 188         uint8_t *second_image = mlt_properties_get_data( second_frame_properties, "image", NULL );
 189
 190         // which frames are buffered?
 191
 192         int error = 0;
 193
 194         if( first_image == NULL )
 195         {
 196                 error = mlt_frame_get_image( first_frame, &first_image, format, width, height, writable );
 197
 198                 if( error != 0 ) {
 199                         fprintf(stderr, "first_image == NULL get image died\n");
 200                         return error;
 201                 }
 202         }
 203
 204         if( second_image == NULL )
 205         {
 206                 error = mlt_frame_get_image( second_frame, &second_image, format, width, height, writable );
 207
 208                 if( error != 0 ) {
 209                         fprintf(stderr, "second_image == NULL get image died\n");
 210                         return error;
 211                 }
 212         }
 213
 214         // These need to passed onto the frame for other
 215         mlt_properties_pass_list( frame_properties, second_frame_properties,
 216                         "motion_est.left_mb, motion_est.right_mb, \
 217                         motion_est.top_mb, motion_est.bottom_mb, \
 218                         motion_est.macroblock_width, motion_est.macroblock_height" );
 219
 220         // Pass the pointer to the vectors without serializing
 221         mlt_properties_set_data( frame_properties, "motion_est.vectors",
 222                                 mlt_properties_get_data( second_frame_properties, "motion_est.vectors", NULL ),
 223                                 0, NULL, NULL );
 224
 225
 226         // Start with a base image
 227         memcpy( output, first_image, size );
 228
 229         if( mlt_properties_get_int( producer_properties, "method" ) == 1 ) {
 230
 231                 mlt_position first_position = mlt_frame_get_position( first_frame );
 232                 double actual_position = mlt_producer_get_speed( producer ) * (double)mlt_frame_get_position( this );
 233                 double scale = actual_position - first_position;
 234
 235                 motion_interpolate
 236                 (
 237                         first_image, second_image, output,
 238                         mlt_properties_get_int( second_frame_properties, "motion_est.top_mb" ),
 239                         mlt_properties_get_int( second_frame_properties, "motion_est.bottom_mb" ),
 240                         mlt_properties_get_int( second_frame_properties, "motion_est.left_mb" ),
 241                         mlt_properties_get_int( second_frame_properties, "motion_est.right_mb" ),
 242                         mlt_properties_get_int( second_frame_properties, "motion_est.macroblock_width" ),
 243                         mlt_properties_get_int( second_frame_properties, "motion_est.macroblock_height" ),
 244                         *width, *height,
 245                         xstride, ystride,
 246                         scale,
 247                         mlt_properties_get_data( second_frame_properties, "motion_est.vectors", NULL )
 248                 );
 249
 250                 if( mlt_properties_get_int( producer_properties, "debug" ) == 1 ) {
 251                         mlt_filter watermark = mlt_properties_get_data( producer_properties, "watermark", NULL );
 252
 253                         if( watermark == NULL ) {
 254                                 mlt_profile profile = mlt_service_profile( MLT_PRODUCER_SERVICE( producer ) );
 255                                 watermark = mlt_factory_filter( profile, "watermark", NULL );
 256                                 mlt_properties_set_data( producer_properties, "watermark", watermark, 0, (mlt_destructor)mlt_filter_close, NULL );
 257                                 mlt_producer_attach( producer, watermark );
 258                         }
 259
 260                         mlt_properties wm_properties = MLT_FILTER_PROPERTIES( watermark );
 261
 262                         char disp[30];
 263                         sprintf(disp, "+%10.2f.txt", actual_position);
 264                         mlt_properties_set( wm_properties, "resource", disp );
 265
 266                 }
 267
 268         }
 269
 270         *image = output;
 271         mlt_properties_set_data( frame_properties, "image", output, size, NULL, NULL );
 272
 273         // Make sure that no further scaling is done
 274         mlt_properties_set( frame_properties, "rescale.interps", "none" );
 275         mlt_properties_set( frame_properties, "scale", "off" );
 276
 277         mlt_frame_close( first_frame );
 278         mlt_frame_close( second_frame );
 279
 280         return 0;
 281 }
 282
 283 static int slowmotion_get_frame( mlt_producer this, mlt_frame_ptr frame, int index )
 284 {
 285         // Construct a new frame
 286         *frame = mlt_frame_init( MLT_PRODUCER_SERVICE( this ) );
 287
 288         mlt_properties properties = MLT_PRODUCER_PROPERTIES(this);
 289
 290
 291         if( frame != NULL )
 292         {
 293
 294                 mlt_frame first_frame = mlt_properties_get_data( properties, "first_frame", NULL );
 295                 mlt_frame second_frame = mlt_properties_get_data( properties, "second_frame", NULL );
 296
 297                 mlt_position first_position = (first_frame != NULL) ? mlt_frame_get_position( first_frame ) : -1;
 298                 mlt_position second_position = (second_frame != NULL) ? mlt_frame_get_position( second_frame ) : -1;
 299
 300                 // Get the real producer
 301                 mlt_producer real_producer = mlt_properties_get_data( properties, "producer", NULL );
 302
 303                 // Our "in" needs to be the same, keep it so
 304                 mlt_properties_pass_list( MLT_PRODUCER_PROPERTIES( real_producer ), properties, "in" );
 305
 306                 // Calculate our positions
 307                 double actual_position = mlt_producer_get_speed( this ) * (double)mlt_producer_position( this );
 308                 mlt_position need_first = floor( actual_position );
 309                 mlt_position need_second = need_first + 1;
 310
 311                 if( need_first != first_position )
 312                 {
 313                         mlt_frame_close( first_frame );
 314                         first_position = -1;
 315                         first_frame = NULL;
 316                 }
 317
 318                 if( need_second != second_position)
 319                 {
 320                         mlt_frame_close( second_frame );
 321                         second_position = -1;
 322                         second_frame = NULL;
 323                 }
 324
 325                 if( first_frame == NULL )
 326                 {
 327                         // Seek the producer to the correct place
 328                         mlt_producer_seek( real_producer, need_first );
 329
 330                         // Get the frame
 331                         mlt_service_get_frame( MLT_PRODUCER_SERVICE( real_producer ), &first_frame, index );
 332                 }
 333
 334                 if( second_frame == NULL )
 335                 {
 336                         // Seek the producer to the correct place
 337                         mlt_producer_seek( real_producer, need_second );
 338
 339                         // Get the frame
 340                         mlt_service_get_frame( MLT_PRODUCER_SERVICE( real_producer ), &second_frame, index );
 341                 }
 342
 343                 // Make sure things are in their place
 344                 mlt_properties_set_data( properties, "first_frame", first_frame, 0, NULL, NULL );
 345                 mlt_properties_set_data( properties, "second_frame", second_frame, 0, NULL, NULL );
 346
 347                 mlt_properties_set_int( MLT_FRAME_PROPERTIES( *frame ), "test_image", 0 );
 348
 349                 // Stack the producer and producer's get image
 350                 mlt_frame_push_service( *frame, first_frame );
 351                 mlt_properties_inc_ref( MLT_FRAME_PROPERTIES( first_frame ) );
 352
 353                 mlt_frame_push_service( *frame, second_frame );
 354                 mlt_properties_inc_ref( MLT_FRAME_PROPERTIES( second_frame ) );
 355
 356                 mlt_frame_push_service( *frame, this );
 357                 mlt_frame_push_service( *frame, slowmotion_get_image );
 358
 359                 // Give the returned frame temporal identity
 360                 mlt_frame_set_position( *frame, mlt_producer_position( this ) );
 361         }
 362
 363         return 0;
 364 }
 365
 366 mlt_producer producer_slowmotion_init( mlt_profile profile, mlt_service_type type, const char *id, char *arg )
 367 {
 368         mlt_producer this = mlt_producer_new( );
 369
 370         // Wrap fezzik
 371         mlt_producer real_producer = mlt_factory_producer( profile, "fezzik", arg );
 372
 373         // We need to apply the motion estimation filter manually
 374         mlt_filter filter = mlt_factory_filter( profile, "motion_est", NULL );
 375
 376         if ( this != NULL && real_producer != NULL && filter != NULL)
 377         {
 378                 // attach the motion_est filter to the real producer
 379                 mlt_producer_attach( real_producer, filter );
 380
 381                 // Get the properties of this producer
 382                 mlt_properties properties = MLT_PRODUCER_PROPERTIES( this );
 383
 384                 // Fezzik normalised it for us already
 385                 mlt_properties_set_int( properties, "fezzik_normalised", 1);
 386
 387                 // Store the producer and fitler
 388                 mlt_properties_set_data( properties, "producer", real_producer, 0, ( mlt_destructor )mlt_producer_close, NULL );
 389                 mlt_properties_set_data( properties, "motion_est", filter, 0, ( mlt_destructor )mlt_filter_close, NULL );
 390                 mlt_properties_set_int( MLT_FILTER_PROPERTIES( filter ), "macroblock_width", 16 );
 391                 mlt_properties_set_int( MLT_FILTER_PROPERTIES( filter ), "macroblock_height", 16 );
 392                 mlt_properties_set_int( MLT_FILTER_PROPERTIES( filter ), "denoise", 0 );
 393
 394                 // Grap some stuff from the real_producer
 395                 mlt_properties_pass_list( properties, MLT_PRODUCER_PROPERTIES( real_producer ),
 396                                 "in, out, length, resource" );
 397
 398                 // Since we control the seeking, prevent it from seeking on its own
 399                 mlt_producer_set_speed( real_producer, 0 );
 400
 401                 //mlt_properties_set( properties, "method", "onefield" );
 402
 403                 // Override the get_frame method
 404                 this->get_frame = slowmotion_get_frame;
 405
 406         }
 407         else
 408         {
 409                 if ( this )
 410                         mlt_producer_close( this );
 411                 if ( real_producer )
 412                         mlt_producer_close( real_producer );
 413                 if ( filter )
 414                         mlt_filter_close( filter );
 415
 416                 this = NULL;
 417         }
 418         return this;
 419 }