git.sesse.net Git - vlc/blob - modules/video_filter/seamcarving.c

   1 /*****************************************************************************
   2  * seamcarving.c: "Seam Carving for Content-Aware Image Resizing"
   3  * Based on paper by Shai Avidan and Ariel Shamir.
   4  *****************************************************************************
   5  * Copyright (C) 2007 the VideoLAN team
   6  * $Id$
   7  *
   8  * Authors: Antoine Cellerier <dionoea -at- videolan -dot- org>
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License as published by
  12  * the Free Software Foundation; either version 2 of the License, or
  13  * (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU General Public License
  21  * along with this program; if not, write to the Free Software
  22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  23  *****************************************************************************/
  24
  25 /*****************************************************************************
  26  * Preamble
  27  *****************************************************************************/
  28
  29 #include <vlc/vlc.h>
  30 #include <vlc_sout.h>
  31 #include <vlc_vout.h>
  32
  33 #include "vlc_filter.h"
  34
  35 #include <assert.h>
  36
  37 /*****************************************************************************
  38  * Local prototypes
  39  *****************************************************************************/
  40 static int  Create    ( vlc_object_t * );
  41 static void Destroy   ( vlc_object_t * );
  42
  43 static picture_t *Filter( filter_t *, picture_t * );
  44 static int CropCallback( vlc_object_t *, char const *,
  45                          vlc_value_t, vlc_value_t,
  46                          void * );
  47
  48 static void FilterSeamCarving( filter_t *, picture_t *, picture_t * );
  49
  50 /*****************************************************************************
  51  * Module descriptor
  52  *****************************************************************************/
  53
  54 #define FILTER_PREFIX "seamcarving-"
  55
  56 vlc_module_begin();
  57     set_description( _("Seam Carving video filter") );
  58     set_shortname( _( "Seam Carvinf" ));
  59     set_capability( "video filter2", 0 );
  60     set_category( CAT_VIDEO );
  61     set_subcategory( SUBCAT_VIDEO_VFILTER );
  62
  63     set_callbacks( Create, Destroy );
  64 vlc_module_end();
  65
  66 static const char *ppsz_filter_options[] = {
  67     NULL
  68 };
  69
  70 struct filter_sys_t
  71 {
  72     int *p_energy;
  73     int *p_grad;
  74
  75     int i_crop;
  76 };
  77
  78 static int Create( vlc_object_t *p_this )
  79 {
  80     filter_t *p_filter = (filter_t *)p_this;
  81
  82     /* Allocate structure */
  83     p_filter->p_sys = malloc( sizeof( filter_sys_t ) );
  84     if( p_filter->p_sys == NULL )
  85     {
  86         msg_Err( p_filter, "out of memory" );
  87         return VLC_ENOMEM;
  88     }
  89
  90     p_filter->pf_video_filter = Filter;
  91     p_filter->p_sys->p_energy = NULL;
  92     p_filter->p_sys->p_grad = NULL;
  93     p_filter->p_sys->i_crop = 0;
  94
  95     config_ChainParse( p_filter, FILTER_PREFIX, ppsz_filter_options,
  96                    p_filter->p_cfg );
  97
  98     var_Create( p_filter, "crop", VLC_VAR_INTEGER|VLC_VAR_ISCOMMAND );
  99     var_AddCallback( p_filter, "crop", CropCallback, p_filter->p_sys );
 100
 101     return VLC_SUCCESS;
 102 }
 103
 104 static void Destroy( vlc_object_t *p_this )
 105 {
 106     filter_t *p_filter = (filter_t *)p_this;
 107
 108     free( p_filter->p_sys->p_energy );
 109     free( p_filter->p_sys->p_grad );
 110
 111     free( p_filter->p_sys );
 112 }
 113
 114 static picture_t *Filter( filter_t *p_filter, picture_t *p_pic )
 115 {
 116     picture_t *p_outpic;
 117
 118     if( !p_pic ) return NULL;
 119
 120     p_outpic = p_filter->pf_vout_buffer_new( p_filter );
 121     if( !p_outpic )
 122     {
 123         msg_Warn( p_filter, "can't get output picture" );
 124         if( p_pic->pf_release )
 125             p_pic->pf_release( p_pic );
 126         return NULL;
 127     }
 128
 129     FilterSeamCarving( p_filter, p_pic, p_outpic );
 130
 131     p_outpic->date = p_pic->date;
 132     p_outpic->b_force = p_pic->b_force;
 133     p_outpic->i_nb_fields = p_pic->i_nb_fields;
 134     p_outpic->b_progressive = p_pic->b_progressive;
 135     p_outpic->b_top_field_first = p_pic->b_top_field_first;
 136
 137     if( p_pic->pf_release )
 138         p_pic->pf_release( p_pic );
 139
 140     return p_outpic;
 141 }
 142
 143 static inline int min3( int a, int b, int c );
 144 static inline int min( int a, int b );
 145 static int RemoveVerticalSeam( filter_t *p_filter, picture_t *p_inpic, picture_t *p_outpic, int i_src_visible );
 146
 147 //#define DRAW_GRADIENT
 148 //#define DRAW_ENERGY
 149 //#define DRAW_SEAM
 150
 151 static void FilterSeamCarving( filter_t *p_filter, picture_t *p_inpic,
 152                                                 picture_t *p_outpic )
 153 {
 154     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
 155     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
 156
 157     int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
 158
 159     if( !p_filter->p_sys->p_energy )
 160         p_filter->p_sys->p_energy = (int*)malloc(i_src_pitch * i_num_lines * sizeof(int));
 161     if( !p_filter->p_sys->p_grad )
 162         p_filter->p_sys->p_grad = (int*)malloc(i_src_pitch * i_num_lines * sizeof(int));
 163
 164 //#if defined( DRAW_GRADIENT ) || defined( DRAW_ENERGY ) || defined( DRAW_SEAM )
 165     p_filter->p_libvlc->pf_memcpy( p_outpic->p[Y_PLANE].p_pixels,
 166                                    p_inpic->p[Y_PLANE].p_pixels,
 167         p_outpic->p[Y_PLANE].i_lines * p_outpic->p[Y_PLANE].i_pitch );
 168 //#else
 169 //    p_filter->p_libvlc->pf_memset( p_outpix, 0x80,
 170 //        p_outpic->p[Y_PLANE].i_lines * p_outpic->p[Y_PLANE].i_pitch );
 171 //#endif
 172     p_filter->p_libvlc->pf_memset( p_outpic->p[U_PLANE].p_pixels, 0x80,
 173         p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
 174     p_filter->p_libvlc->pf_memset( p_outpic->p[V_PLANE].p_pixels, 0x80,
 175         p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
 176
 177 #if defined( DRAW_GRADIENT ) || defined( DRAW_ENERGY ) || defined( DRAW_SEAM )
 178     i_src_visible = RemoveVerticalSeam( p_filter, p_outpic, p_outpic, i_src_visible );
 179 #else
 180     static int j = 1;
 181     static int k = 1;
 182     int i;
 183     if( p_filter->p_sys->i_crop != 0 )
 184         j = p_filter->p_sys->i_crop;
 185     for( i = 0; i < j; i++ )
 186     i_src_visible = RemoveVerticalSeam( p_filter, p_outpic, p_outpic, i_src_visible );
 187     int y;
 188     for( y = 0; y < p_outpic->p[Y_PLANE].i_lines; y++ )
 189         p_filter->p_libvlc->pf_memset( p_outpic->p[Y_PLANE].p_pixels + y*p_outpic->p[Y_PLANE].i_pitch + i_src_visible, 0x00, p_outpic->p[Y_PLANE].i_pitch - i_src_visible );
 190     j += k;
 191     if( j == 100 ) k = -1;
 192     if( j == 1 ) k = 1;
 193 #endif
 194 }
 195
 196 static int ComputeGradient( filter_t *p_filter, picture_t *p_inpic, int i_src_visible )
 197 {
 198     int x, y;
 199     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
 200     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
 201
 202     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
 203     int *p_grad = p_filter->p_sys->p_grad;
 204
 205     for( y = 1; y < i_num_lines - 1;
 206          y++, p_grad += i_src_pitch )
 207     {
 208         /* Compute line y's gradient */
 209 #define GRADx( x ) \
 210             ( \
 211               abs( \
 212                  ( p_inpix[(y-1)*i_src_pitch+x-1] \
 213                    - p_inpix[(y+1)*i_src_pitch+x-1] ) \
 214                + ( ( p_inpix[(y-1)*i_src_pitch+x] \
 215                     - p_inpix[(y+1)*i_src_pitch+x] ) <<1 ) \
 216                + ( p_inpix[(y-1)*i_src_pitch+x+1] \
 217                    - p_inpix[(y+1)*i_src_pitch+x+1] ) \
 218               ) \
 219             + \
 220               abs( \
 221                  ( p_inpix[(y-1)*i_src_pitch+x-1] \
 222                    - p_inpix[(y-1)*i_src_pitch+x+1] ) \
 223                + ( ( p_inpix[y*i_src_pitch+x-1] \
 224                     - p_inpix[y*i_src_pitch+x+1] ) <<1 ) \
 225                + ( p_inpix[(y+1)*i_src_pitch+x-1] \
 226                    - p_inpix[(y+1)*i_src_pitch+x+1] ) \
 227               ) \
 228             )
 229         for( x = 1; x < i_src_visible - 1; x++ )
 230         {
 231             p_grad[x] = GRADx( x );
 232 #ifdef DRAW_GRADIENT
 233             p_outpix[y*i_src_pitch+x] = p_grad[x]>>3;
 234 #endif
 235         }
 236     }
 237     return 0;
 238 }
 239
 240 static int RemoveVerticalSeam( filter_t *p_filter, picture_t *p_inpic, picture_t *p_outpic, int i_src_visible )
 241 {
 242     int x, y;
 243     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
 244     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
 245
 246     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
 247     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
 248
 249     int *p_energy = p_filter->p_sys->p_energy;
 250     int *p_grad = p_filter->p_sys->p_grad;
 251
 252     ComputeGradient( p_filter, p_inpic, i_src_visible );
 253
 254     /** Compute the image's energy (using a sobel gradient as the base energy
 255      ** function) */
 256     /* Set the first energy line to 0 */
 257     memset( p_energy, 0, i_src_pitch*sizeof(int));
 258
 259     int *p_energy_prev = p_energy;
 260     for( y = 1; y < i_num_lines - 1;
 261          y++, p_energy_prev = p_energy, p_energy += i_src_pitch,
 262          p_grad += i_src_pitch )
 263     {
 264         /* Compute line y's minimum energy value for paths ending on
 265          * each x */
 266         x = 1;
 267         p_energy[x] = min( p_energy_prev[x  ]+p_grad[x  ],
 268                            p_energy_prev[x+1]+p_grad[x+1] );
 269         for( x = 2; x < i_src_visible - 2; x++ )
 270         {
 271             p_energy[x] = min3( p_energy_prev[x-1]+p_grad[x-1],
 272                                 p_energy_prev[x  ]+p_grad[x  ],
 273                                 p_energy_prev[x+1]+p_grad[x+1] );
 274         }
 275         p_energy[x] = min( p_energy_prev[x-1]+p_grad[x-1],
 276                            p_energy_prev[x  ]+p_grad[x  ] );
 277
 278 #ifdef DRAW_ENERGY
 279         int max = p_energy[1];
 280         for( x = 1; x < i_src_visible - 1; x++ )
 281             if( p_energy[x] > max ) max = p_energy[x];
 282         for( x = 1; x < i_src_visible - 1; x++ )
 283             p_outpix[y*i_src_pitch+x] = p_energy[x]*0xff/max;
 284 #endif
 285     }
 286
 287     /* Find the minimum energy point on the last line */
 288     y--;
 289     p_energy -= i_src_pitch;
 290     p_grad -= i_src_pitch;
 291
 292     int m = p_energy[1];
 293     int xmin = 1;
 294     for( x = 1; x < i_src_visible - 1; x++ )
 295     {
 296         if( p_energy[x] < m )
 297         {
 298             m = p_energy[x];
 299             xmin = x;
 300         }
 301     }
 302
 303 #ifdef DRAW_SEAM
 304     p_outpix[y*i_src_pitch+xmin] = 0xff;
 305     p_outpix[(y+1)*i_src_pitch+xmin] = 0xff;
 306 #else
 307     memmove( p_outpix+y*i_src_pitch+xmin, p_outpix+y*i_src_pitch+xmin+1, i_src_pitch-(xmin+1) );
 308     memmove( p_outpix+(y+1)*i_src_pitch+xmin, p_outpix+(y+1)*i_src_pitch+xmin+1, i_src_pitch-(xmin+1) );
 309 #endif
 310
 311     p_energy -= i_src_pitch;
 312     for( ; y>1; y--, p_energy -= i_src_pitch, p_grad -= i_src_pitch )
 313     {
 314         if( m != p_energy[xmin]+p_grad[xmin] )
 315         {
 316             if( xmin > 1 && m == p_energy[xmin-1]+p_grad[xmin-1] )
 317             {
 318                 xmin--;
 319             }
 320             else if( xmin < i_src_visible - 2 && m == p_energy[xmin+1]+p_grad[xmin+1] )
 321             {
 322                 xmin++;
 323             }
 324             else
 325             {
 326                 printf("Alarm! %d\n" ,y);
 327                 //assert( 0 );
 328             }
 329         }
 330         m = p_energy[xmin];
 331 #ifdef DRAW_SEAM
 332         p_outpix[y*i_src_pitch+xmin] = 0xff;
 333 #else
 334         memmove( p_outpix+y*i_src_pitch+xmin, p_outpix+y*i_src_pitch+xmin+1, i_src_pitch-(xmin+1) );
 335 #endif
 336     }
 337 #ifdef DRAW_SEAM
 338     p_outpix[y*i_src_pitch+xmin] = 0xff;
 339 #else
 340     memmove( p_outpix+y*i_src_pitch+xmin, p_outpix+y*i_src_pitch+xmin+1, i_src_pitch-(xmin+1) );
 341 #endif
 342     y--;
 343 #ifdef DRAW_SEAM
 344     p_outpix[y*i_src_pitch+xmin] = 0xff;
 345 #else
 346     memmove( p_outpix+y*i_src_pitch+xmin, p_outpix+y*i_src_pitch+xmin+1, i_src_pitch-(xmin+1) );
 347 #endif
 348
 349 #if defined( DRAW_SEAM )
 350     return i_src_visible;
 351 #else
 352     return i_src_visible-1;
 353 #endif
 354 }
 355
 356 static inline int min3( int a, int b, int c )
 357 {
 358     if( a < b )
 359     {
 360         if( a < c ) return a;
 361         return c;
 362     }
 363     if( b < c ) return b;
 364     return c;
 365 }
 366 static inline int min( int a, int b )
 367 {
 368     return a < b ? a : b;
 369 }
 370
 371 static int CropCallback( vlc_object_t *p_this, char const *psz_var,
 372                                 vlc_value_t oldval, vlc_value_t newval,
 373                                 void *p_data )
 374 {
 375     filter_sys_t *p_sys = (filter_sys_t *)p_data;
 376     p_sys->i_crop = newval.i_int;
 377     return VLC_SUCCESS;
 378 }