git.sesse.net Git - vlc/blob - modules/video_filter/seamcarving.c

   1 /*****************************************************************************
   2  * seamcarving.c: "Seam Carving for Content-Aware Image Resizing"
   3  * Based on paper by Shai Avidan and Ariel Shamir.
   4  *****************************************************************************
   5  * Copyright (C) 2007 the VideoLAN team
   6  * $Id$
   7  *
   8  * Authors: Antoine Cellerier <dionoea -at- videolan -dot- org>
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License as published by
  12  * the Free Software Foundation; either version 2 of the License, or
  13  * (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU General Public License
  21  * along with this program; if not, write to the Free Software
  22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  23  *****************************************************************************/
  24
  25 /*****************************************************************************
  26  * Preamble
  27  *****************************************************************************/
  28
  29 #ifdef HAVE_CONFIG_H
  30 # include "config.h"
  31 #endif
  32
  33 #include <vlc/vlc.h>
  34 #include <vlc_plugin.h>
  35 #include <vlc_sout.h>
  36 #include <vlc_vout.h>
  37
  38 #include "vlc_filter.h"
  39
  40 #include <assert.h>
  41
  42 /*****************************************************************************
  43  * Local prototypes
  44  *****************************************************************************/
  45 static int  Create    ( vlc_object_t * );
  46 static void Destroy   ( vlc_object_t * );
  47
  48 static picture_t *Filter( filter_t *, picture_t * );
  49 static int CropCallback( vlc_object_t *, char const *,
  50                          vlc_value_t, vlc_value_t,
  51                          void * );
  52
  53 static void FilterSeamCarving( filter_t *, picture_t *, picture_t * );
  54
  55 /*****************************************************************************
  56  * Module descriptor
  57  *****************************************************************************/
  58
  59 #define FILTER_PREFIX "seamcarving-"
  60
  61 vlc_module_begin();
  62     set_description( N_("Seam Carving video filter") );
  63     set_shortname( N_( "Seam Carving" ));
  64     set_capability( "video filter2", 0 );
  65     set_category( CAT_VIDEO );
  66     set_subcategory( SUBCAT_VIDEO_VFILTER );
  67
  68     set_callbacks( Create, Destroy );
  69 vlc_module_end();
  70
  71 struct filter_sys_t
  72 {
  73     int *p_energy;
  74     int *p_grad;
  75
  76     int i_crop;
  77 };
  78
  79 static int Create( vlc_object_t *p_this )
  80 {
  81     filter_t *p_filter = (filter_t *)p_this;
  82
  83     /* Allocate structure */
  84     p_filter->p_sys = malloc( sizeof( filter_sys_t ) );
  85     if( p_filter->p_sys == NULL )
  86     {
  87         msg_Err( p_filter, "out of memory" );
  88         return VLC_ENOMEM;
  89     }
  90
  91     p_filter->pf_video_filter = Filter;
  92     p_filter->p_sys->p_energy = NULL;
  93     p_filter->p_sys->p_grad = NULL;
  94     p_filter->p_sys->i_crop = 0;
  95
  96     var_Create( p_filter, "crop", VLC_VAR_INTEGER|VLC_VAR_ISCOMMAND );
  97     var_AddCallback( p_filter, "crop", CropCallback, p_filter->p_sys );
  98
  99     return VLC_SUCCESS;
 100 }
 101
 102 static void Destroy( vlc_object_t *p_this )
 103 {
 104     filter_t *p_filter = (filter_t *)p_this;
 105
 106     free( p_filter->p_sys->p_energy );
 107     free( p_filter->p_sys->p_grad );
 108
 109     free( p_filter->p_sys );
 110 }
 111
 112 static picture_t *Filter( filter_t *p_filter, picture_t *p_pic )
 113 {
 114     picture_t *p_outpic;
 115
 116     if( !p_pic ) return NULL;
 117
 118     p_outpic = p_filter->pf_vout_buffer_new( p_filter );
 119     if( !p_outpic )
 120     {
 121         msg_Warn( p_filter, "can't get output picture" );
 122         if( p_pic->pf_release )
 123             p_pic->pf_release( p_pic );
 124         return NULL;
 125     }
 126
 127     FilterSeamCarving( p_filter, p_pic, p_outpic );
 128
 129     p_outpic->date = p_pic->date;
 130     p_outpic->b_force = p_pic->b_force;
 131     p_outpic->i_nb_fields = p_pic->i_nb_fields;
 132     p_outpic->b_progressive = p_pic->b_progressive;
 133     p_outpic->b_top_field_first = p_pic->b_top_field_first;
 134
 135     if( p_pic->pf_release )
 136         p_pic->pf_release( p_pic );
 137
 138     return p_outpic;
 139 }
 140
 141 static inline int my_min3( int a, int b, int c );
 142 static inline int my_min( int a, int b );
 143 static int RemoveVerticalSeam( filter_t *p_filter, picture_t *p_inpic, picture_t *p_outpic, int i_src_visible );
 144
 145 //#define DRAW_GRADIENT
 146 //#define DRAW_ENERGY
 147 //#define DRAW_SEAM
 148
 149 static void FilterSeamCarving( filter_t *p_filter, picture_t *p_inpic,
 150                                                 picture_t *p_outpic )
 151 {
 152     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
 153     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
 154
 155     int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
 156
 157     if( !p_filter->p_sys->p_energy )
 158         p_filter->p_sys->p_energy = (int*)malloc(i_src_pitch * i_num_lines * sizeof(int));
 159     if( !p_filter->p_sys->p_grad )
 160         p_filter->p_sys->p_grad = (int*)malloc(i_src_pitch * i_num_lines * sizeof(int));
 161
 162 //#if defined( DRAW_GRADIENT ) || defined( DRAW_ENERGY ) || defined( DRAW_SEAM )
 163     vlc_memcpy( p_outpic->p[Y_PLANE].p_pixels, p_inpic->p[Y_PLANE].p_pixels,
 164         p_outpic->p[Y_PLANE].i_lines * p_outpic->p[Y_PLANE].i_pitch );
 165 //#else
 166 //    vlc_memset( p_outpix, 0x80,
 167 //        p_outpic->p[Y_PLANE].i_lines * p_outpic->p[Y_PLANE].i_pitch );
 168 //#endif
 169     vlc_memset( p_outpic->p[U_PLANE].p_pixels, 0x80,
 170         p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
 171     vlc_memset( p_outpic->p[V_PLANE].p_pixels, 0x80,
 172         p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
 173
 174 #if defined( DRAW_GRADIENT ) || defined( DRAW_ENERGY ) || defined( DRAW_SEAM )
 175     i_src_visible = RemoveVerticalSeam( p_filter, p_outpic, p_outpic, i_src_visible );
 176 #else
 177     static int j = 1;
 178     static int k = 1;
 179     int i;
 180     if( p_filter->p_sys->i_crop != 0 )
 181         j = p_filter->p_sys->i_crop;
 182     for( i = 0; i < j; i++ )
 183     i_src_visible = RemoveVerticalSeam( p_filter, p_outpic, p_outpic, i_src_visible );
 184     int y;
 185     for( y = 0; y < p_outpic->p[Y_PLANE].i_lines; y++ )
 186         vlc_memset( p_outpic->p[Y_PLANE].p_pixels + y*p_outpic->p[Y_PLANE].i_pitch + i_src_visible, 0x00, p_outpic->p[Y_PLANE].i_pitch - i_src_visible );
 187     j += k;
 188     if( j == 100 ) k = -1;
 189     if( j == 1 ) k = 1;
 190 #endif
 191 }
 192
 193 static int ComputeGradient( filter_t *p_filter, picture_t *p_inpic, int i_src_visible )
 194 {
 195     int x, y;
 196     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
 197     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
 198
 199     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
 200     int *p_grad = p_filter->p_sys->p_grad;
 201
 202     for( y = 1; y < i_num_lines - 1;
 203          y++, p_grad += i_src_pitch )
 204     {
 205         /* Compute line y's gradient */
 206 #define GRADx( x ) \
 207             ( \
 208               abs( \
 209                  ( p_inpix[(y-1)*i_src_pitch+x-1] \
 210                    - p_inpix[(y+1)*i_src_pitch+x-1] ) \
 211                + ( ( p_inpix[(y-1)*i_src_pitch+x] \
 212                     - p_inpix[(y+1)*i_src_pitch+x] ) <<1 ) \
 213                + ( p_inpix[(y-1)*i_src_pitch+x+1] \
 214                    - p_inpix[(y+1)*i_src_pitch+x+1] ) \
 215               ) \
 216             + \
 217               abs( \
 218                  ( p_inpix[(y-1)*i_src_pitch+x-1] \
 219                    - p_inpix[(y-1)*i_src_pitch+x+1] ) \
 220                + ( ( p_inpix[y*i_src_pitch+x-1] \
 221                     - p_inpix[y*i_src_pitch+x+1] ) <<1 ) \
 222                + ( p_inpix[(y+1)*i_src_pitch+x-1] \
 223                    - p_inpix[(y+1)*i_src_pitch+x+1] ) \
 224               ) \
 225             )
 226         for( x = 1; x < i_src_visible - 1; x++ )
 227         {
 228             p_grad[x] = GRADx( x );
 229 #ifdef DRAW_GRADIENT
 230             p_outpix[y*i_src_pitch+x] = p_grad[x]>>3;
 231 #endif
 232         }
 233     }
 234     return 0;
 235 }
 236
 237 static int RemoveVerticalSeam( filter_t *p_filter, picture_t *p_inpic, picture_t *p_outpic, int i_src_visible )
 238 {
 239     int x, y;
 240     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
 241     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
 242
 243     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
 244
 245     int *p_energy = p_filter->p_sys->p_energy;
 246     int *p_grad = p_filter->p_sys->p_grad;
 247
 248     ComputeGradient( p_filter, p_inpic, i_src_visible );
 249
 250     /** Compute the image's energy (using a sobel gradient as the base energy
 251      ** function) */
 252     /* Set the first energy line to 0 */
 253     memset( p_energy, 0, i_src_pitch*sizeof(int));
 254
 255     int *p_energy_prev = p_energy;
 256     for( y = 1; y < i_num_lines - 1;
 257          y++, p_energy_prev = p_energy, p_energy += i_src_pitch,
 258          p_grad += i_src_pitch )
 259     {
 260         /* Compute line y's minimum energy value for paths ending on
 261          * each x */
 262         x = 1;
 263         p_energy[x] = my_min( p_energy_prev[x  ]+p_grad[x  ],
 264                            p_energy_prev[x+1]+p_grad[x+1] );
 265         for( x = 2; x < i_src_visible - 2; x++ )
 266         {
 267             p_energy[x] = my_min3( p_energy_prev[x-1]+p_grad[x-1],
 268                                 p_energy_prev[x  ]+p_grad[x  ],
 269                                 p_energy_prev[x+1]+p_grad[x+1] );
 270         }
 271         p_energy[x] = my_min( p_energy_prev[x-1]+p_grad[x-1],
 272                            p_energy_prev[x  ]+p_grad[x  ] );
 273
 274 #ifdef DRAW_ENERGY
 275         int max = p_energy[1];
 276         for( x = 1; x < i_src_visible - 1; x++ )
 277             if( p_energy[x] > max ) max = p_energy[x];
 278         for( x = 1; x < i_src_visible - 1; x++ )
 279             p_outpix[y*i_src_pitch+x] = p_energy[x]*0xff/max;
 280 #endif
 281     }
 282
 283     /* Find the minimum energy point on the last line */
 284     y--;
 285     p_energy -= i_src_pitch;
 286     p_grad -= i_src_pitch;
 287
 288     int m = p_energy[1];
 289     int xmin = 1;
 290     for( x = 1; x < i_src_visible - 1; x++ )
 291     {
 292         if( p_energy[x] < m )
 293         {
 294             m = p_energy[x];
 295             xmin = x;
 296         }
 297     }
 298
 299 #ifdef DRAW_SEAM
 300     p_outpix[y*i_src_pitch+xmin] = 0xff;
 301     p_outpix[(y+1)*i_src_pitch+xmin] = 0xff;
 302 #else
 303     memmove( p_outpix+y*i_src_pitch+xmin, p_outpix+y*i_src_pitch+xmin+1, i_src_pitch-(xmin+1) );
 304     memmove( p_outpix+(y+1)*i_src_pitch+xmin, p_outpix+(y+1)*i_src_pitch+xmin+1, i_src_pitch-(xmin+1) );
 305 #endif
 306
 307     p_energy -= i_src_pitch;
 308     for( ; y>1; y--, p_energy -= i_src_pitch, p_grad -= i_src_pitch )
 309     {
 310         if( m != p_energy[xmin]+p_grad[xmin] )
 311         {
 312             if( xmin > 1 && m == p_energy[xmin-1]+p_grad[xmin-1] )
 313             {
 314                 xmin--;
 315             }
 316             else if( xmin < i_src_visible - 2 && m == p_energy[xmin+1]+p_grad[xmin+1] )
 317             {
 318                 xmin++;
 319             }
 320             else
 321             {
 322                 printf("Alarm! %d\n" ,y);
 323                 //assert( 0 );
 324             }
 325         }
 326         m = p_energy[xmin];
 327 #ifdef DRAW_SEAM
 328         p_outpix[y*i_src_pitch+xmin] = 0xff;
 329 #else
 330         memmove( p_outpix+y*i_src_pitch+xmin, p_outpix+y*i_src_pitch+xmin+1, i_src_pitch-(xmin+1) );
 331 #endif
 332     }
 333 #ifdef DRAW_SEAM
 334     p_outpix[y*i_src_pitch+xmin] = 0xff;
 335 #else
 336     memmove( p_outpix+y*i_src_pitch+xmin, p_outpix+y*i_src_pitch+xmin+1, i_src_pitch-(xmin+1) );
 337 #endif
 338     y--;
 339 #ifdef DRAW_SEAM
 340     p_outpix[y*i_src_pitch+xmin] = 0xff;
 341 #else
 342     memmove( p_outpix+y*i_src_pitch+xmin, p_outpix+y*i_src_pitch+xmin+1, i_src_pitch-(xmin+1) );
 343 #endif
 344
 345 #if defined( DRAW_SEAM )
 346     return i_src_visible;
 347 #else
 348     return i_src_visible-1;
 349 #endif
 350 }
 351
 352 static inline int my_min3( int a, int b, int c )
 353 {
 354     if( a < b )
 355     {
 356         if( a < c ) return a;
 357         return c;
 358     }
 359     if( b < c ) return b;
 360     return c;
 361 }
 362 static inline int my_min( int a, int b )
 363 {
 364     return a < b ? a : b;
 365 }
 366
 367 static int CropCallback( vlc_object_t *p_this, char const *psz_var,
 368                                 vlc_value_t oldval, vlc_value_t newval,
 369                                 void *p_data )
 370 {
 371     VLC_UNUSED(p_this); VLC_UNUSED(psz_var); VLC_UNUSED(oldval);
 372     filter_sys_t *p_sys = (filter_sys_t *)p_data;
 373     p_sys->i_crop = newval.i_int;
 374     return VLC_SUCCESS;
 375 }