git.sesse.net Git - vlc/blob - modules/video_filter/gradient.c

   1 /*****************************************************************************
   2  * gradient.c : Gradient and edge detection video effects plugin for vlc
   3  *****************************************************************************
   4  * Copyright (C) 2000-2008 the VideoLAN team
   5  * $Id$
   6  *
   7  * Authors: Samuel Hocevar <sam@zoy.org>
   8  *          Antoine Cellerier <dionoea -at- videolan -dot- org>
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License as published by
  12  * the Free Software Foundation; either version 2 of the License, or
  13  * (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU General Public License
  21  * along with this program; if not, write to the Free Software
  22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  23  *****************************************************************************/
  24
  25 /*****************************************************************************
  26  * Preamble
  27  *****************************************************************************/
  28
  29 #ifdef HAVE_CONFIG_H
  30 # include "config.h"
  31 #endif
  32
  33 #include <math.h>                                            /* sin(), cos() */
  34
  35 #include <vlc_common.h>
  36 #include <vlc_plugin.h>
  37 #include <vlc_sout.h>
  38 #include <vlc_vout.h>
  39
  40 #include "vlc_filter.h"
  41 #include "filter_picture.h"
  42
  43 enum { GRADIENT, EDGE, HOUGH };
  44
  45 /*****************************************************************************
  46  * Local prototypes
  47  *****************************************************************************/
  48 static int  Create    ( vlc_object_t * );
  49 static void Destroy   ( vlc_object_t * );
  50
  51 static picture_t *Filter( filter_t *, picture_t * );
  52 static int GradientCallback( vlc_object_t *, char const *,
  53                              vlc_value_t, vlc_value_t,
  54                              void * );
  55
  56 static void FilterGradient( filter_t *, picture_t *, picture_t * );
  57 static void FilterEdge    ( filter_t *, picture_t *, picture_t * );
  58 static void FilterHough   ( filter_t *, picture_t *, picture_t * );
  59
  60 /*****************************************************************************
  61  * Module descriptor
  62  *****************************************************************************/
  63 #define MODE_TEXT N_("Distort mode")
  64 #define MODE_LONGTEXT N_("Distort mode, one of \"gradient\", \"edge\" and \"hough\".")
  65
  66 #define GRADIENT_TEXT N_("Gradient image type")
  67 #define GRADIENT_LONGTEXT N_("Gradient image type (0 or 1). 0 will " \
  68         "turn the image to white while 1 will keep colors." )
  69
  70 #define CARTOON_TEXT N_("Apply cartoon effect")
  71 #define CARTOON_LONGTEXT N_("Apply cartoon effect. It is only used by " \
  72     "\"gradient\" and \"edge\".")
  73
  74 static const char *const mode_list[] = { "gradient", "edge", "hough" };
  75 static const char *const mode_list_text[] = { N_("Gradient"), N_("Edge"), N_("Hough") };
  76
  77 #define FILTER_PREFIX "gradient-"
  78
  79 vlc_module_begin ()
  80     set_description( N_("Gradient video filter") )
  81     set_shortname( N_( "Gradient" ))
  82     set_capability( "video filter2", 0 )
  83     set_category( CAT_VIDEO )
  84     set_subcategory( SUBCAT_VIDEO_VFILTER )
  85
  86     add_string( FILTER_PREFIX "mode", "gradient", NULL,
  87                 MODE_TEXT, MODE_LONGTEXT, false )
  88         change_string_list( mode_list, mode_list_text, 0 )
  89
  90     add_integer_with_range( FILTER_PREFIX "type", 0, 0, 1, NULL,
  91                 GRADIENT_TEXT, GRADIENT_LONGTEXT, false )
  92     add_bool( FILTER_PREFIX "cartoon", 1, NULL,
  93                 CARTOON_TEXT, CARTOON_LONGTEXT, false )
  94
  95     add_shortcut( "gradient" )
  96     set_callbacks( Create, Destroy )
  97 vlc_module_end ()
  98
  99 static const char *const ppsz_filter_options[] = {
 100     "mode", "type", "cartoon", NULL
 101 };
 102
 103 /*****************************************************************************
 104  * filter_sys_t: Distort video output method descriptor
 105  *****************************************************************************
 106  * This structure is part of the video output thread descriptor.
 107  * It describes the Distort specific properties of an output thread.
 108  *****************************************************************************/
 109 struct filter_sys_t
 110 {
 111     vlc_mutex_t lock;
 112     int i_mode;
 113
 114     /* For the gradient mode */
 115     int i_gradient_type;
 116     bool b_cartoon;
 117
 118     uint32_t *p_buf32;
 119     uint32_t *p_buf32_bis;
 120     uint8_t *p_buf8;
 121
 122     /* For hough mode */
 123     int *p_pre_hough;
 124 };
 125
 126 /*****************************************************************************
 127  * Create: allocates Distort video thread output method
 128  *****************************************************************************
 129  * This function allocates and initializes a Distort vout method.
 130  *****************************************************************************/
 131 static int Create( vlc_object_t *p_this )
 132 {
 133     filter_t *p_filter = (filter_t *)p_this;
 134     char *psz_method;
 135
 136     switch( p_filter->fmt_in.video.i_chroma )
 137     {
 138         CASE_PLANAR_YUV
 139             break;
 140
 141         default:
 142              msg_Err( p_filter, "Unsupported input chroma (%4s)",
 143                       (char*)&(p_filter->fmt_in.video.i_chroma) );
 144             return VLC_EGENERIC;
 145     }
 146
 147     /* Allocate structure */
 148     p_filter->p_sys = malloc( sizeof( filter_sys_t ) );
 149     if( p_filter->p_sys == NULL )
 150         return VLC_ENOMEM;
 151
 152     p_filter->pf_video_filter = Filter;
 153
 154     p_filter->p_sys->p_pre_hough = NULL;
 155
 156     config_ChainParse( p_filter, FILTER_PREFIX, ppsz_filter_options,
 157                    p_filter->p_cfg );
 158
 159     if( !(psz_method =
 160         var_CreateGetNonEmptyStringCommand( p_filter, FILTER_PREFIX "mode" )) )
 161     {
 162         msg_Err( p_filter, "configuration variable "
 163                  FILTER_PREFIX "mode empty" );
 164         p_filter->p_sys->i_mode = GRADIENT;
 165     }
 166     else
 167     {
 168         if( !strcmp( psz_method, "gradient" ) )
 169         {
 170             p_filter->p_sys->i_mode = GRADIENT;
 171         }
 172         else if( !strcmp( psz_method, "edge" ) )
 173         {
 174             p_filter->p_sys->i_mode = EDGE;
 175         }
 176         else if( !strcmp( psz_method, "hough" ) )
 177         {
 178             p_filter->p_sys->i_mode = HOUGH;
 179         }
 180         else
 181         {
 182             msg_Err( p_filter, "no valid gradient mode provided (%s)", psz_method );
 183             p_filter->p_sys->i_mode = GRADIENT;
 184         }
 185     }
 186     free( psz_method );
 187
 188     p_filter->p_sys->i_gradient_type =
 189         var_CreateGetIntegerCommand( p_filter, FILTER_PREFIX "type" );
 190     p_filter->p_sys->b_cartoon =
 191         var_CreateGetBoolCommand( p_filter, FILTER_PREFIX "cartoon" );
 192
 193     vlc_mutex_init( &p_filter->p_sys->lock );
 194     var_AddCallback( p_filter, FILTER_PREFIX "mode",
 195                      GradientCallback, p_filter->p_sys );
 196     var_AddCallback( p_filter, FILTER_PREFIX "type",
 197                      GradientCallback, p_filter->p_sys );
 198     var_AddCallback( p_filter, FILTER_PREFIX "cartoon",
 199                      GradientCallback, p_filter->p_sys );
 200
 201     p_filter->p_sys->p_buf32 = NULL;
 202     p_filter->p_sys->p_buf32_bis = NULL;
 203     p_filter->p_sys->p_buf8 = NULL;
 204
 205     return VLC_SUCCESS;
 206 }
 207
 208 /*****************************************************************************
 209  * Destroy: destroy Distort video thread output method
 210  *****************************************************************************
 211  * Terminate an output method created by DistortCreateOutputMethod
 212  *****************************************************************************/
 213 static void Destroy( vlc_object_t *p_this )
 214 {
 215     filter_t *p_filter = (filter_t *)p_this;
 216     filter_sys_t *p_sys = p_filter->p_sys;
 217
 218     var_DelCallback( p_filter, FILTER_PREFIX "mode",
 219                      GradientCallback, p_sys );
 220     var_DelCallback( p_filter, FILTER_PREFIX "type",
 221                      GradientCallback, p_sys );
 222     var_DelCallback( p_filter, FILTER_PREFIX "cartoon",
 223                      GradientCallback, p_sys );
 224     vlc_mutex_destroy( &p_sys->lock );
 225
 226     free( p_sys->p_buf32 );
 227     free( p_sys->p_buf32_bis );
 228     free( p_sys->p_buf8 );
 229     free( p_sys->p_pre_hough );
 230
 231     free( p_sys );
 232 }
 233
 234 /*****************************************************************************
 235  * Render: displays previously rendered output
 236  *****************************************************************************
 237  * This function send the currently rendered image to Distort image, waits
 238  * until it is displayed and switch the two rendering buffers, preparing next
 239  * frame.
 240  *****************************************************************************/
 241 static picture_t *Filter( filter_t *p_filter, picture_t *p_pic )
 242 {
 243     picture_t *p_outpic;
 244
 245     if( !p_pic ) return NULL;
 246
 247     p_outpic = filter_NewPicture( p_filter );
 248     if( !p_outpic )
 249     {
 250         picture_Release( p_pic );
 251         return NULL;
 252     }
 253
 254     vlc_mutex_lock( &p_filter->p_sys->lock );
 255     switch( p_filter->p_sys->i_mode )
 256     {
 257         case EDGE:
 258             FilterEdge( p_filter, p_pic, p_outpic );
 259             break;
 260
 261         case GRADIENT:
 262             FilterGradient( p_filter, p_pic, p_outpic );
 263             break;
 264
 265         case HOUGH:
 266             FilterHough( p_filter, p_pic, p_outpic );
 267             break;
 268
 269         default:
 270             break;
 271     }
 272     vlc_mutex_unlock( &p_filter->p_sys->lock );
 273
 274     return CopyInfoAndRelease( p_outpic, p_pic );
 275 }
 276
 277 /*****************************************************************************
 278  * Gaussian Convolution
 279  *****************************************************************************
 280  *    Gaussian convolution ( sigma == 1.4 )
 281  *
 282  *    |  2  4  5  4  2  |   |  2  4  4  4  2 |
 283  *    |  4  9 12  9  4  |   |  4  8 12  8  4 |
 284  *    |  5 12 15 12  5  | ~ |  4 12 16 12  4 |
 285  *    |  4  9 12  9  4  |   |  4  8 12  8  4 |
 286  *    |  2  4  5  4  2  |   |  2  4  4  4  2 |
 287  *****************************************************************************/
 288 static void GaussianConvolution( picture_t *p_inpic, uint32_t *p_smooth )
 289 {
 290     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
 291     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
 292     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
 293     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
 294
 295     int x,y;
 296     for( y = 2; y < i_num_lines - 2; y++ )
 297     {
 298         for( x = 2; x < i_src_visible - 2; x++ )
 299         {
 300             p_smooth[y*i_src_visible+x] = (uint32_t)(
 301               /* 2 rows up */
 302                 ( p_inpix[(y-2)*i_src_pitch+x-2] )
 303               + ((p_inpix[(y-2)*i_src_pitch+x-1]
 304               +   p_inpix[(y-2)*i_src_pitch+x]
 305               +   p_inpix[(y-2)*i_src_pitch+x+1])<<1 )
 306               + ( p_inpix[(y-2)*i_src_pitch+x+2] )
 307               /* 1 row up */
 308               + ((p_inpix[(y-1)*i_src_pitch+x-2]
 309               + ( p_inpix[(y-1)*i_src_pitch+x-1]<<1 )
 310               + ( p_inpix[(y-1)*i_src_pitch+x]*3 )
 311               + ( p_inpix[(y-1)*i_src_pitch+x+1]<<1 )
 312               +   p_inpix[(y-1)*i_src_pitch+x+2]
 313               /* */
 314               +   p_inpix[y*i_src_pitch+x-2]
 315               + ( p_inpix[y*i_src_pitch+x-1]*3 )
 316               + ( p_inpix[y*i_src_pitch+x]<<2 )
 317               + ( p_inpix[y*i_src_pitch+x+1]*3 )
 318               +   p_inpix[y*i_src_pitch+x+2]
 319               /* 1 row down */
 320               +   p_inpix[(y+1)*i_src_pitch+x-2]
 321               + ( p_inpix[(y+1)*i_src_pitch+x-1]<<1 )
 322               + ( p_inpix[(y+1)*i_src_pitch+x]*3 )
 323               + ( p_inpix[(y+1)*i_src_pitch+x+1]<<1 )
 324               +   p_inpix[(y+1)*i_src_pitch+x+2] )<<1 )
 325               /* 2 rows down */
 326               + ( p_inpix[(y+2)*i_src_pitch+x-2] )
 327               + ((p_inpix[(y+2)*i_src_pitch+x-1]
 328               +   p_inpix[(y+2)*i_src_pitch+x]
 329               +   p_inpix[(y+2)*i_src_pitch+x+1])<<1 )
 330               + ( p_inpix[(y+2)*i_src_pitch+x+2] )
 331               ) >> 6 /* 115 */;
 332         }
 333     }
 334 }
 335
 336 /*****************************************************************************
 337  * FilterGradient: Sobel
 338  *****************************************************************************/
 339 static void FilterGradient( filter_t *p_filter, picture_t *p_inpic,
 340                                                 picture_t *p_outpic )
 341 {
 342     int x, y;
 343     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
 344     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
 345     const int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
 346     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
 347
 348     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
 349     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
 350
 351     uint32_t *p_smooth;
 352     if( !p_filter->p_sys->p_buf32 )
 353         p_filter->p_sys->p_buf32 =
 354         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
 355     p_smooth = p_filter->p_sys->p_buf32;
 356
 357     if( !p_smooth ) return;
 358
 359     if( p_filter->p_sys->b_cartoon )
 360     {
 361         vlc_memcpy( p_outpic->p[U_PLANE].p_pixels,
 362             p_inpic->p[U_PLANE].p_pixels,
 363             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
 364         vlc_memcpy( p_outpic->p[V_PLANE].p_pixels,
 365             p_inpic->p[V_PLANE].p_pixels,
 366             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
 367     }
 368     else
 369     {
 370         vlc_memset( p_outpic->p[U_PLANE].p_pixels, 0x80,
 371             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
 372         vlc_memset( p_outpic->p[V_PLANE].p_pixels, 0x80,
 373             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
 374     }
 375
 376     GaussianConvolution( p_inpic, p_smooth );
 377
 378     /* Sobel gradient
 379
 380      | -1 0 1 |     |  1  2  1 |
 381      | -2 0 2 | and |  0  0  0 |
 382      | -1 0 1 |     | -1 -2 -1 | */
 383
 384 #define FOR                                                     \
 385     for( y = 1; y < i_num_lines - 1; y++ )                      \
 386     {                                                           \
 387         for( x = 1; x < i_src_visible - 1; x++ )                \
 388         {                                                       \
 389             const uint32_t a =                                  \
 390             (                                                   \
 391               abs(                                              \
 392                  ( p_smooth[(y-1)*i_src_visible+x-1]            \
 393                    - p_smooth[(y+1)*i_src_visible+x-1] )        \
 394                + ( ( p_smooth[(y-1)*i_src_visible+x]            \
 395                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )   \
 396                + ( p_smooth[(y-1)*i_src_visible+x+1]            \
 397                    - p_smooth[(y+1)*i_src_visible+x+1] )        \
 398               )                                                 \
 399             +                                                   \
 400               abs(                                              \
 401                  ( p_smooth[(y-1)*i_src_visible+x-1]            \
 402                    - p_smooth[(y-1)*i_src_visible+x+1] )        \
 403                + ( ( p_smooth[y*i_src_visible+x-1]              \
 404                     - p_smooth[y*i_src_visible+x+1] ) <<1 )     \
 405                + ( p_smooth[(y+1)*i_src_visible+x-1]            \
 406                    - p_smooth[(y+1)*i_src_visible+x+1] )        \
 407               )                                                 \
 408             );
 409     if( p_filter->p_sys->i_gradient_type )
 410     {
 411         if( p_filter->p_sys->b_cartoon )
 412         {
 413             FOR
 414             if( a > 60 )
 415             {
 416                 p_outpix[y*i_dst_pitch+x] = 0x00;
 417             }
 418             else
 419             {
 420                 if( p_smooth[y*i_src_visible+x] > 0xa0 )
 421                     p_outpix[y*i_dst_pitch+x] =
 422                         0xff - ((0xff - p_inpix[y*i_src_pitch+x] )>>2);
 423                 else if( p_smooth[y*i_src_visible+x] > 0x70 )
 424                     p_outpix[y*i_dst_pitch+x] =
 425                         0xa0 - ((0xa0 - p_inpix[y*i_src_pitch+x] )>>2);
 426                 else if( p_smooth[y*i_src_visible+x] > 0x28 )
 427                     p_outpix[y*i_dst_pitch+x] =
 428                         0x70 - ((0x70 - p_inpix[y*i_src_pitch+x] )>>2);
 429                 else
 430                     p_outpix[y*i_dst_pitch+x] =
 431                         0x28 - ((0x28 - p_inpix[y*i_src_pitch+x] )>>2);
 432             }
 433             }}
 434         }
 435         else
 436         {
 437             FOR
 438             p_outpix[y*i_dst_pitch+x] = clip_uint8_vlc( a );
 439             }}
 440         }
 441     }
 442     else
 443     {
 444         FOR
 445         if( a>>8 )
 446             p_outpix[y*i_dst_pitch+x] = 0;
 447         else
 448             p_outpix[y*i_dst_pitch+x] = 0xff-(uint8_t)a;
 449         }}
 450     }
 451 #undef FOR
 452 }
 453
 454 /*****************************************************************************
 455  * FilterEdge: Canny edge detection algorithm
 456  *****************************************************************************
 457  * http://fourier.eng.hmc.edu/e161/lectures/canny/node1.html
 458  * (well ... my implementation isn't really the canny algorithm ... but some
 459  * ideas are the same)
 460  *****************************************************************************/
 461 /* angle : | */
 462 #define THETA_Y 0
 463 /* angle : - */
 464 #define THETA_X 1
 465 /* angle : / */
 466 #define THETA_P 2
 467 /* angle : \ */
 468 #define THETA_M 3
 469 static void FilterEdge( filter_t *p_filter, picture_t *p_inpic,
 470                                             picture_t *p_outpic )
 471 {
 472     int x, y;
 473
 474     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
 475     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
 476     const int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
 477     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
 478
 479     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
 480     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
 481
 482     uint32_t *p_smooth;
 483     uint32_t *p_grad;
 484     uint8_t *p_theta;
 485
 486     if( !p_filter->p_sys->p_buf32 )
 487         p_filter->p_sys->p_buf32 =
 488         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
 489     p_smooth = p_filter->p_sys->p_buf32;
 490
 491     if( !p_filter->p_sys->p_buf32_bis )
 492         p_filter->p_sys->p_buf32_bis =
 493         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
 494     p_grad = p_filter->p_sys->p_buf32_bis;
 495
 496     if( !p_filter->p_sys->p_buf8 )
 497         p_filter->p_sys->p_buf8 =
 498         (uint8_t *)malloc( i_num_lines * i_src_visible * sizeof(uint8_t));
 499     p_theta = p_filter->p_sys->p_buf8;
 500
 501     if( !p_smooth || !p_grad || !p_theta ) return;
 502
 503     if( p_filter->p_sys->b_cartoon )
 504     {
 505         vlc_memcpy( p_outpic->p[U_PLANE].p_pixels,
 506             p_inpic->p[U_PLANE].p_pixels,
 507             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
 508         vlc_memcpy( p_outpic->p[V_PLANE].p_pixels,
 509             p_inpic->p[V_PLANE].p_pixels,
 510             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
 511     }
 512     else
 513     {
 514         vlc_memset( p_outpic->p[Y_PLANE].p_pixels, 0xff,
 515               p_outpic->p[Y_PLANE].i_lines * p_outpic->p[Y_PLANE].i_pitch );
 516         vlc_memset( p_outpic->p[U_PLANE].p_pixels, 0x80,
 517             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
 518         vlc_memset( p_outpic->p[V_PLANE].p_pixels, 0x80,
 519             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
 520     }
 521
 522     GaussianConvolution( p_inpic, p_smooth );
 523
 524     /* Sobel gradient
 525
 526      | -1 0 1 |     |  1  2  1 |
 527      | -2 0 2 | and |  0  0  0 |
 528      | -1 0 1 |     | -1 -2 -1 | */
 529
 530     for( y = 1; y < i_num_lines - 1; y++ )
 531     {
 532         for( x = 1; x < i_src_visible - 1; x++ )
 533         {
 534
 535             const int gradx =
 536                  ( p_smooth[(y-1)*i_src_visible+x-1]
 537                    - p_smooth[(y+1)*i_src_visible+x-1] )
 538                + ( ( p_smooth[(y-1)*i_src_visible+x]
 539                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )
 540                + ( p_smooth[(y-1)*i_src_visible+x+1]
 541                    - p_smooth[(y+1)*i_src_visible+x+1] );
 542             const int grady =
 543                  ( p_smooth[(y-1)*i_src_visible+x-1]
 544                    - p_smooth[(y-1)*i_src_visible+x+1] )
 545                + ( ( p_smooth[y*i_src_visible+x-1]
 546                     - p_smooth[y*i_src_visible+x+1] ) <<1 )
 547                + ( p_smooth[(y+1)*i_src_visible+x-1]
 548                    - p_smooth[(y+1)*i_src_visible+x+1] );
 549
 550             p_grad[y*i_src_visible+x] = (uint32_t)(abs( gradx ) + abs( grady ));
 551
 552             /* tan( 22.5 ) = 0,414213562 .. * 128 = 53
 553              * tan( 26,565051177 ) = 0.5
 554              * tan( 45 + 22.5 ) = 2,414213562 .. * 128 = 309
 555              * tan( 63,434948823 ) 2 */
 556             if( (grady<<1) > gradx )
 557                 p_theta[y*i_src_visible+x] = THETA_P;
 558             else if( (grady<<1) < -gradx )
 559                 p_theta[y*i_src_visible+x] = THETA_M;
 560             else if( !gradx || abs(grady) > abs(gradx)<<1 )
 561                 p_theta[y*i_src_visible+x] = THETA_Y;
 562             else
 563                 p_theta[y*i_src_visible+x] = THETA_X;
 564         }
 565     }
 566
 567     /* edge computing */
 568     for( y = 1; y < i_num_lines - 1; y++ )
 569     {
 570         for( x = 1; x < i_src_visible - 1; x++ )
 571         {
 572             if( p_grad[y*i_src_visible+x] > 40 )
 573             {
 574                 switch( p_theta[y*i_src_visible+x] )
 575                 {
 576                     case THETA_Y:
 577                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x]
 578                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x] )
 579                         {
 580                             p_outpix[y*i_dst_pitch+x] = 0;
 581                             break;
 582                         } else goto colorize;
 583                     case THETA_P:
 584                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x-1]
 585                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x+1] )
 586                         {
 587                             p_outpix[y*i_dst_pitch+x] = 0;
 588                             break;
 589                         } else goto colorize;
 590                     case THETA_M:
 591                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x+1]
 592                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x-1] )
 593                         {
 594                             p_outpix[y*i_dst_pitch+x] = 0;
 595                             break;
 596                         } else goto colorize;
 597                     case THETA_X:
 598                         if(    p_grad[y*i_src_visible+x] > p_grad[y*i_src_visible+x-1]
 599                             && p_grad[y*i_src_visible+x] > p_grad[y*i_src_visible+x+1] )
 600                         {
 601                             p_outpix[y*i_dst_pitch+x] = 0;
 602                             break;
 603                         } else goto colorize;
 604                 }
 605             }
 606             else
 607             {
 608                 colorize:
 609                 if( p_filter->p_sys->b_cartoon )
 610                 {
 611                     if( p_smooth[y*i_src_visible+x] > 0xa0 )
 612                         p_outpix[y*i_dst_pitch+x] = (uint8_t)
 613                             0xff - ((0xff - p_inpix[y*i_src_pitch+x] )>>2);
 614                     else if( p_smooth[y*i_src_visible+x] > 0x70 )
 615                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
 616                             0xa0 - ((0xa0 - p_inpix[y*i_src_pitch+x] )>>2);
 617                     else if( p_smooth[y*i_src_visible+x] > 0x28 )
 618                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
 619                             0x70 - ((0x70 - p_inpix[y*i_src_pitch+x] )>>2);
 620                     else
 621                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
 622                             0x28 - ((0x28 - p_inpix[y*i_src_pitch+x] )>>2);
 623                 }
 624             }
 625         }
 626     }
 627 }
 628
 629 /*****************************************************************************
 630  * FilterHough
 631  *****************************************************************************/
 632 #define p_pre_hough p_filter->p_sys->p_pre_hough
 633 static void FilterHough( filter_t *p_filter, picture_t *p_inpic,
 634                                              picture_t *p_outpic )
 635 {
 636     int x, y, i;
 637     int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
 638     int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
 639     int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
 640
 641     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
 642
 643     int i_diag = sqrt( i_num_lines * i_num_lines +
 644                         i_src_visible * i_src_visible);
 645     int i_max, i_phi_max, i_rho, i_rho_max;
 646     int i_nb_steps = 90;
 647     double d_step = M_PI / i_nb_steps;
 648     double d_sin;
 649     double d_cos;
 650     uint32_t *p_smooth;
 651
 652     int *p_hough = malloc( i_diag * i_nb_steps * sizeof(int) );
 653     if( ! p_hough ) return;
 654
 655     p_smooth = (uint32_t *)malloc( i_num_lines*i_src_visible*sizeof(uint32_t));
 656     if( !p_smooth )
 657     {
 658         free( p_hough );
 659         return;
 660     }
 661
 662     if( ! p_pre_hough )
 663     {
 664         msg_Dbg(p_filter, "Starting precalculation");
 665         p_pre_hough = malloc( i_num_lines*i_src_visible*i_nb_steps*sizeof(int));
 666         if( ! p_pre_hough )
 667         {
 668             free( p_smooth );
 669             free( p_hough );
 670             return;
 671         }
 672         for( i = 0 ; i < i_nb_steps ; i++)
 673         {
 674             d_sin = sin(d_step * i);
 675             d_cos = cos(d_step * i);
 676             for( y = 0 ; y < i_num_lines ; y++ )
 677                 for( x = 0 ; x < i_src_visible ; x++ )
 678                 {
 679                     p_pre_hough[(i*i_num_lines+y)*i_src_visible + x] =
 680                         ceil(x*d_sin + y*d_cos);
 681                 }
 682         }
 683         msg_Dbg(p_filter, "Precalculation done");
 684     }
 685
 686     vlc_memset( p_hough, 0, i_diag * i_nb_steps * sizeof(int) );
 687
 688     vlc_memcpy(
 689         p_outpic->p[Y_PLANE].p_pixels, p_inpic->p[Y_PLANE].p_pixels,
 690         p_outpic->p[Y_PLANE].i_lines * p_outpic->p[Y_PLANE].i_pitch );
 691     vlc_memcpy(
 692         p_outpic->p[U_PLANE].p_pixels, p_inpic->p[U_PLANE].p_pixels,
 693         p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
 694     vlc_memcpy(
 695         p_outpic->p[V_PLANE].p_pixels, p_inpic->p[V_PLANE].p_pixels,
 696         p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
 697
 698     GaussianConvolution( p_inpic, p_smooth );
 699
 700     /* Sobel gradient
 701
 702      | -1 0 1 |     |  1  2  1 |
 703      | -2 0 2 | and |  0  0  0 |
 704      | -1 0 1 |     | -1 -2 -1 | */
 705
 706     i_max = 0;
 707     i_rho_max = 0;
 708     i_phi_max = 0;
 709     for( y = 4; y < i_num_lines - 4; y++ )
 710     {
 711         for( x = 4; x < i_src_visible - 4; x++ )
 712         {
 713             uint32_t a =
 714             (
 715               abs(
 716                 ( ( p_smooth[(y-1)*i_src_visible+x]
 717                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )
 718                + ( p_smooth[(y-1)*i_src_visible+x-1]
 719                    - p_smooth[(y+1)*i_src_visible+x-1] )
 720                + ( p_smooth[(y-1)*i_src_visible+x+1]
 721                    - p_smooth[(y+1)*i_src_visible+x+1] )
 722               )
 723             +
 724               abs(
 725                 ( ( p_smooth[y*i_src_visible+x-1]
 726                     - p_smooth[y*i_src_visible+x+1] ) <<1 )
 727                + ( p_smooth[(y-1)*i_src_visible+x-1]
 728                    - p_smooth[(y-1)*i_src_visible+x+1] )
 729                + ( p_smooth[(y+1)*i_src_visible+x-1]
 730                    - p_smooth[(y+1)*i_src_visible+x+1] )
 731               )
 732             );
 733             if( a>>8 )
 734             {
 735                 for( i = 0 ; i < i_nb_steps ; i ++ )
 736                 {
 737                     i_rho = p_pre_hough[(i*i_num_lines+y)*i_src_visible + x];
 738                     if( p_hough[i_rho + i_diag/2 + i * i_diag]++ > i_max )
 739                     {
 740                         i_max = p_hough[i_rho + i_diag/2 + i * i_diag];
 741                         i_rho_max = i_rho;
 742                         i_phi_max = i;
 743                     }
 744                 }
 745             }
 746         }
 747     }
 748
 749     d_sin = sin(i_phi_max*d_step);
 750     d_cos = cos(i_phi_max*d_step);
 751     if( d_cos != 0 )
 752     {
 753         for( x = 0 ; x < i_src_visible ; x++ )
 754         {
 755             y = (i_rho_max - x * d_sin) / d_cos;
 756             if( y >= 0 && y < i_num_lines )
 757                 p_outpix[y*i_dst_pitch+x] = 255;
 758         }
 759     }
 760
 761     free( p_hough );
 762     free( p_smooth );
 763 }
 764 #undef p_pre_hough
 765
 766
 767 static int GradientCallback( vlc_object_t *p_this, char const *psz_var,
 768                              vlc_value_t oldval, vlc_value_t newval,
 769                              void *p_data )
 770 {
 771     VLC_UNUSED(oldval);
 772     filter_sys_t *p_sys = (filter_sys_t *)p_data;
 773
 774     vlc_mutex_lock( &p_sys->lock );
 775     if( !strcmp( psz_var, FILTER_PREFIX "mode" ) )
 776     {
 777         if( !strcmp( newval.psz_string, "gradient" ) )
 778         {
 779             p_sys->i_mode = GRADIENT;
 780         }
 781         else if( !strcmp( newval.psz_string, "edge" ) )
 782         {
 783             p_sys->i_mode = EDGE;
 784         }
 785         else if( !strcmp( newval.psz_string, "hough" ) )
 786         {
 787             p_sys->i_mode = HOUGH;
 788         }
 789         else
 790         {
 791             msg_Err( p_this, "no valid gradient mode provided (%s)", newval.psz_string );
 792             p_sys->i_mode = GRADIENT;
 793         }
 794     }
 795     else if( !strcmp( psz_var, FILTER_PREFIX "type" ) )
 796     {
 797         p_sys->i_gradient_type = newval.i_int;
 798     }
 799     else if( !strcmp( psz_var, FILTER_PREFIX "cartoon" ) )
 800     {
 801         p_sys->b_cartoon = newval.b_bool;
 802     }
 803     vlc_mutex_unlock( &p_sys->lock );
 804
 805     return VLC_SUCCESS;
 806 }