git.sesse.net Git - vlc/blob - modules/video_filter/gradient.c

   1 /*****************************************************************************
   2  * gradient.c : Gradient and edge detection video effects plugin for vlc
   3  *****************************************************************************
   4  * Copyright (C) 2000-2008 the VideoLAN team
   5  * $Id$
   6  *
   7  * Authors: Samuel Hocevar <sam@zoy.org>
   8  *          Antoine Cellerier <dionoea -at- videolan -dot- org>
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License as published by
  12  * the Free Software Foundation; either version 2 of the License, or
  13  * (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU General Public License
  21  * along with this program; if not, write to the Free Software
  22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  23  *****************************************************************************/
  24
  25 /*****************************************************************************
  26  * Preamble
  27  *****************************************************************************/
  28
  29 #ifdef HAVE_CONFIG_H
  30 # include "config.h"
  31 #endif
  32
  33 #include <math.h>                                            /* sin(), cos() */
  34
  35 #include <vlc_common.h>
  36 #include <vlc_plugin.h>
  37 #include <vlc_sout.h>
  38
  39 #include "vlc_filter.h"
  40 #include "filter_picture.h"
  41
  42 enum { GRADIENT, EDGE, HOUGH };
  43
  44 /*****************************************************************************
  45  * Local prototypes
  46  *****************************************************************************/
  47 static int  Create    ( vlc_object_t * );
  48 static void Destroy   ( vlc_object_t * );
  49
  50 static picture_t *Filter( filter_t *, picture_t * );
  51 static int GradientCallback( vlc_object_t *, char const *,
  52                              vlc_value_t, vlc_value_t,
  53                              void * );
  54
  55 static void FilterGradient( filter_t *, picture_t *, picture_t * );
  56 static void FilterEdge    ( filter_t *, picture_t *, picture_t * );
  57 static void FilterHough   ( filter_t *, picture_t *, picture_t * );
  58
  59 /*****************************************************************************
  60  * Module descriptor
  61  *****************************************************************************/
  62 #define MODE_TEXT N_("Distort mode")
  63 #define MODE_LONGTEXT N_("Distort mode, one of \"gradient\", \"edge\" and \"hough\".")
  64
  65 #define GRADIENT_TEXT N_("Gradient image type")
  66 #define GRADIENT_LONGTEXT N_("Gradient image type (0 or 1). 0 will " \
  67         "turn the image to white while 1 will keep colors." )
  68
  69 #define CARTOON_TEXT N_("Apply cartoon effect")
  70 #define CARTOON_LONGTEXT N_("Apply cartoon effect. It is only used by " \
  71     "\"gradient\" and \"edge\".")
  72
  73 static const char *const mode_list[] = { "gradient", "edge", "hough" };
  74 static const char *const mode_list_text[] = { N_("Gradient"), N_("Edge"), N_("Hough") };
  75
  76 #define FILTER_PREFIX "gradient-"
  77
  78 vlc_module_begin ()
  79     set_description( N_("Gradient video filter") )
  80     set_shortname( N_( "Gradient" ))
  81     set_capability( "video filter2", 0 )
  82     set_category( CAT_VIDEO )
  83     set_subcategory( SUBCAT_VIDEO_VFILTER )
  84
  85     add_string( FILTER_PREFIX "mode", "gradient", NULL,
  86                 MODE_TEXT, MODE_LONGTEXT, false )
  87         change_string_list( mode_list, mode_list_text, 0 )
  88
  89     add_integer_with_range( FILTER_PREFIX "type", 0, 0, 1, NULL,
  90                 GRADIENT_TEXT, GRADIENT_LONGTEXT, false )
  91     add_bool( FILTER_PREFIX "cartoon", 1, NULL,
  92                 CARTOON_TEXT, CARTOON_LONGTEXT, false )
  93
  94     add_shortcut( "gradient" )
  95     set_callbacks( Create, Destroy )
  96 vlc_module_end ()
  97
  98 static const char *const ppsz_filter_options[] = {
  99     "mode", "type", "cartoon", NULL
 100 };
 101
 102 /*****************************************************************************
 103  * filter_sys_t: Distort video output method descriptor
 104  *****************************************************************************
 105  * This structure is part of the video output thread descriptor.
 106  * It describes the Distort specific properties of an output thread.
 107  *****************************************************************************/
 108 struct filter_sys_t
 109 {
 110     vlc_mutex_t lock;
 111     int i_mode;
 112
 113     /* For the gradient mode */
 114     int i_gradient_type;
 115     bool b_cartoon;
 116
 117     uint32_t *p_buf32;
 118     uint32_t *p_buf32_bis;
 119     uint8_t *p_buf8;
 120
 121     /* For hough mode */
 122     int *p_pre_hough;
 123 };
 124
 125 /*****************************************************************************
 126  * Create: allocates Distort video thread output method
 127  *****************************************************************************
 128  * This function allocates and initializes a Distort vout method.
 129  *****************************************************************************/
 130 static int Create( vlc_object_t *p_this )
 131 {
 132     filter_t *p_filter = (filter_t *)p_this;
 133     char *psz_method;
 134
 135     switch( p_filter->fmt_in.video.i_chroma )
 136     {
 137         CASE_PLANAR_YUV
 138             break;
 139
 140         default:
 141              msg_Err( p_filter, "Unsupported input chroma (%4s)",
 142                       (char*)&(p_filter->fmt_in.video.i_chroma) );
 143             return VLC_EGENERIC;
 144     }
 145
 146     /* Allocate structure */
 147     p_filter->p_sys = malloc( sizeof( filter_sys_t ) );
 148     if( p_filter->p_sys == NULL )
 149         return VLC_ENOMEM;
 150
 151     p_filter->pf_video_filter = Filter;
 152
 153     p_filter->p_sys->p_pre_hough = NULL;
 154
 155     config_ChainParse( p_filter, FILTER_PREFIX, ppsz_filter_options,
 156                    p_filter->p_cfg );
 157
 158     if( !(psz_method =
 159         var_CreateGetNonEmptyStringCommand( p_filter, FILTER_PREFIX "mode" )) )
 160     {
 161         msg_Err( p_filter, "configuration variable "
 162                  FILTER_PREFIX "mode empty" );
 163         p_filter->p_sys->i_mode = GRADIENT;
 164     }
 165     else
 166     {
 167         if( !strcmp( psz_method, "gradient" ) )
 168         {
 169             p_filter->p_sys->i_mode = GRADIENT;
 170         }
 171         else if( !strcmp( psz_method, "edge" ) )
 172         {
 173             p_filter->p_sys->i_mode = EDGE;
 174         }
 175         else if( !strcmp( psz_method, "hough" ) )
 176         {
 177             p_filter->p_sys->i_mode = HOUGH;
 178         }
 179         else
 180         {
 181             msg_Err( p_filter, "no valid gradient mode provided (%s)", psz_method );
 182             p_filter->p_sys->i_mode = GRADIENT;
 183         }
 184     }
 185     free( psz_method );
 186
 187     p_filter->p_sys->i_gradient_type =
 188         var_CreateGetIntegerCommand( p_filter, FILTER_PREFIX "type" );
 189     p_filter->p_sys->b_cartoon =
 190         var_CreateGetBoolCommand( p_filter, FILTER_PREFIX "cartoon" );
 191
 192     vlc_mutex_init( &p_filter->p_sys->lock );
 193     var_AddCallback( p_filter, FILTER_PREFIX "mode",
 194                      GradientCallback, p_filter->p_sys );
 195     var_AddCallback( p_filter, FILTER_PREFIX "type",
 196                      GradientCallback, p_filter->p_sys );
 197     var_AddCallback( p_filter, FILTER_PREFIX "cartoon",
 198                      GradientCallback, p_filter->p_sys );
 199
 200     p_filter->p_sys->p_buf32 = NULL;
 201     p_filter->p_sys->p_buf32_bis = NULL;
 202     p_filter->p_sys->p_buf8 = NULL;
 203
 204     return VLC_SUCCESS;
 205 }
 206
 207 /*****************************************************************************
 208  * Destroy: destroy Distort video thread output method
 209  *****************************************************************************
 210  * Terminate an output method created by DistortCreateOutputMethod
 211  *****************************************************************************/
 212 static void Destroy( vlc_object_t *p_this )
 213 {
 214     filter_t *p_filter = (filter_t *)p_this;
 215     filter_sys_t *p_sys = p_filter->p_sys;
 216
 217     var_DelCallback( p_filter, FILTER_PREFIX "mode",
 218                      GradientCallback, p_sys );
 219     var_DelCallback( p_filter, FILTER_PREFIX "type",
 220                      GradientCallback, p_sys );
 221     var_DelCallback( p_filter, FILTER_PREFIX "cartoon",
 222                      GradientCallback, p_sys );
 223     vlc_mutex_destroy( &p_sys->lock );
 224
 225     free( p_sys->p_buf32 );
 226     free( p_sys->p_buf32_bis );
 227     free( p_sys->p_buf8 );
 228     free( p_sys->p_pre_hough );
 229
 230     free( p_sys );
 231 }
 232
 233 /*****************************************************************************
 234  * Render: displays previously rendered output
 235  *****************************************************************************
 236  * This function send the currently rendered image to Distort image, waits
 237  * until it is displayed and switch the two rendering buffers, preparing next
 238  * frame.
 239  *****************************************************************************/
 240 static picture_t *Filter( filter_t *p_filter, picture_t *p_pic )
 241 {
 242     picture_t *p_outpic;
 243
 244     if( !p_pic ) return NULL;
 245
 246     p_outpic = filter_NewPicture( p_filter );
 247     if( !p_outpic )
 248     {
 249         picture_Release( p_pic );
 250         return NULL;
 251     }
 252
 253     vlc_mutex_lock( &p_filter->p_sys->lock );
 254     switch( p_filter->p_sys->i_mode )
 255     {
 256         case EDGE:
 257             FilterEdge( p_filter, p_pic, p_outpic );
 258             break;
 259
 260         case GRADIENT:
 261             FilterGradient( p_filter, p_pic, p_outpic );
 262             break;
 263
 264         case HOUGH:
 265             FilterHough( p_filter, p_pic, p_outpic );
 266             break;
 267
 268         default:
 269             break;
 270     }
 271     vlc_mutex_unlock( &p_filter->p_sys->lock );
 272
 273     return CopyInfoAndRelease( p_outpic, p_pic );
 274 }
 275
 276 /*****************************************************************************
 277  * Gaussian Convolution
 278  *****************************************************************************
 279  *    Gaussian convolution ( sigma == 1.4 )
 280  *
 281  *    |  2  4  5  4  2  |   |  2  4  4  4  2 |
 282  *    |  4  9 12  9  4  |   |  4  8 12  8  4 |
 283  *    |  5 12 15 12  5  | ~ |  4 12 16 12  4 |
 284  *    |  4  9 12  9  4  |   |  4  8 12  8  4 |
 285  *    |  2  4  5  4  2  |   |  2  4  4  4  2 |
 286  *****************************************************************************/
 287 static void GaussianConvolution( picture_t *p_inpic, uint32_t *p_smooth )
 288 {
 289     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
 290     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
 291     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
 292     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
 293
 294     int x,y;
 295     for( y = 2; y < i_num_lines - 2; y++ )
 296     {
 297         for( x = 2; x < i_src_visible - 2; x++ )
 298         {
 299             p_smooth[y*i_src_visible+x] = (uint32_t)(
 300               /* 2 rows up */
 301                 ( p_inpix[(y-2)*i_src_pitch+x-2] )
 302               + ((p_inpix[(y-2)*i_src_pitch+x-1]
 303               +   p_inpix[(y-2)*i_src_pitch+x]
 304               +   p_inpix[(y-2)*i_src_pitch+x+1])<<1 )
 305               + ( p_inpix[(y-2)*i_src_pitch+x+2] )
 306               /* 1 row up */
 307               + ((p_inpix[(y-1)*i_src_pitch+x-2]
 308               + ( p_inpix[(y-1)*i_src_pitch+x-1]<<1 )
 309               + ( p_inpix[(y-1)*i_src_pitch+x]*3 )
 310               + ( p_inpix[(y-1)*i_src_pitch+x+1]<<1 )
 311               +   p_inpix[(y-1)*i_src_pitch+x+2]
 312               /* */
 313               +   p_inpix[y*i_src_pitch+x-2]
 314               + ( p_inpix[y*i_src_pitch+x-1]*3 )
 315               + ( p_inpix[y*i_src_pitch+x]<<2 )
 316               + ( p_inpix[y*i_src_pitch+x+1]*3 )
 317               +   p_inpix[y*i_src_pitch+x+2]
 318               /* 1 row down */
 319               +   p_inpix[(y+1)*i_src_pitch+x-2]
 320               + ( p_inpix[(y+1)*i_src_pitch+x-1]<<1 )
 321               + ( p_inpix[(y+1)*i_src_pitch+x]*3 )
 322               + ( p_inpix[(y+1)*i_src_pitch+x+1]<<1 )
 323               +   p_inpix[(y+1)*i_src_pitch+x+2] )<<1 )
 324               /* 2 rows down */
 325               + ( p_inpix[(y+2)*i_src_pitch+x-2] )
 326               + ((p_inpix[(y+2)*i_src_pitch+x-1]
 327               +   p_inpix[(y+2)*i_src_pitch+x]
 328               +   p_inpix[(y+2)*i_src_pitch+x+1])<<1 )
 329               + ( p_inpix[(y+2)*i_src_pitch+x+2] )
 330               ) >> 6 /* 115 */;
 331         }
 332     }
 333 }
 334
 335 /*****************************************************************************
 336  * FilterGradient: Sobel
 337  *****************************************************************************/
 338 static void FilterGradient( filter_t *p_filter, picture_t *p_inpic,
 339                                                 picture_t *p_outpic )
 340 {
 341     int x, y;
 342     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
 343     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
 344     const int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
 345     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
 346
 347     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
 348     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
 349
 350     uint32_t *p_smooth;
 351     if( !p_filter->p_sys->p_buf32 )
 352         p_filter->p_sys->p_buf32 =
 353         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
 354     p_smooth = p_filter->p_sys->p_buf32;
 355
 356     if( !p_smooth ) return;
 357
 358     if( p_filter->p_sys->b_cartoon )
 359     {
 360         vlc_memcpy( p_outpic->p[U_PLANE].p_pixels,
 361             p_inpic->p[U_PLANE].p_pixels,
 362             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
 363         vlc_memcpy( p_outpic->p[V_PLANE].p_pixels,
 364             p_inpic->p[V_PLANE].p_pixels,
 365             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
 366     }
 367     else
 368     {
 369         vlc_memset( p_outpic->p[U_PLANE].p_pixels, 0x80,
 370             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
 371         vlc_memset( p_outpic->p[V_PLANE].p_pixels, 0x80,
 372             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
 373     }
 374
 375     GaussianConvolution( p_inpic, p_smooth );
 376
 377     /* Sobel gradient
 378
 379      | -1 0 1 |     |  1  2  1 |
 380      | -2 0 2 | and |  0  0  0 |
 381      | -1 0 1 |     | -1 -2 -1 | */
 382
 383 #define FOR                                                     \
 384     for( y = 1; y < i_num_lines - 1; y++ )                      \
 385     {                                                           \
 386         for( x = 1; x < i_src_visible - 1; x++ )                \
 387         {                                                       \
 388             const uint32_t a =                                  \
 389             (                                                   \
 390               abs(                                              \
 391                  ( p_smooth[(y-1)*i_src_visible+x-1]            \
 392                    - p_smooth[(y+1)*i_src_visible+x-1] )        \
 393                + ( ( p_smooth[(y-1)*i_src_visible+x]            \
 394                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )   \
 395                + ( p_smooth[(y-1)*i_src_visible+x+1]            \
 396                    - p_smooth[(y+1)*i_src_visible+x+1] )        \
 397               )                                                 \
 398             +                                                   \
 399               abs(                                              \
 400                  ( p_smooth[(y-1)*i_src_visible+x-1]            \
 401                    - p_smooth[(y-1)*i_src_visible+x+1] )        \
 402                + ( ( p_smooth[y*i_src_visible+x-1]              \
 403                     - p_smooth[y*i_src_visible+x+1] ) <<1 )     \
 404                + ( p_smooth[(y+1)*i_src_visible+x-1]            \
 405                    - p_smooth[(y+1)*i_src_visible+x+1] )        \
 406               )                                                 \
 407             );
 408     if( p_filter->p_sys->i_gradient_type )
 409     {
 410         if( p_filter->p_sys->b_cartoon )
 411         {
 412             FOR
 413             if( a > 60 )
 414             {
 415                 p_outpix[y*i_dst_pitch+x] = 0x00;
 416             }
 417             else
 418             {
 419                 if( p_smooth[y*i_src_visible+x] > 0xa0 )
 420                     p_outpix[y*i_dst_pitch+x] =
 421                         0xff - ((0xff - p_inpix[y*i_src_pitch+x] )>>2);
 422                 else if( p_smooth[y*i_src_visible+x] > 0x70 )
 423                     p_outpix[y*i_dst_pitch+x] =
 424                         0xa0 - ((0xa0 - p_inpix[y*i_src_pitch+x] )>>2);
 425                 else if( p_smooth[y*i_src_visible+x] > 0x28 )
 426                     p_outpix[y*i_dst_pitch+x] =
 427                         0x70 - ((0x70 - p_inpix[y*i_src_pitch+x] )>>2);
 428                 else
 429                     p_outpix[y*i_dst_pitch+x] =
 430                         0x28 - ((0x28 - p_inpix[y*i_src_pitch+x] )>>2);
 431             }
 432             }}
 433         }
 434         else
 435         {
 436             FOR
 437             p_outpix[y*i_dst_pitch+x] = clip_uint8_vlc( a );
 438             }}
 439         }
 440     }
 441     else
 442     {
 443         FOR
 444         if( a>>8 )
 445             p_outpix[y*i_dst_pitch+x] = 0;
 446         else
 447             p_outpix[y*i_dst_pitch+x] = 0xff-(uint8_t)a;
 448         }}
 449     }
 450 #undef FOR
 451 }
 452
 453 /*****************************************************************************
 454  * FilterEdge: Canny edge detection algorithm
 455  *****************************************************************************
 456  * http://fourier.eng.hmc.edu/e161/lectures/canny/node1.html
 457  * (well ... my implementation isn't really the canny algorithm ... but some
 458  * ideas are the same)
 459  *****************************************************************************/
 460 /* angle : | */
 461 #define THETA_Y 0
 462 /* angle : - */
 463 #define THETA_X 1
 464 /* angle : / */
 465 #define THETA_P 2
 466 /* angle : \ */
 467 #define THETA_M 3
 468 static void FilterEdge( filter_t *p_filter, picture_t *p_inpic,
 469                                             picture_t *p_outpic )
 470 {
 471     int x, y;
 472
 473     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
 474     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
 475     const int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
 476     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
 477
 478     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
 479     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
 480
 481     uint32_t *p_smooth;
 482     uint32_t *p_grad;
 483     uint8_t *p_theta;
 484
 485     if( !p_filter->p_sys->p_buf32 )
 486         p_filter->p_sys->p_buf32 =
 487         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
 488     p_smooth = p_filter->p_sys->p_buf32;
 489
 490     if( !p_filter->p_sys->p_buf32_bis )
 491         p_filter->p_sys->p_buf32_bis =
 492         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
 493     p_grad = p_filter->p_sys->p_buf32_bis;
 494
 495     if( !p_filter->p_sys->p_buf8 )
 496         p_filter->p_sys->p_buf8 =
 497         (uint8_t *)malloc( i_num_lines * i_src_visible * sizeof(uint8_t));
 498     p_theta = p_filter->p_sys->p_buf8;
 499
 500     if( !p_smooth || !p_grad || !p_theta ) return;
 501
 502     if( p_filter->p_sys->b_cartoon )
 503     {
 504         vlc_memcpy( p_outpic->p[U_PLANE].p_pixels,
 505             p_inpic->p[U_PLANE].p_pixels,
 506             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
 507         vlc_memcpy( p_outpic->p[V_PLANE].p_pixels,
 508             p_inpic->p[V_PLANE].p_pixels,
 509             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
 510     }
 511     else
 512     {
 513         vlc_memset( p_outpic->p[Y_PLANE].p_pixels, 0xff,
 514               p_outpic->p[Y_PLANE].i_lines * p_outpic->p[Y_PLANE].i_pitch );
 515         vlc_memset( p_outpic->p[U_PLANE].p_pixels, 0x80,
 516             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
 517         vlc_memset( p_outpic->p[V_PLANE].p_pixels, 0x80,
 518             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
 519     }
 520
 521     GaussianConvolution( p_inpic, p_smooth );
 522
 523     /* Sobel gradient
 524
 525      | -1 0 1 |     |  1  2  1 |
 526      | -2 0 2 | and |  0  0  0 |
 527      | -1 0 1 |     | -1 -2 -1 | */
 528
 529     for( y = 1; y < i_num_lines - 1; y++ )
 530     {
 531         for( x = 1; x < i_src_visible - 1; x++ )
 532         {
 533
 534             const int gradx =
 535                  ( p_smooth[(y-1)*i_src_visible+x-1]
 536                    - p_smooth[(y+1)*i_src_visible+x-1] )
 537                + ( ( p_smooth[(y-1)*i_src_visible+x]
 538                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )
 539                + ( p_smooth[(y-1)*i_src_visible+x+1]
 540                    - p_smooth[(y+1)*i_src_visible+x+1] );
 541             const int grady =
 542                  ( p_smooth[(y-1)*i_src_visible+x-1]
 543                    - p_smooth[(y-1)*i_src_visible+x+1] )
 544                + ( ( p_smooth[y*i_src_visible+x-1]
 545                     - p_smooth[y*i_src_visible+x+1] ) <<1 )
 546                + ( p_smooth[(y+1)*i_src_visible+x-1]
 547                    - p_smooth[(y+1)*i_src_visible+x+1] );
 548
 549             p_grad[y*i_src_visible+x] = (uint32_t)(abs( gradx ) + abs( grady ));
 550
 551             /* tan( 22.5 ) = 0,414213562 .. * 128 = 53
 552              * tan( 26,565051177 ) = 0.5
 553              * tan( 45 + 22.5 ) = 2,414213562 .. * 128 = 309
 554              * tan( 63,434948823 ) 2 */
 555             if( (grady<<1) > gradx )
 556                 p_theta[y*i_src_visible+x] = THETA_P;
 557             else if( (grady<<1) < -gradx )
 558                 p_theta[y*i_src_visible+x] = THETA_M;
 559             else if( !gradx || abs(grady) > abs(gradx)<<1 )
 560                 p_theta[y*i_src_visible+x] = THETA_Y;
 561             else
 562                 p_theta[y*i_src_visible+x] = THETA_X;
 563         }
 564     }
 565
 566     /* edge computing */
 567     for( y = 1; y < i_num_lines - 1; y++ )
 568     {
 569         for( x = 1; x < i_src_visible - 1; x++ )
 570         {
 571             if( p_grad[y*i_src_visible+x] > 40 )
 572             {
 573                 switch( p_theta[y*i_src_visible+x] )
 574                 {
 575                     case THETA_Y:
 576                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x]
 577                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x] )
 578                         {
 579                             p_outpix[y*i_dst_pitch+x] = 0;
 580                             break;
 581                         } else goto colorize;
 582                     case THETA_P:
 583                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x-1]
 584                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x+1] )
 585                         {
 586                             p_outpix[y*i_dst_pitch+x] = 0;
 587                             break;
 588                         } else goto colorize;
 589                     case THETA_M:
 590                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x+1]
 591                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x-1] )
 592                         {
 593                             p_outpix[y*i_dst_pitch+x] = 0;
 594                             break;
 595                         } else goto colorize;
 596                     case THETA_X:
 597                         if(    p_grad[y*i_src_visible+x] > p_grad[y*i_src_visible+x-1]
 598                             && p_grad[y*i_src_visible+x] > p_grad[y*i_src_visible+x+1] )
 599                         {
 600                             p_outpix[y*i_dst_pitch+x] = 0;
 601                             break;
 602                         } else goto colorize;
 603                 }
 604             }
 605             else
 606             {
 607                 colorize:
 608                 if( p_filter->p_sys->b_cartoon )
 609                 {
 610                     if( p_smooth[y*i_src_visible+x] > 0xa0 )
 611                         p_outpix[y*i_dst_pitch+x] = (uint8_t)
 612                             0xff - ((0xff - p_inpix[y*i_src_pitch+x] )>>2);
 613                     else if( p_smooth[y*i_src_visible+x] > 0x70 )
 614                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
 615                             0xa0 - ((0xa0 - p_inpix[y*i_src_pitch+x] )>>2);
 616                     else if( p_smooth[y*i_src_visible+x] > 0x28 )
 617                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
 618                             0x70 - ((0x70 - p_inpix[y*i_src_pitch+x] )>>2);
 619                     else
 620                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
 621                             0x28 - ((0x28 - p_inpix[y*i_src_pitch+x] )>>2);
 622                 }
 623             }
 624         }
 625     }
 626 }
 627
 628 /*****************************************************************************
 629  * FilterHough
 630  *****************************************************************************/
 631 #define p_pre_hough p_filter->p_sys->p_pre_hough
 632 static void FilterHough( filter_t *p_filter, picture_t *p_inpic,
 633                                              picture_t *p_outpic )
 634 {
 635     int x, y, i;
 636     int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
 637     int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
 638     int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
 639
 640     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
 641
 642     int i_diag = sqrt( i_num_lines * i_num_lines +
 643                         i_src_visible * i_src_visible);
 644     int i_max, i_phi_max, i_rho, i_rho_max;
 645     int i_nb_steps = 90;
 646     double d_step = M_PI / i_nb_steps;
 647     double d_sin;
 648     double d_cos;
 649     uint32_t *p_smooth;
 650
 651     int *p_hough = malloc( i_diag * i_nb_steps * sizeof(int) );
 652     if( ! p_hough ) return;
 653
 654     p_smooth = (uint32_t *)malloc( i_num_lines*i_src_visible*sizeof(uint32_t));
 655     if( !p_smooth )
 656     {
 657         free( p_hough );
 658         return;
 659     }
 660
 661     if( ! p_pre_hough )
 662     {
 663         msg_Dbg(p_filter, "Starting precalculation");
 664         p_pre_hough = malloc( i_num_lines*i_src_visible*i_nb_steps*sizeof(int));
 665         if( ! p_pre_hough )
 666         {
 667             free( p_smooth );
 668             free( p_hough );
 669             return;
 670         }
 671         for( i = 0 ; i < i_nb_steps ; i++)
 672         {
 673             d_sin = sin(d_step * i);
 674             d_cos = cos(d_step * i);
 675             for( y = 0 ; y < i_num_lines ; y++ )
 676                 for( x = 0 ; x < i_src_visible ; x++ )
 677                 {
 678                     p_pre_hough[(i*i_num_lines+y)*i_src_visible + x] =
 679                         ceil(x*d_sin + y*d_cos);
 680                 }
 681         }
 682         msg_Dbg(p_filter, "Precalculation done");
 683     }
 684
 685     vlc_memset( p_hough, 0, i_diag * i_nb_steps * sizeof(int) );
 686
 687     vlc_memcpy(
 688         p_outpic->p[Y_PLANE].p_pixels, p_inpic->p[Y_PLANE].p_pixels,
 689         p_outpic->p[Y_PLANE].i_lines * p_outpic->p[Y_PLANE].i_pitch );
 690     vlc_memcpy(
 691         p_outpic->p[U_PLANE].p_pixels, p_inpic->p[U_PLANE].p_pixels,
 692         p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
 693     vlc_memcpy(
 694         p_outpic->p[V_PLANE].p_pixels, p_inpic->p[V_PLANE].p_pixels,
 695         p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
 696
 697     GaussianConvolution( p_inpic, p_smooth );
 698
 699     /* Sobel gradient
 700
 701      | -1 0 1 |     |  1  2  1 |
 702      | -2 0 2 | and |  0  0  0 |
 703      | -1 0 1 |     | -1 -2 -1 | */
 704
 705     i_max = 0;
 706     i_rho_max = 0;
 707     i_phi_max = 0;
 708     for( y = 4; y < i_num_lines - 4; y++ )
 709     {
 710         for( x = 4; x < i_src_visible - 4; x++ )
 711         {
 712             uint32_t a =
 713             (
 714               abs(
 715                 ( ( p_smooth[(y-1)*i_src_visible+x]
 716                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )
 717                + ( p_smooth[(y-1)*i_src_visible+x-1]
 718                    - p_smooth[(y+1)*i_src_visible+x-1] )
 719                + ( p_smooth[(y-1)*i_src_visible+x+1]
 720                    - p_smooth[(y+1)*i_src_visible+x+1] )
 721               )
 722             +
 723               abs(
 724                 ( ( p_smooth[y*i_src_visible+x-1]
 725                     - p_smooth[y*i_src_visible+x+1] ) <<1 )
 726                + ( p_smooth[(y-1)*i_src_visible+x-1]
 727                    - p_smooth[(y-1)*i_src_visible+x+1] )
 728                + ( p_smooth[(y+1)*i_src_visible+x-1]
 729                    - p_smooth[(y+1)*i_src_visible+x+1] )
 730               )
 731             );
 732             if( a>>8 )
 733             {
 734                 for( i = 0 ; i < i_nb_steps ; i ++ )
 735                 {
 736                     i_rho = p_pre_hough[(i*i_num_lines+y)*i_src_visible + x];
 737                     if( p_hough[i_rho + i_diag/2 + i * i_diag]++ > i_max )
 738                     {
 739                         i_max = p_hough[i_rho + i_diag/2 + i * i_diag];
 740                         i_rho_max = i_rho;
 741                         i_phi_max = i;
 742                     }
 743                 }
 744             }
 745         }
 746     }
 747
 748     d_sin = sin(i_phi_max*d_step);
 749     d_cos = cos(i_phi_max*d_step);
 750     if( d_cos != 0 )
 751     {
 752         for( x = 0 ; x < i_src_visible ; x++ )
 753         {
 754             y = (i_rho_max - x * d_sin) / d_cos;
 755             if( y >= 0 && y < i_num_lines )
 756                 p_outpix[y*i_dst_pitch+x] = 255;
 757         }
 758     }
 759
 760     free( p_hough );
 761     free( p_smooth );
 762 }
 763 #undef p_pre_hough
 764
 765
 766 static int GradientCallback( vlc_object_t *p_this, char const *psz_var,
 767                              vlc_value_t oldval, vlc_value_t newval,
 768                              void *p_data )
 769 {
 770     VLC_UNUSED(oldval);
 771     filter_sys_t *p_sys = (filter_sys_t *)p_data;
 772
 773     vlc_mutex_lock( &p_sys->lock );
 774     if( !strcmp( psz_var, FILTER_PREFIX "mode" ) )
 775     {
 776         if( !strcmp( newval.psz_string, "gradient" ) )
 777         {
 778             p_sys->i_mode = GRADIENT;
 779         }
 780         else if( !strcmp( newval.psz_string, "edge" ) )
 781         {
 782             p_sys->i_mode = EDGE;
 783         }
 784         else if( !strcmp( newval.psz_string, "hough" ) )
 785         {
 786             p_sys->i_mode = HOUGH;
 787         }
 788         else
 789         {
 790             msg_Err( p_this, "no valid gradient mode provided (%s)", newval.psz_string );
 791             p_sys->i_mode = GRADIENT;
 792         }
 793     }
 794     else if( !strcmp( psz_var, FILTER_PREFIX "type" ) )
 795     {
 796         p_sys->i_gradient_type = newval.i_int;
 797     }
 798     else if( !strcmp( psz_var, FILTER_PREFIX "cartoon" ) )
 799     {
 800         p_sys->b_cartoon = newval.b_bool;
 801     }
 802     vlc_mutex_unlock( &p_sys->lock );
 803
 804     return VLC_SUCCESS;
 805 }