git.sesse.net Git - vlc/blob - modules/video_filter/gradient.c

   1 /*****************************************************************************
   2  * gradient.c : Gradient and edge detection video effects plugin for vlc
   3  *****************************************************************************
   4  * Copyright (C) 2000-2006 the VideoLAN team
   5  * $Id$
   6  *
   7  * Authors: Samuel Hocevar <sam@zoy.org>
   8  *          Antoine Cellerier <dionoea -at- videolan -dot- org>
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License as published by
  12  * the Free Software Foundation; either version 2 of the License, or
  13  * (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU General Public License
  21  * along with this program; if not, write to the Free Software
  22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  23  *****************************************************************************/
  24
  25 /*****************************************************************************
  26  * Preamble
  27  *****************************************************************************/
  28 #include <stdlib.h>                                      /* malloc(), free() */
  29 #include <string.h>
  30
  31 #include <math.h>                                            /* sin(), cos() */
  32
  33 #include <vlc/vlc.h>
  34 #include <vlc_sout.h>
  35 #include <vlc_vout.h>
  36
  37 #include "vlc_filter.h"
  38
  39 enum { GRADIENT, EDGE, HOUGH };
  40
  41 /*****************************************************************************
  42  * Local prototypes
  43  *****************************************************************************/
  44 static int  Create    ( vlc_object_t * );
  45 static void Destroy   ( vlc_object_t * );
  46
  47 static picture_t *Filter( filter_t *, picture_t * );
  48
  49 static void FilterGradient( filter_t *, picture_t *, picture_t * );
  50 static void FilterEdge    ( filter_t *, picture_t *, picture_t * );
  51 static void FilterHough   ( filter_t *, picture_t *, picture_t * );
  52
  53 /*****************************************************************************
  54  * Module descriptor
  55  *****************************************************************************/
  56 #define MODE_TEXT N_("Distort mode")
  57 #define MODE_LONGTEXT N_("Distort mode, one of \"gradient\", \"edge\" and \"hough\".")
  58
  59 #define GRADIENT_TEXT N_("Gradient image type")
  60 #define GRADIENT_LONGTEXT N_("Gradient image type (0 or 1). 0 will " \
  61         "turn the image to white while 1 will keep colors." )
  62
  63 #define CARTOON_TEXT N_("Apply cartoon effect")
  64 #define CARTOON_LONGTEXT N_("Apply cartoon effect. It is only used by " \
  65     "\"gradient\" and \"edge\".")
  66
  67 static const char *mode_list[] = { "gradient", "edge", "hough" };
  68 static const char *mode_list_text[] = { N_("Gradient"), N_("Edge"), N_("Hough") };
  69
  70 #define FILTER_PREFIX "gradient-"
  71
  72 vlc_module_begin();
  73     set_description( _("Gradient video filter") );
  74     set_shortname( _( "Gradient" ));
  75     set_capability( "video filter2", 0 );
  76     set_category( CAT_VIDEO );
  77     set_subcategory( SUBCAT_VIDEO_VFILTER );
  78
  79     add_string( FILTER_PREFIX "mode", "gradient", NULL,
  80                 MODE_TEXT, MODE_LONGTEXT, VLC_FALSE );
  81         change_string_list( mode_list, mode_list_text, 0 );
  82
  83     add_integer_with_range( FILTER_PREFIX "type", 0, 0, 1, NULL,
  84                 GRADIENT_TEXT, GRADIENT_LONGTEXT, VLC_FALSE );
  85     add_bool( FILTER_PREFIX "cartoon", 1, NULL,
  86                 CARTOON_TEXT, CARTOON_LONGTEXT, VLC_FALSE );
  87
  88     add_shortcut( "gradient" );
  89     set_callbacks( Create, Destroy );
  90 vlc_module_end();
  91
  92 static const char *ppsz_filter_options[] = {
  93     "mode", "type", "cartoon", NULL
  94 };
  95
  96 /*****************************************************************************
  97  * vout_sys_t: Distort video output method descriptor
  98  *****************************************************************************
  99  * This structure is part of the video output thread descriptor.
 100  * It describes the Distort specific properties of an output thread.
 101  *****************************************************************************/
 102 struct filter_sys_t
 103 {
 104     int i_mode;
 105
 106     /* For the gradient mode */
 107     int i_gradient_type;
 108     vlc_bool_t b_cartoon;
 109
 110     uint32_t *p_buf32;
 111     uint32_t *p_buf32_bis;
 112     uint8_t *p_buf8;
 113
 114     /* For hough mode */
 115     int *p_pre_hough;
 116 };
 117
 118 /*****************************************************************************
 119  * Create: allocates Distort video thread output method
 120  *****************************************************************************
 121  * This function allocates and initializes a Distort vout method.
 122  *****************************************************************************/
 123 static int Create( vlc_object_t *p_this )
 124 {
 125     filter_t *p_filter = (filter_t *)p_this;
 126     char *psz_method;
 127
 128     /* Allocate structure */
 129     p_filter->p_sys = malloc( sizeof( filter_sys_t ) );
 130     if( p_filter->p_sys == NULL )
 131     {
 132         msg_Err( p_filter, "out of memory" );
 133         return VLC_ENOMEM;
 134     }
 135
 136     p_filter->pf_video_filter = Filter;
 137
 138     p_filter->p_sys->p_pre_hough = NULL;
 139
 140     config_ChainParse( p_filter, FILTER_PREFIX, ppsz_filter_options,
 141                    p_filter->p_cfg );
 142
 143     var_Create( p_filter, FILTER_PREFIX "mode",
 144                 VLC_VAR_STRING | VLC_VAR_DOINHERIT );
 145     var_Create( p_filter, FILTER_PREFIX "type",
 146                 VLC_VAR_INTEGER | VLC_VAR_DOINHERIT );
 147     var_Create( p_filter, FILTER_PREFIX "cartoon",
 148                 VLC_VAR_BOOL | VLC_VAR_DOINHERIT );
 149
 150     if( !(psz_method = var_GetString( p_filter, FILTER_PREFIX "mode" )) )
 151     {
 152         msg_Err( p_filter, "configuration variable "
 153                  FILTER_PREFIX "mode empty" );
 154         p_filter->p_sys->i_mode = GRADIENT;
 155     }
 156     else
 157     {
 158         if( !strcmp( psz_method, "gradient" ) )
 159         {
 160             p_filter->p_sys->i_mode = GRADIENT;
 161         }
 162         else if( !strcmp( psz_method, "edge" ) )
 163         {
 164             p_filter->p_sys->i_mode = EDGE;
 165         }
 166         else if( !strcmp( psz_method, "hough" ) )
 167         {
 168             p_filter->p_sys->i_mode = HOUGH;
 169         }
 170         else
 171         {
 172             msg_Err( p_filter, "no valid gradient mode provided" );
 173             p_filter->p_sys->i_mode = GRADIENT;
 174         }
 175     }
 176     free( psz_method );
 177
 178     p_filter->p_sys->i_gradient_type =
 179         var_GetInteger( p_filter, FILTER_PREFIX "type" );
 180     p_filter->p_sys->b_cartoon =
 181         var_GetInteger( p_filter, FILTER_PREFIX "cartoon" );
 182
 183     p_filter->p_sys->p_buf32 = NULL;
 184     p_filter->p_sys->p_buf32_bis = NULL;
 185     p_filter->p_sys->p_buf8 = NULL;
 186
 187     return VLC_SUCCESS;
 188 }
 189
 190 /*****************************************************************************
 191  * Destroy: destroy Distort video thread output method
 192  *****************************************************************************
 193  * Terminate an output method created by DistortCreateOutputMethod
 194  *****************************************************************************/
 195 static void Destroy( vlc_object_t *p_this )
 196 {
 197     filter_t *p_filter = (filter_t *)p_this;
 198
 199     free( p_filter->p_sys->p_buf32 );
 200     free( p_filter->p_sys->p_buf32_bis );
 201     free( p_filter->p_sys->p_buf8 );
 202     free( p_filter->p_sys->p_pre_hough );
 203
 204     free( p_filter->p_sys );
 205 }
 206
 207 /*****************************************************************************
 208  * Render: displays previously rendered output
 209  *****************************************************************************
 210  * This function send the currently rendered image to Distort image, waits
 211  * until it is displayed and switch the two rendering buffers, preparing next
 212  * frame.
 213  *****************************************************************************/
 214 static picture_t *Filter( filter_t *p_filter, picture_t *p_pic )
 215 {
 216     picture_t *p_outpic;
 217
 218     if( !p_pic ) return NULL;
 219
 220     p_outpic = p_filter->pf_vout_buffer_new( p_filter );
 221     if( !p_outpic )
 222     {
 223         msg_Warn( p_filter, "can't get output picture" );
 224         if( p_pic->pf_release )
 225             p_pic->pf_release( p_pic );
 226         return NULL;
 227     }
 228
 229     switch( p_filter->p_sys->i_mode )
 230     {
 231         case EDGE:
 232             FilterEdge( p_filter, p_pic, p_outpic );
 233             break;
 234
 235         case GRADIENT:
 236             FilterGradient( p_filter, p_pic, p_outpic );
 237             break;
 238
 239         case HOUGH:
 240             FilterHough( p_filter, p_pic, p_outpic );
 241             break;
 242
 243         default:
 244             break;
 245     }
 246
 247     p_outpic->date = p_pic->date;
 248     p_outpic->b_force = p_pic->b_force;
 249     p_outpic->i_nb_fields = p_pic->i_nb_fields;
 250     p_outpic->b_progressive = p_pic->b_progressive;
 251     p_outpic->b_top_field_first = p_pic->b_top_field_first;
 252
 253     if( p_pic->pf_release )
 254         p_pic->pf_release( p_pic );
 255
 256     return p_outpic;
 257 }
 258
 259
 260 /*****************************************************************************
 261  * Gaussian Convolution
 262  *****************************************************************************
 263  *    Gaussian convolution ( sigma == 1.4 )
 264  *
 265  *    |  2  4  5  4  2  |   |  2  4  4  4  2 |
 266  *    |  4  9 12  9  4  |   |  4  8 12  8  4 |
 267  *    |  5 12 15 12  5  | ~ |  4 12 16 12  4 |
 268  *    |  4  9 12  9  4  |   |  4  8 12  8  4 |
 269  *    |  2  4  5  4  2  |   |  2  4  4  4  2 |
 270  *****************************************************************************/
 271 static void GaussianConvolution( picture_t *p_inpic, uint32_t *p_smooth )
 272 {
 273     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
 274     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
 275     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
 276     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
 277
 278     int x,y;
 279     for( y = 2; y < i_num_lines - 2; y++ )
 280     {
 281         for( x = 2; x < i_src_visible - 2; x++ )
 282         {
 283             p_smooth[y*i_src_visible+x] = (uint32_t)(
 284               /* 2 rows up */
 285                 ( p_inpix[(y-2)*i_src_pitch+x-2] )
 286               + ((p_inpix[(y-2)*i_src_pitch+x-1]
 287               +   p_inpix[(y-2)*i_src_pitch+x]
 288               +   p_inpix[(y-2)*i_src_pitch+x+1])<<1 )
 289               + ( p_inpix[(y-2)*i_src_pitch+x+2] )
 290               /* 1 row up */
 291               + ((p_inpix[(y-1)*i_src_pitch+x-2]
 292               + ( p_inpix[(y-1)*i_src_pitch+x-1]<<1 )
 293               + ( p_inpix[(y-1)*i_src_pitch+x]*3 )
 294               + ( p_inpix[(y-1)*i_src_pitch+x+1]<<1 )
 295               +   p_inpix[(y-1)*i_src_pitch+x+2]
 296               /* */
 297               +   p_inpix[y*i_src_pitch+x-2]
 298               + ( p_inpix[y*i_src_pitch+x-1]*3 )
 299               + ( p_inpix[y*i_src_pitch+x]<<2 )
 300               + ( p_inpix[y*i_src_pitch+x+1]*3 )
 301               +   p_inpix[y*i_src_pitch+x+2]
 302               /* 1 row down */
 303               +   p_inpix[(y+1)*i_src_pitch+x-2]
 304               + ( p_inpix[(y+1)*i_src_pitch+x-1]<<1 )
 305               + ( p_inpix[(y+1)*i_src_pitch+x]*3 )
 306               + ( p_inpix[(y+1)*i_src_pitch+x+1]<<1 )
 307               +   p_inpix[(y+1)*i_src_pitch+x+2] )<<1 )
 308               /* 2 rows down */
 309               + ( p_inpix[(y+2)*i_src_pitch+x-2] )
 310               + ((p_inpix[(y+2)*i_src_pitch+x-1]
 311               +   p_inpix[(y+2)*i_src_pitch+x]
 312               +   p_inpix[(y+2)*i_src_pitch+x+1])<<1 )
 313               + ( p_inpix[(y+2)*i_src_pitch+x+2] )
 314               ) >> 6 /* 115 */;
 315         }
 316     }
 317 }
 318
 319 /*****************************************************************************
 320  * FilterGradient: Sobel
 321  *****************************************************************************/
 322 static void FilterGradient( filter_t *p_filter, picture_t *p_inpic,
 323                                                 picture_t *p_outpic )
 324 {
 325     int x, y;
 326     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
 327     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
 328     const int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
 329     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
 330
 331     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
 332     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
 333
 334     uint32_t *p_smooth;
 335     if( !p_filter->p_sys->p_buf32 )
 336         p_filter->p_sys->p_buf32 =
 337         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
 338     p_smooth = p_filter->p_sys->p_buf32;
 339
 340     if( !p_smooth ) return;
 341
 342     if( p_filter->p_sys->b_cartoon )
 343     {
 344         p_filter->p_libvlc->pf_memcpy( p_outpic->p[U_PLANE].p_pixels,
 345             p_inpic->p[U_PLANE].p_pixels,
 346             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
 347         p_filter->p_libvlc->pf_memcpy( p_outpic->p[V_PLANE].p_pixels,
 348             p_inpic->p[V_PLANE].p_pixels,
 349             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
 350     }
 351     else
 352     {
 353         p_filter->p_libvlc->pf_memset( p_outpic->p[U_PLANE].p_pixels, 0x80,
 354             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
 355         p_filter->p_libvlc->pf_memset( p_outpic->p[V_PLANE].p_pixels, 0x80,
 356             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
 357     }
 358
 359     GaussianConvolution( p_inpic, p_smooth );
 360
 361     /* Sobel gradient
 362
 363      | -1 0 1 |     |  1  2  1 |
 364      | -2 0 2 | and |  0  0  0 |
 365      | -1 0 1 |     | -1 -2 -1 | */
 366
 367 #define FOR                                                     \
 368     for( y = 1; y < i_num_lines - 1; y++ )                      \
 369     {                                                           \
 370         for( x = 1; x < i_src_visible - 1; x++ )                \
 371         {                                                       \
 372             const uint32_t a =                                  \
 373             (                                                   \
 374               abs(                                              \
 375                  ( p_smooth[(y-1)*i_src_visible+x-1]            \
 376                    - p_smooth[(y+1)*i_src_visible+x-1] )        \
 377                + ( ( p_smooth[(y-1)*i_src_visible+x]            \
 378                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )   \
 379                + ( p_smooth[(y-1)*i_src_visible+x+1]            \
 380                    - p_smooth[(y+1)*i_src_visible+x+1] )        \
 381               )                                                 \
 382             +                                                   \
 383               abs(                                              \
 384                  ( p_smooth[(y-1)*i_src_visible+x-1]            \
 385                    - p_smooth[(y-1)*i_src_visible+x+1] )        \
 386                + ( ( p_smooth[y*i_src_visible+x-1]              \
 387                     - p_smooth[y*i_src_visible+x+1] ) <<1 )     \
 388                + ( p_smooth[(y+1)*i_src_visible+x-1]            \
 389                    - p_smooth[(y+1)*i_src_visible+x+1] )        \
 390               )                                                 \
 391             );
 392     if( p_filter->p_sys->i_gradient_type )
 393     {
 394         if( p_filter->p_sys->b_cartoon )
 395         {
 396             FOR
 397             if( a > 60 )
 398             {
 399                 p_outpix[y*i_dst_pitch+x] = 0x00;
 400             }
 401             else
 402             {
 403                 if( p_smooth[y*i_src_visible+x] > 0xa0 )
 404                     p_outpix[y*i_dst_pitch+x] =
 405                         0xff - ((0xff - p_inpix[y*i_src_pitch+x] )>>2);
 406                 else if( p_smooth[y*i_src_visible+x] > 0x70 )
 407                     p_outpix[y*i_dst_pitch+x] =
 408                         0xa0 - ((0xa0 - p_inpix[y*i_src_pitch+x] )>>2);
 409                 else if( p_smooth[y*i_src_visible+x] > 0x28 )
 410                     p_outpix[y*i_dst_pitch+x] =
 411                         0x70 - ((0x70 - p_inpix[y*i_src_pitch+x] )>>2);
 412                 else
 413                     p_outpix[y*i_dst_pitch+x] =
 414                         0x28 - ((0x28 - p_inpix[y*i_src_pitch+x] )>>2);
 415             }
 416             }}
 417         }
 418         else
 419         {
 420             FOR
 421             p_outpix[y*i_dst_pitch+x] = clip_uint8_vlc( a );
 422             }}
 423         }
 424     }
 425     else
 426     {
 427         FOR
 428         if( a>>8 )
 429             p_outpix[y*i_dst_pitch+x] = 0;
 430         else
 431             p_outpix[y*i_dst_pitch+x] = 0xff-(uint8_t)a;
 432         }}
 433     }
 434 #undef FOR
 435 }
 436
 437 /*****************************************************************************
 438  * FilterEdge: Canny edge detection algorithm
 439  *****************************************************************************
 440  * http://fourier.eng.hmc.edu/e161/lectures/canny/node1.html
 441  * (well ... my implementation isn't really the canny algorithm ... but some
 442  * ideas are the same)
 443  *****************************************************************************/
 444 /* angle : | */
 445 #define THETA_Y 0
 446 /* angle : - */
 447 #define THETA_X 1
 448 /* angle : / */
 449 #define THETA_P 2
 450 /* angle : \ */
 451 #define THETA_M 3
 452 static void FilterEdge( filter_t *p_filter, picture_t *p_inpic,
 453                                             picture_t *p_outpic )
 454 {
 455     int x, y;
 456
 457     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
 458     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
 459     const int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
 460     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
 461
 462     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
 463     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
 464
 465     uint32_t *p_smooth;
 466     uint32_t *p_grad;
 467     uint8_t *p_theta;
 468
 469     if( !p_filter->p_sys->p_buf32 )
 470         p_filter->p_sys->p_buf32 =
 471         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
 472     p_smooth = p_filter->p_sys->p_buf32;
 473
 474     if( !p_filter->p_sys->p_buf32_bis )
 475         p_filter->p_sys->p_buf32_bis =
 476         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
 477     p_grad = p_filter->p_sys->p_buf32_bis;
 478
 479     if( !p_filter->p_sys->p_buf8 )
 480         p_filter->p_sys->p_buf8 =
 481         (uint8_t *)malloc( i_num_lines * i_src_visible * sizeof(uint8_t));
 482     p_theta = p_filter->p_sys->p_buf8;
 483
 484     if( !p_smooth || !p_grad || !p_theta ) return;
 485
 486     if( p_filter->p_sys->b_cartoon )
 487     {
 488         p_filter->p_libvlc->pf_memcpy( p_outpic->p[U_PLANE].p_pixels,
 489             p_inpic->p[U_PLANE].p_pixels,
 490             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
 491         p_filter->p_libvlc->pf_memcpy( p_outpic->p[V_PLANE].p_pixels,
 492             p_inpic->p[V_PLANE].p_pixels,
 493             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
 494     }
 495     else
 496     {
 497         p_filter->p_libvlc->pf_memset( p_outpic->p[Y_PLANE].p_pixels, 0xff,
 498               p_outpic->p[Y_PLANE].i_lines * p_outpic->p[Y_PLANE].i_pitch );
 499         p_filter->p_libvlc->pf_memset( p_outpic->p[U_PLANE].p_pixels, 0x80,
 500             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
 501         p_filter->p_libvlc->pf_memset( p_outpic->p[V_PLANE].p_pixels, 0x80,
 502             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
 503     }
 504
 505     GaussianConvolution( p_inpic, p_smooth );
 506
 507     /* Sobel gradient
 508
 509      | -1 0 1 |     |  1  2  1 |
 510      | -2 0 2 | and |  0  0  0 |
 511      | -1 0 1 |     | -1 -2 -1 | */
 512
 513     for( y = 1; y < i_num_lines - 1; y++ )
 514     {
 515         for( x = 1; x < i_src_visible - 1; x++ )
 516         {
 517
 518             const int gradx =
 519                  ( p_smooth[(y-1)*i_src_visible+x-1]
 520                    - p_smooth[(y+1)*i_src_visible+x-1] )
 521                + ( ( p_smooth[(y-1)*i_src_visible+x]
 522                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )
 523                + ( p_smooth[(y-1)*i_src_visible+x+1]
 524                    - p_smooth[(y+1)*i_src_visible+x+1] );
 525             const int grady =
 526                  ( p_smooth[(y-1)*i_src_visible+x-1]
 527                    - p_smooth[(y-1)*i_src_visible+x+1] )
 528                + ( ( p_smooth[y*i_src_visible+x-1]
 529                     - p_smooth[y*i_src_visible+x+1] ) <<1 )
 530                + ( p_smooth[(y+1)*i_src_visible+x-1]
 531                    - p_smooth[(y+1)*i_src_visible+x+1] );
 532
 533             p_grad[y*i_src_visible+x] = (uint32_t)(abs( gradx ) + abs( grady ));
 534
 535             /* tan( 22.5 ) = 0,414213562 .. * 128 = 53
 536              * tan( 26,565051177 ) = 0.5
 537              * tan( 45 + 22.5 ) = 2,414213562 .. * 128 = 309
 538              * tan( 63,434948823 ) 2 */
 539             if( (grady<<1) > gradx )
 540                 p_theta[y*i_src_visible+x] = THETA_P;
 541             else if( (grady<<1) < -gradx )
 542                 p_theta[y*i_src_visible+x] = THETA_M;
 543             else if( !gradx || abs(grady) > abs(gradx)<<1 )
 544                 p_theta[y*i_src_visible+x] = THETA_Y;
 545             else
 546                 p_theta[y*i_src_visible+x] = THETA_X;
 547         }
 548     }
 549
 550     /* edge computing */
 551     for( y = 1; y < i_num_lines - 1; y++ )
 552     {
 553         for( x = 1; x < i_src_visible - 1; x++ )
 554         {
 555             if( p_grad[y*i_src_visible+x] > 40 )
 556             {
 557                 switch( p_theta[y*i_src_visible+x] )
 558                 {
 559                     case THETA_Y:
 560                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x]
 561                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x] )
 562                         {
 563                             p_outpix[y*i_dst_pitch+x] = 0;
 564                             break;
 565                         } else goto colorize;
 566                     case THETA_P:
 567                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x-1]
 568                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x+1] )
 569                         {
 570                             p_outpix[y*i_dst_pitch+x] = 0;
 571                             break;
 572                         } else goto colorize;
 573                     case THETA_M:
 574                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x+1]
 575                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x-1] )
 576                         {
 577                             p_outpix[y*i_dst_pitch+x] = 0;
 578                             break;
 579                         } else goto colorize;
 580                     case THETA_X:
 581                         if(    p_grad[y*i_src_visible+x] > p_grad[y*i_src_visible+x-1]
 582                             && p_grad[y*i_src_visible+x] > p_grad[y*i_src_visible+x+1] )
 583                         {
 584                             p_outpix[y*i_dst_pitch+x] = 0;
 585                             break;
 586                         } else goto colorize;
 587                 }
 588             }
 589             else
 590             {
 591                 colorize:
 592                 if( p_filter->p_sys->b_cartoon )
 593                 {
 594                     if( p_smooth[y*i_src_visible+x] > 0xa0 )
 595                         p_outpix[y*i_dst_pitch+x] = (uint8_t)
 596                             0xff - ((0xff - p_inpix[y*i_src_pitch+x] )>>2);
 597                     else if( p_smooth[y*i_src_visible+x] > 0x70 )
 598                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
 599                             0xa0 - ((0xa0 - p_inpix[y*i_src_pitch+x] )>>2);
 600                     else if( p_smooth[y*i_src_visible+x] > 0x28 )
 601                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
 602                             0x70 - ((0x70 - p_inpix[y*i_src_pitch+x] )>>2);
 603                     else
 604                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
 605                             0x28 - ((0x28 - p_inpix[y*i_src_pitch+x] )>>2);
 606                 }
 607             }
 608         }
 609     }
 610 }
 611
 612 /*****************************************************************************
 613  * FilterHough
 614  *****************************************************************************/
 615 #define p_pre_hough p_filter->p_sys->p_pre_hough
 616 static void FilterHough( filter_t *p_filter, picture_t *p_inpic,
 617                                              picture_t *p_outpic )
 618 {
 619     int x, y, i;
 620     int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
 621     int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
 622     int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
 623
 624     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
 625
 626     int i_diag = sqrt( i_num_lines * i_num_lines +
 627                         i_src_visible * i_src_visible);
 628     int i_max, i_phi_max, i_rho, i_rho_max;
 629     int i_nb_steps = 90;
 630     double d_step = M_PI / i_nb_steps;
 631     double d_sin;
 632     double d_cos;
 633     uint32_t *p_smooth;
 634     int *p_hough = malloc( i_diag * i_nb_steps * sizeof(int) );
 635     if( ! p_hough ) return;
 636     p_smooth = (uint32_t *)malloc( i_num_lines*i_src_visible*sizeof(uint32_t));
 637     if( !p_smooth ) return;
 638
 639     if( ! p_pre_hough )
 640     {
 641         msg_Dbg(p_filter, "Starting precalculation");
 642         p_pre_hough = malloc( i_num_lines*i_src_visible*i_nb_steps*sizeof(int));
 643         if( ! p_pre_hough ) return;
 644         for( i = 0 ; i < i_nb_steps ; i++)
 645         {
 646             d_sin = sin(d_step * i);
 647             d_cos = cos(d_step * i);
 648             for( y = 0 ; y < i_num_lines ; y++ )
 649                 for( x = 0 ; x < i_src_visible ; x++ )
 650                 {
 651                     p_pre_hough[(i*i_num_lines+y)*i_src_visible + x] =
 652                         ceil(x*d_sin + y*d_cos);
 653                 }
 654         }
 655         msg_Dbg(p_filter, "Precalculation done");
 656     }
 657
 658     p_filter->p_libvlc->pf_memset( p_hough, 0,
 659                                    i_diag * i_nb_steps * sizeof(int) );
 660
 661     p_filter->p_libvlc->pf_memcpy(
 662         p_outpic->p[Y_PLANE].p_pixels, p_inpic->p[Y_PLANE].p_pixels,
 663         p_outpic->p[Y_PLANE].i_lines * p_outpic->p[Y_PLANE].i_pitch );
 664     p_filter->p_libvlc->pf_memcpy(
 665         p_outpic->p[U_PLANE].p_pixels, p_inpic->p[U_PLANE].p_pixels,
 666         p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
 667     p_filter->p_libvlc->pf_memcpy(
 668         p_outpic->p[V_PLANE].p_pixels, p_inpic->p[V_PLANE].p_pixels,
 669         p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
 670
 671     GaussianConvolution( p_inpic, p_smooth );
 672
 673     /* Sobel gradient
 674
 675      | -1 0 1 |     |  1  2  1 |
 676      | -2 0 2 | and |  0  0  0 |
 677      | -1 0 1 |     | -1 -2 -1 | */
 678
 679     i_max = 0;
 680     i_rho_max = 0;
 681     i_phi_max = 0;
 682     for( y = 4; y < i_num_lines - 4; y++ )
 683     {
 684         for( x = 4; x < i_src_visible - 4; x++ )
 685         {
 686             uint32_t a =
 687             (
 688               abs(
 689                 ( ( p_smooth[(y-1)*i_src_visible+x]
 690                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )
 691                + ( p_smooth[(y-1)*i_src_visible+x-1]
 692                    - p_smooth[(y+1)*i_src_visible+x-1] )
 693                + ( p_smooth[(y-1)*i_src_visible+x+1]
 694                    - p_smooth[(y+1)*i_src_visible+x+1] )
 695               )
 696             +
 697               abs(
 698                 ( ( p_smooth[y*i_src_visible+x-1]
 699                     - p_smooth[y*i_src_visible+x+1] ) <<1 )
 700                + ( p_smooth[(y-1)*i_src_visible+x-1]
 701                    - p_smooth[(y-1)*i_src_visible+x+1] )
 702                + ( p_smooth[(y+1)*i_src_visible+x-1]
 703                    - p_smooth[(y+1)*i_src_visible+x+1] )
 704               )
 705             );
 706             if( a>>8 )
 707             {
 708                 for( i = 0 ; i < i_nb_steps ; i ++ )
 709                 {
 710                     i_rho = p_pre_hough[(i*i_num_lines+y)*i_src_visible + x];
 711                     if( p_hough[i_rho + i_diag/2 + i * i_diag]++ > i_max )
 712                     {
 713                         i_max = p_hough[i_rho + i_diag/2 + i * i_diag];
 714                         i_rho_max = i_rho;
 715                         i_phi_max = i;
 716                     }
 717                 }
 718             }
 719         }
 720     }
 721
 722     d_sin = sin(i_phi_max*d_step);
 723     d_cos = cos(i_phi_max*d_step);
 724     if( d_cos != 0 )
 725     {
 726         for( x = 0 ; x < i_src_visible ; x++ )
 727         {
 728             y = (i_rho_max - x * d_sin) / d_cos;
 729             if( y >= 0 && y < i_num_lines )
 730                 p_outpix[y*i_dst_pitch+x] = 255;
 731         }
 732     }
 733
 734     if( p_hough ) free( p_hough );
 735     if( p_smooth ) free( p_smooth );
 736 }
 737 #undef p_pre_hough