git.sesse.net Git - vlc/blob - modules/video_filter/opencv_wrapper.c

   1 /*****************************************************************************
   2  * opencv_wrapper.c : OpenCV wrapper video filter
   3  *****************************************************************************
   4  * Copyright (C) 2006-2012 the VideoLAN team
   5  * Copyright (C) 2012 Edward Wang
   6  *
   7  * Authors: Dugal Harris <dugalh@protoclea.co.za>
   8  *          Edward Wang <edward.c.wang@compdigitec.com>
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License as published by
  12  * the Free Software Foundation; either version 2 of the License, or
  13  * (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU General Public License
  21  * along with this program; if not, write to the Free Software
  22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  23  *****************************************************************************/
  24
  25 /*****************************************************************************
  26  * Preamble
  27  *****************************************************************************/
  28
  29 #ifdef HAVE_CONFIG_H
  30 # include "config.h"
  31 #endif
  32
  33 #include <vlc_common.h>
  34 #include <vlc_plugin.h>
  35 #include <vlc_vout.h>
  36 #include <vlc_modules.h>
  37
  38 #include <vlc_filter.h>
  39 #include <vlc_image.h>
  40 #include "filter_picture.h"
  41
  42 #include <cxcore.h>
  43 #include <cv.h>
  44
  45 /*****************************************************************************
  46  * Local prototypes
  47  *****************************************************************************/
  48 static int  Create    ( vlc_object_t * );
  49 static void Destroy   ( vlc_object_t * );
  50
  51 static picture_t* Filter( filter_t*, picture_t* );
  52
  53 static void ReleaseImages( filter_t* p_filter );
  54 static void VlcPictureToIplImage( filter_t* p_filter, picture_t* p_in );
  55
  56 /*****************************************************************************
  57  * Module descriptor
  58  *****************************************************************************/
  59
  60 static const char *const chroma_list[] = { "input", "I420", "RGB32"};
  61 static const char *const chroma_list_text[] = { N_("Use input chroma unaltered"),
  62   N_("I420 - first plane is greyscale"), N_("RGB32")};
  63
  64 static const char *const output_list[] = { "none", "input", "processed"};
  65 static const char *const output_list_text[] = { N_("Don't display any video"),
  66   N_("Display the input video"), N_("Display the processed video")};
  67
  68 static const char *const verbosity_list[] = { "error", "warning", "debug"};
  69 static const char *const verbosity_list_text[] = { N_("Show only errors"),
  70   N_("Show errors and warnings"), N_("Show everything including debug messages")};
  71
  72 vlc_module_begin ()
  73     set_description( N_("OpenCV video filter wrapper") )
  74     set_shortname( N_("OpenCV" ))
  75     set_category( CAT_VIDEO )
  76     set_subcategory( SUBCAT_VIDEO_VFILTER )
  77     set_capability( "video filter2", 0 )
  78     add_shortcut( "opencv_wrapper" )
  79     set_callbacks( Create, Destroy )
  80     add_float_with_range( "opencv-scale", 1.0, 0.1, 2.0,
  81                           N_("Scale factor (0.1-2.0)"),
  82                           N_("Ammount by which to scale the picture before sending it to the internal OpenCV filter"),
  83                           false )
  84     add_string( "opencv-chroma", "input",
  85                           N_("OpenCV filter chroma"),
  86                           N_("Chroma to convert picture to before sending it to the internal OpenCV filter"), false);
  87         change_string_list( chroma_list, chroma_list_text, 0);
  88     add_string( "opencv-output", "input",
  89                           N_("Wrapper filter output"),
  90                           N_("Determines what (if any) video is displayed by the wrapper filter"), false);
  91         change_string_list( output_list, output_list_text, 0);
  92     add_string( "opencv-filter-name", "none",
  93                           N_("OpenCV internal filter name"),
  94                           N_("Name of internal OpenCV plugin filter to use"), false);
  95 vlc_module_end ()
  96
  97
  98 /*****************************************************************************
  99  * wrapper_output_t: what video is output
 100  *****************************************************************************/
 101 enum wrapper_output_t
 102 {
 103    NONE,
 104    VINPUT,
 105    PROCESSED
 106 };
 107
 108 /*****************************************************************************
 109  * internal_chroma_t: what chroma is sent to the internal opencv filter
 110  *****************************************************************************/
 111 enum internal_chroma_t
 112 {
 113    CINPUT,
 114    GREY,
 115    RGB
 116 };
 117
 118 /*****************************************************************************
 119  * filter_sys_t: opencv_wrapper video output method descriptor
 120  *****************************************************************************
 121  * This structure is part of the video output thread descriptor.
 122  * It describes the opencv_wrapper specific properties of an output thread.
 123  *****************************************************************************/
 124 struct filter_sys_t
 125 {
 126     image_handler_t *p_image;
 127
 128     int i_cv_image_size;
 129
 130     picture_t *p_proc_image;
 131     picture_t *p_to_be_freed;
 132
 133     float f_scale;
 134
 135     int i_wrapper_output;
 136     int i_internal_chroma;
 137
 138     IplImage *p_cv_image[VOUT_MAX_PLANES];
 139
 140     filter_t *p_opencv;
 141     char* psz_inner_name;
 142
 143     picture_t hacked_pic;
 144 };
 145
 146 /*****************************************************************************
 147  * Create: allocates opencv_wrapper video thread output method
 148  *****************************************************************************
 149  * This function allocates and initializes a opencv_wrapper vout method.
 150  *****************************************************************************/
 151 static int Create( vlc_object_t *p_this )
 152 {
 153     filter_t* p_filter = (filter_t*)p_this;
 154     char *psz_chroma, *psz_output;
 155
 156     /* Allocate structure */
 157     p_filter->p_sys = malloc( sizeof( filter_sys_t ) );
 158     if( p_filter->p_sys == NULL )
 159         return VLC_ENOMEM;
 160
 161     /* Load the internal OpenCV filter.
 162      *
 163      * This filter object is needed to call the internal OpenCV filter
 164      * for processing, the wrapper just converts into an IplImage* for
 165      * the other filter.
 166      *
 167      * We don't need to set up video formats for this filter as it not
 168      * actually using a picture_t.
 169      */
 170     p_filter->p_sys->p_opencv = vlc_object_create( p_filter, sizeof(filter_t) );
 171     if( !p_filter->p_sys->p_opencv ) {
 172         free( p_filter->p_sys );
 173         return VLC_ENOMEM;
 174     }
 175
 176     p_filter->p_sys->psz_inner_name = var_InheritString( p_filter, "opencv-filter-name" );
 177     if( p_filter->p_sys->psz_inner_name )
 178         p_filter->p_sys->p_opencv->p_module =
 179             module_need( p_filter->p_sys->p_opencv,
 180                          "opencv internal filter",
 181                          p_filter->p_sys->psz_inner_name,
 182                          true );
 183
 184     if( !p_filter->p_sys->p_opencv->p_module )
 185     {
 186         msg_Err( p_filter, "can't open internal opencv filter: %s", p_filter->p_sys->psz_inner_name );
 187         free( p_filter->p_sys->psz_inner_name );
 188         p_filter->p_sys->psz_inner_name = NULL;
 189         vlc_object_release( p_filter->p_sys->p_opencv );
 190         free( p_filter->p_sys );
 191
 192         return VLC_ENOMOD;
 193     }
 194
 195
 196     /* Init structure */
 197     p_filter->p_sys->p_image = image_HandlerCreate( p_filter );
 198     for( int i = 0; i < VOUT_MAX_PLANES; i++ )
 199         p_filter->p_sys->p_cv_image[i] = NULL;
 200     p_filter->p_sys->p_proc_image = NULL;
 201     p_filter->p_sys->p_to_be_freed = NULL;
 202     p_filter->p_sys->i_cv_image_size = 0;
 203
 204     /* Retrieve and apply config */
 205     psz_chroma = var_InheritString( p_filter, "opencv-chroma" );
 206     if( psz_chroma == NULL )
 207     {
 208         msg_Err( p_filter, "configuration variable %s empty, using 'grey'",
 209                          "opencv-chroma" );
 210         p_filter->p_sys->i_internal_chroma = GREY;
 211     } else if( !strcmp( psz_chroma, "input" ) )
 212         p_filter->p_sys->i_internal_chroma = CINPUT;
 213     else if( !strcmp( psz_chroma, "I420" ) )
 214         p_filter->p_sys->i_internal_chroma = GREY;
 215     else if( !strcmp( psz_chroma, "RGB32" ) )
 216         p_filter->p_sys->i_internal_chroma = RGB;
 217     else {
 218         msg_Err( p_filter, "no valid opencv-chroma provided, using 'grey'" );
 219         p_filter->p_sys->i_internal_chroma = GREY;
 220     }
 221
 222     free( psz_chroma );
 223
 224     psz_output = var_InheritString( p_filter, "opencv-output" );
 225     if( psz_output == NULL )
 226     {
 227         msg_Err( p_filter, "configuration variable %s empty, using 'input'",
 228                          "opencv-output" );
 229         p_filter->p_sys->i_wrapper_output = VINPUT;
 230     } else if( !strcmp( psz_output, "none" ) )
 231         p_filter->p_sys->i_wrapper_output = NONE;
 232     else if( !strcmp( psz_output, "input" ) )
 233         p_filter->p_sys->i_wrapper_output = VINPUT;
 234     else if( !strcmp( psz_output, "processed" ) )
 235         p_filter->p_sys->i_wrapper_output = PROCESSED;
 236     else {
 237         msg_Err( p_filter, "no valid opencv-output provided, using 'input'" );
 238         p_filter->p_sys->i_wrapper_output = VINPUT;
 239     }
 240     free( psz_output );
 241
 242     p_filter->p_sys->f_scale =
 243         var_InheritFloat( p_filter, "opencv-scale" );
 244
 245     msg_Info(p_filter, "Configuration: opencv-scale: %f, opencv-chroma: %d, "
 246         "opencv-output: %d, opencv-filter %s",
 247         p_filter->p_sys->f_scale,
 248         p_filter->p_sys->i_internal_chroma,
 249         p_filter->p_sys->i_wrapper_output,
 250         p_filter->p_sys->psz_inner_name);
 251
 252 #ifndef NDEBUG
 253     msg_Dbg( p_filter, "opencv_wrapper successfully started" );
 254 #endif
 255
 256     p_filter->pf_video_filter = Filter;
 257
 258     return VLC_SUCCESS;
 259 }
 260
 261 /*****************************************************************************
 262  * Destroy: destroy opencv_wrapper video thread output method
 263  *****************************************************************************
 264  * Terminate an output method created by opencv_wrapperCreateOutputMethod
 265  *****************************************************************************/
 266 static void Destroy( vlc_object_t *p_this )
 267 {
 268     filter_t* p_filter = (filter_t*)p_this;
 269     ReleaseImages( p_filter );
 270
 271     // Release the internal OpenCV filter.
 272     module_unneed( p_filter->p_sys->p_opencv, p_filter->p_sys->p_opencv->p_module );
 273     vlc_object_release( p_filter->p_sys->p_opencv );
 274     p_filter->p_sys->p_opencv = NULL;
 275
 276     free( p_filter->p_sys );
 277 }
 278
 279 /*****************************************************************************
 280  * ReleaseImages: Release OpenCV images in filter_sys_t.
 281  *****************************************************************************/
 282 static void ReleaseImages( filter_t* p_filter )
 283 {
 284     filter_sys_t* p_sys = p_filter->p_sys;
 285
 286     if (p_sys->p_cv_image)
 287     {
 288         for( int i = 0; i < VOUT_MAX_PLANES; i++ )
 289         {
 290             if (p_sys->p_cv_image[i]) {
 291                 cvReleaseImageHeader(&(p_sys->p_cv_image[i]));
 292                 p_sys->p_cv_image[i] = NULL;
 293             }
 294         }
 295     }
 296     p_sys->i_cv_image_size = 0;
 297
 298     /* Release temp picture_t if it exists */
 299     if (p_sys->p_to_be_freed)
 300     {
 301         picture_Release( p_sys->p_to_be_freed );
 302         p_sys->p_to_be_freed = NULL;
 303     }
 304
 305 #ifndef NDEBUG
 306     msg_Dbg( p_filter, "images released" );
 307 #endif
 308 }
 309
 310 /*****************************************************************************
 311  * VlcPictureToIplImage: Convert picture_t to IplImage
 312  *****************************************************************************
 313  * Converts given picture_t into IplImage(s) according to module config.
 314  * IplImage(s) are stored in vout_sys_t.
 315  *****************************************************************************/
 316 static void VlcPictureToIplImage( filter_t* p_filter, picture_t* p_in )
 317 {
 318     int planes = p_in->i_planes;    //num input video planes
 319     // input video size
 320     CvSize sz = cvSize(abs(p_in->format.i_width), abs(p_in->format.i_height));
 321     video_format_t fmt_out;
 322     filter_sys_t* p_sys = p_filter->p_sys;
 323
 324     memset( &fmt_out, 0, sizeof(video_format_t) );
 325
 326     //do scale / color conversion according to p_sys config
 327     if ((p_sys->f_scale != 1) || (p_sys->i_internal_chroma != CINPUT))
 328     {
 329         fmt_out = p_in->format;
 330
 331         //calc the scaled video size
 332         fmt_out.i_width = p_in->format.i_width * p_sys->f_scale;
 333         fmt_out.i_height = p_in->format.i_height * p_sys->f_scale;
 334
 335         if (p_sys->i_internal_chroma == RGB)
 336         {
 337             //rgb2 gives 3 separate planes, this gives 1 interleaved plane
 338             //rv24 gives is about 20% faster but gives r&b the wrong way round
 339             //and I can't think of an easy way to fix this
 340             fmt_out.i_chroma = VLC_CODEC_RGB24;
 341         }
 342         else if (p_sys->i_internal_chroma == GREY)
 343         {
 344             //take the I (gray) plane (video seems to commonly be in this fmt so usually the
 345             //conversion does nothing)
 346             fmt_out.i_chroma = VLC_CODEC_I420;
 347         }
 348
 349         //convert from the input image
 350         p_sys->p_proc_image = image_Convert( p_sys->p_image, p_in,
 351                                      &(p_in->format), &fmt_out );
 352
 353         if (!p_sys->p_proc_image)
 354         {
 355             msg_Err(p_filter, "can't convert (unsupported formats?), aborting...");
 356             return;
 357         }
 358
 359         p_sys->p_to_be_freed = p_sys->p_proc_image;    //remember this so we can free it later
 360
 361     }
 362     else    //((p_sys->f_scale != 1) || (p_sys->i_internal_chroma != CINPUT))
 363     {
 364         // In theory, you could use the input image without conversion,
 365         // but it seems to cause weird picture effects (like repeated
 366         // image filtering) and picture leaking.
 367         p_sys->p_proc_image = filter_NewPicture( p_filter ); //p_in
 368         picture_Copy( p_sys->p_proc_image, p_in );
 369         p_sys->p_to_be_freed = p_sys->p_proc_image;
 370     }
 371
 372     //Convert to the IplImage array that is to be processed.
 373     //If there are multiple planes in p_sys->p_proc_image, then 1 IplImage
 374     //is created for each plane.
 375     planes = p_sys->p_proc_image->i_planes;
 376     p_sys->i_cv_image_size = planes;
 377     for( int i = 0; i < planes; i++ )
 378     {
 379         sz = cvSize(abs(p_sys->p_proc_image->p[i].i_visible_pitch /
 380             p_sys->p_proc_image->p[i].i_pixel_pitch),
 381             abs(p_sys->p_proc_image->p[i].i_visible_lines));
 382
 383         p_sys->p_cv_image[i] = cvCreateImageHeader(sz, IPL_DEPTH_8U,
 384             p_sys->p_proc_image->p[i].i_pixel_pitch);
 385
 386         cvSetData( p_sys->p_cv_image[i],
 387             (char*)(p_sys->p_proc_image->p[i].p_pixels), p_sys->p_proc_image->p[i].i_pitch );
 388     }
 389
 390     //Hack the above opencv image array into a picture_t so that it can be sent to
 391     //another video filter
 392     p_sys->hacked_pic.i_planes = planes;
 393     p_sys->hacked_pic.format.i_chroma = fmt_out.i_chroma;
 394
 395 #ifndef NDEBUG
 396     msg_Dbg( p_filter, "VlcPictureToIplImageRgb() completed" );
 397 #endif
 398 }
 399
 400 /*****************************************************************************
 401  * Filter: displays previously rendered output
 402  *****************************************************************************
 403  * This function send the currently rendered image to the internal opencv
 404  * filter for processing.
 405  *****************************************************************************/
 406 static picture_t* Filter( filter_t* p_filter, picture_t* p_pic )
 407 {
 408     picture_t* p_outpic = filter_NewPicture( p_filter );
 409     if( p_outpic == NULL ) {
 410         msg_Err( p_filter, "couldn't get a p_outpic!" );
 411         picture_Release( p_pic );
 412         return NULL;
 413     }
 414
 415     video_format_t fmt_out;
 416
 417     // Make a copy if we want to show the original input
 418     if (p_filter->p_sys->i_wrapper_output == VINPUT)
 419         picture_Copy( p_outpic, p_pic );
 420
 421     VlcPictureToIplImage( p_filter, p_pic );
 422     // Pass the image (as a pointer to the first IplImage*) to the
 423     // internal OpenCV filter for processing.
 424     p_filter->p_sys->p_opencv->pf_video_filter( p_filter->p_sys->p_opencv, &(p_filter->p_sys->p_cv_image[0]) );
 425
 426     if(p_filter->p_sys->i_wrapper_output == PROCESSED) {
 427         // Processed video
 428         if( (p_filter->p_sys->p_proc_image) &&
 429             (p_filter->p_sys->p_proc_image->i_planes > 0) &&
 430             (p_filter->p_sys->i_internal_chroma != CINPUT) ) {
 431             //p_filter->p_sys->p_proc_image->format.i_chroma = VLC_CODEC_RGB24;
 432
 433             memset( &fmt_out, 0, sizeof(video_format_t) );
 434             fmt_out = p_pic->format;
 435             //picture_Release( p_outpic );
 436
 437             /*
 438              * We have to copy out the image from image_Convert(), otherwise
 439              * you leak pictures for some reason:
 440              * main video output error: pictures leaked, trying to workaround
 441              */
 442             picture_t* p_outpic_tmp = image_Convert(
 443                         p_filter->p_sys->p_image,
 444                         p_filter->p_sys->p_proc_image,
 445                         &(p_filter->p_sys->p_proc_image->format),
 446                         &fmt_out );
 447
 448             picture_CopyPixels( p_outpic, p_outpic_tmp );
 449             CopyInfoAndRelease( p_outpic, p_outpic_tmp );
 450         } else if( p_filter->p_sys->i_internal_chroma == CINPUT ) {
 451             picture_CopyPixels( p_outpic, p_filter->p_sys->p_proc_image );
 452             picture_CopyProperties( p_outpic, p_filter->p_sys->p_proc_image );
 453         }
 454     }
 455
 456     ReleaseImages( p_filter );
 457     picture_Release( p_pic );
 458
 459 #ifndef NDEBUG
 460     msg_Dbg( p_filter, "Filter() done" );
 461 #endif
 462
 463     if( p_filter->p_sys->i_wrapper_output != NONE ) {
 464         return p_outpic;
 465     } else { // NONE
 466         picture_Release( p_outpic );
 467         return NULL;
 468     }
 469 }
 470