]> git.sesse.net Git - vlc/blob - modules/video_filter/opencv_wrapper.c
utf8_* -> vlc_* (sed roxxors)
[vlc] / modules / video_filter / opencv_wrapper.c
1 /*****************************************************************************
2  * opencv_wrapper.c : OpenCV wrapper video filter
3  *****************************************************************************
4  * Copyright (C) 2006 the VideoLAN team
5  *
6  * Authors: Dugal Harris <dugalh@protoclea.co.za>
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16  * GNU General Public License for more details.
17  *
18  * You should have received a copy of the GNU General Public License
19  * along with this program; if not, write to the Free Software
20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
21  *****************************************************************************/
22
23 /*****************************************************************************
24  * Preamble
25  *****************************************************************************/
26
27 #include <cxcore.h>
28 #include <cv.h>
29 #include <highgui.h>
30
31 #ifdef HAVE_CONFIG_H
32 # include "config.h"
33 #endif
34
35 #include <vlc_common.h>
36 #include <vlc_plugin.h>
37 #include <vlc_vout.h>
38
39 #include <math.h>
40 #include <time.h>
41
42 #include <vlc_filter.h>
43 #include "filter_common.h"
44 #include <vlc_image.h>
45 #include <vlc_input.h>
46 #include <vlc_playlist.h>
47
48
49 /*****************************************************************************
50  * Local prototypes
51  *****************************************************************************/
52 static int  Create    ( vlc_object_t * );
53 static void Destroy   ( vlc_object_t * );
54
55 static int  Init      ( vout_thread_t * );
56 static void End       ( vout_thread_t * );
57 static void Render    ( vout_thread_t *, picture_t * );
58
59 static void ReleaseImages( vout_thread_t *p_vout );
60 static void VlcPictureToIplImage( vout_thread_t *p_vout, picture_t *p_in );
61
62 /*****************************************************************************
63  * Module descriptor
64  *****************************************************************************/
65
66 static const char *const chroma_list[] = { "input", "I420", "RGB32"};
67 static const char *const chroma_list_text[] = { N_("Use input chroma unaltered"),
68   N_("I420 - first plane is greyscale"), N_("RGB32")};
69
70 static const char *const output_list[] = { "none", "input", "processed"};
71 static const char *const output_list_text[] = { N_("Don't display any video"),
72   N_("Display the input video"), N_("Display the processed video")};
73
74 static const char *const verbosity_list[] = { "error", "warning", "debug"};
75 static const char *const verbosity_list_text[] = { N_("Show only errors"),
76   N_("Show errors and warnings"), N_("Show everything including debug messages")};
77
78 vlc_module_begin ()
79     set_description( N_("OpenCV video filter wrapper") )
80     set_shortname( N_("OpenCV" ))
81     set_category( CAT_VIDEO )
82     set_subcategory( SUBCAT_VIDEO_VFILTER )
83     set_capability( "video filter", 0 )
84     add_shortcut( "opencv_wrapper" )
85     set_callbacks( Create, Destroy )
86     add_float_with_range( "opencv-scale", 1.0, 0.1, 2.0, NULL,
87                           N_("Scale factor (0.1-2.0)"),
88                           N_("Ammount by which to scale the picture before sending it to the internal OpenCV filter"),
89                           false )
90     add_string( "opencv-chroma", "input", NULL,
91                           N_("OpenCV filter chroma"),
92                           N_("Chroma to convert picture to before sending it to the internal OpenCV filter"), false);
93         change_string_list( chroma_list, chroma_list_text, 0);
94     add_string( "opencv-output", "input", NULL,
95                           N_("Wrapper filter output"),
96                           N_("Determines what (if any) video is displayed by the wrapper filter"), false);
97         change_string_list( output_list, output_list_text, 0);
98     add_string( "opencv-verbosity", "error", NULL,
99                           N_("Wrapper filter verbosity"),
100                           N_("Determines wrapper filter verbosity level"), false);
101         change_string_list( verbosity_list, verbosity_list_text, 0);
102     add_string( "opencv-filter-name", "none", NULL,
103                           N_("OpenCV internal filter name"),
104                           N_("Name of internal OpenCV plugin filter to use"), false);
105 vlc_module_end ()
106
107
108 /*****************************************************************************
109  * wrapper_output_t: what video is output
110  *****************************************************************************/
111 enum wrapper_output_t
112 {
113    NONE,    //not working yet
114    VINPUT,
115    PROCESSED
116 };
117
118 /*****************************************************************************
119  * internal_chroma_t: what chroma is sent to the internal opencv filter
120  *****************************************************************************/
121 enum internal_chroma_t
122 {
123    CINPUT,
124    GREY,
125    RGB
126 };
127
128 /*****************************************************************************
129  * verbosity_t:
130  *****************************************************************************/
131 enum verbosity_t
132 {
133    VERB_ERROR,
134    VERB_WARN,
135    VERB_DEBUG
136 };
137
138 /*****************************************************************************
139  * vout_sys_t: opencv_wrapper video output method descriptor
140  *****************************************************************************
141  * This structure is part of the video output thread descriptor.
142  * It describes the opencv_wrapper specific properties of an output thread.
143  *****************************************************************************/
144 struct vout_sys_t
145 {
146     vout_thread_t *p_vout;
147
148     image_handler_t *p_image;
149
150     int i_cv_image_size;
151
152     picture_t *p_proc_image;
153     picture_t *p_to_be_freed;
154
155     float f_scale;
156
157     int i_wrapper_output;
158     int i_internal_chroma;
159     int i_verbosity;
160
161     IplImage *p_cv_image[VOUT_MAX_PLANES];
162
163     filter_t *p_opencv;
164     char* psz_inner_name;
165
166     picture_t hacked_pic;
167 };
168
169 /*****************************************************************************
170  * Control: control facility for the vout (forwards to child vout)
171  *****************************************************************************/
172 static int Control( vout_thread_t *p_vout, int i_query, va_list args )
173 {
174     return vout_vaControl( p_vout->p_sys->p_vout, i_query, args );
175 }
176
177 /*****************************************************************************
178  * Create: allocates opencv_wrapper video thread output method
179  *****************************************************************************
180  * This function allocates and initializes a opencv_wrapper vout method.
181  *****************************************************************************/
182 static int Create( vlc_object_t *p_this )
183 {
184     vout_thread_t *p_vout = (vout_thread_t *)p_this;
185     char *psz_chroma, *psz_output, *psz_verbosity;
186     int i = 0;
187
188     /* Allocate structure */
189     p_vout->p_sys = malloc( sizeof( vout_sys_t ) );
190     if( p_vout->p_sys == NULL )
191         return VLC_ENOMEM;
192
193     /* Init structure */
194     p_vout->p_sys->p_image = image_HandlerCreate( p_vout );
195     for (i = 0; i < VOUT_MAX_PLANES; i++)
196         p_vout->p_sys->p_cv_image[i] = NULL;
197     p_vout->p_sys->p_proc_image = NULL;
198     p_vout->p_sys->p_to_be_freed = NULL;
199     p_vout->p_sys->i_cv_image_size = 0;
200
201     p_vout->pf_init = Init;
202     p_vout->pf_end = End;
203     p_vout->pf_manage = NULL;
204     p_vout->pf_render = Render;
205     p_vout->pf_display = NULL;
206     p_vout->pf_control = Control;
207
208     /* Retrieve and apply config */
209     psz_chroma = var_InheritString( p_vout, "opencv-chroma" );
210     if( psz_chroma == NULL )
211     {
212         msg_Err( p_vout, "configuration variable %s empty, using 'grey'",
213                          "opencv-chroma" );
214         p_vout->p_sys->i_internal_chroma = GREY;
215     }
216     else
217     {
218         if( !strcmp( psz_chroma, "input" ) )
219             p_vout->p_sys->i_internal_chroma = CINPUT;
220         else if( !strcmp( psz_chroma, "I420" ) )
221             p_vout->p_sys->i_internal_chroma = GREY;
222         else if( !strcmp( psz_chroma, "RGB32" ) )
223             p_vout->p_sys->i_internal_chroma = RGB;
224         else
225         {
226             msg_Err( p_vout, "no valid opencv-chroma provided, using 'grey'" );
227             p_vout->p_sys->i_internal_chroma = GREY;
228         }
229     }
230     free( psz_chroma);
231
232     psz_output = var_InheritString( p_vout, "opencv-output" );
233     if( psz_output == NULL )
234     {
235         msg_Err( p_vout, "configuration variable %s empty, using 'input'",
236                          "opencv-output" );
237         p_vout->p_sys->i_wrapper_output = VINPUT;
238     }
239     else
240     {
241         if( !strcmp( psz_output, "none" ) )
242             p_vout->p_sys->i_wrapper_output = NONE;
243         else if( !strcmp( psz_output, "input" ) )
244             p_vout->p_sys->i_wrapper_output = VINPUT;
245         else if( !strcmp( psz_output, "processed" ) )
246             p_vout->p_sys->i_wrapper_output = PROCESSED;
247         else
248         {
249             msg_Err( p_vout, "no valid opencv-output provided, using 'input'" );
250             p_vout->p_sys->i_wrapper_output = VINPUT;
251         }
252     }
253     free( psz_output);
254
255     psz_verbosity = var_InheritString( p_vout, "opencv-verbosity" );
256     if( psz_verbosity == NULL )
257     {
258         msg_Err( p_vout, "configuration variable %s empty, using 'input'",
259                          "opencv-verbosity" );
260         p_vout->p_sys->i_verbosity = VERB_ERROR;
261     }
262     else
263     {
264         if( !strcmp( psz_verbosity, "error" ) )
265             p_vout->p_sys->i_verbosity = VERB_ERROR;
266         else if( !strcmp( psz_verbosity, "warning" ) )
267             p_vout->p_sys->i_verbosity = VERB_WARN;
268         else if( !strcmp( psz_verbosity, "debug" ) )
269             p_vout->p_sys->i_verbosity = VERB_DEBUG;
270         else
271         {
272             msg_Err( p_vout, "no valid opencv-verbosity provided, using 'error'" );
273             p_vout->p_sys->i_verbosity = VERB_ERROR;
274         }
275     }
276     free( psz_verbosity);
277
278     p_vout->p_sys->psz_inner_name =
279         var_InheritString( p_vout, "opencv-filter-name" );
280     p_vout->p_sys->f_scale =
281         var_InheritFloat( p_vout, "opencv-scale" );
282
283     if (p_vout->p_sys->i_verbosity > VERB_WARN)
284         msg_Info(p_vout, "Configuration: opencv-scale: %f, opencv-chroma: %d, "
285             "opencv-output: %d, opencv-verbosity %d, opencv-filter %s",
286             p_vout->p_sys->f_scale,
287             p_vout->p_sys->i_internal_chroma,
288             p_vout->p_sys->i_wrapper_output,
289             p_vout->p_sys->i_verbosity,
290             p_vout->p_sys->psz_inner_name);
291
292     return VLC_SUCCESS;
293 }
294
295 /*****************************************************************************
296  * Init: initialize opencv_wrapper video thread output method
297  *****************************************************************************/
298 static int Init( vout_thread_t *p_vout )
299 {
300     video_format_t fmt;
301     vout_sys_t *p_sys = p_vout->p_sys;
302     I_OUTPUTPICTURES = 0;
303
304     /* Initialize the output video format */
305     memset( &fmt, 0, sizeof(video_format_t) );
306     p_vout->output.i_chroma = p_vout->render.i_chroma;
307     p_vout->output.i_width  = p_vout->render.i_width;
308     p_vout->output.i_height = p_vout->render.i_height;
309     p_vout->output.i_aspect = p_vout->render.i_aspect;
310     p_vout->fmt_out = p_vout->fmt_in;           //set to input video format
311
312     fmt = p_vout->fmt_out;
313     if (p_sys->i_wrapper_output == PROCESSED)   //set to processed video format
314     {
315         fmt.i_width = fmt.i_width * p_sys->f_scale;
316         fmt.i_height = fmt.i_height * p_sys->f_scale;
317         fmt.i_visible_width = fmt.i_visible_width * p_sys->f_scale;
318         fmt.i_visible_height = fmt.i_visible_height * p_sys->f_scale;
319         fmt.i_x_offset = fmt.i_x_offset * p_sys->f_scale;
320         fmt.i_y_offset = fmt.i_y_offset * p_sys->f_scale;
321
322         if (p_sys->i_internal_chroma == GREY)
323             fmt.i_chroma = VLC_CODEC_I420;
324         else if (p_sys->i_internal_chroma == RGB)
325             fmt.i_chroma = VLC_CODEC_RGB32;
326     }
327
328     /* Load the internal opencv filter */
329     /* We don't need to set up video formats for this filter as it not actually using a picture_t */
330     p_sys->p_opencv = vlc_object_create( p_vout, sizeof(filter_t) );
331     vlc_object_attach( p_sys->p_opencv, p_vout );
332
333     if (p_vout->p_sys->psz_inner_name)
334         p_sys->p_opencv->p_module =
335             module_need( p_sys->p_opencv, p_sys->psz_inner_name, NULL, false );
336
337     if( !p_sys->p_opencv->p_module )
338     {
339         msg_Err( p_vout, "can't open internal opencv filter: %s", p_vout->p_sys->psz_inner_name );
340         p_vout->p_sys->psz_inner_name = NULL;
341         vlc_object_release( p_sys->p_opencv );
342         p_sys->p_opencv = NULL;
343     }
344
345     /* Try to open the real video output */
346     if (p_sys->i_verbosity > VERB_WARN)
347         msg_Dbg( p_vout, "spawning the real video output" );
348
349     p_vout->p_sys->p_vout = vout_Create( p_vout, &fmt );
350
351     /* Everything failed */
352     if( p_vout->p_sys->p_vout == NULL )
353     {
354         msg_Err( p_vout, "can't open vout, aborting" );
355         return VLC_EGENERIC;
356     }
357
358     vout_filter_AllocateDirectBuffers( p_vout, VOUT_MAX_PICTURES );
359
360     vout_filter_AddChild( p_vout, p_vout->p_sys->p_vout, NULL );
361
362     return VLC_SUCCESS;
363 }
364
365 /*****************************************************************************
366  * End: terminate opencv_wrapper video thread output method
367  *****************************************************************************/
368 static void End( vout_thread_t *p_vout )
369 {
370     vout_sys_t *p_sys = p_vout->p_sys;
371
372     vout_filter_DelChild( p_vout, p_sys->p_vout, NULL );
373     vout_CloseAndRelease( p_sys->p_vout );
374
375     vout_filter_ReleaseDirectBuffers( p_vout );
376
377     if( p_sys->p_opencv )
378     {
379         //release the internal opencv filter
380         if( p_sys->p_opencv->p_module )
381             module_unneed( p_sys->p_opencv, p_sys->p_opencv->p_module );
382         vlc_object_release( p_sys->p_opencv );
383         p_sys->p_opencv = NULL;
384     }
385 }
386
387 /*****************************************************************************
388  * Destroy: destroy opencv_wrapper video thread output method
389  *****************************************************************************
390  * Terminate an output method created by opencv_wrapperCreateOutputMethod
391  *****************************************************************************/
392 static void Destroy( vlc_object_t *p_this )
393 {
394     vout_thread_t *p_vout = (vout_thread_t *)p_this;
395
396     ReleaseImages(p_vout);
397
398     if( p_vout->p_sys->p_image )
399         image_HandlerDelete( p_vout->p_sys->p_image );
400
401     free( p_vout->p_sys );
402 }
403
404 /*****************************************************************************
405  * ReleaseImages: Release OpenCV images in vout_sys_t.
406  *****************************************************************************/
407 static void ReleaseImages(vout_thread_t *p_vout)
408 {
409     int i = 0;
410     if (p_vout->p_sys->p_cv_image)
411     {
412         for (i = 0; i < VOUT_MAX_PLANES; i++)
413         {
414             if (p_vout->p_sys->p_cv_image[i])
415                 cvReleaseImageHeader(&(p_vout->p_sys->p_cv_image[i]));
416             p_vout->p_sys->p_cv_image[i] = NULL;
417         }
418     }
419     p_vout->p_sys->i_cv_image_size = 0;
420
421     /* Release temp picture_t if it exists */
422     if (p_vout->p_sys->p_to_be_freed)
423     {
424         picture_Release( p_vout->p_sys->p_to_be_freed );
425         p_vout->p_sys->p_to_be_freed = NULL;
426     }
427     if (p_vout->p_sys->i_verbosity > VERB_WARN)
428         msg_Dbg( p_vout, "images released" );
429 }
430
431 /*****************************************************************************
432  * VlcPictureToIplImage: Convert picture_t to IplImage
433  *****************************************************************************
434  * Converts given picture_t into IplImage(s) according to module config.
435  * IplImage(s) are stored in vout_sys_t.
436  *****************************************************************************/
437 static void VlcPictureToIplImage( vout_thread_t *p_vout, picture_t *p_in )
438 {
439     int planes = p_in->i_planes;    //num input video planes
440     // input video size
441     CvSize sz = cvSize(abs(p_in->format.i_width), abs(p_in->format.i_height));
442     video_format_t fmt_out;
443     clock_t start, finish;  //performance measures
444     double  duration;
445     int i = 0;
446     vout_sys_t* p_sys = p_vout->p_sys;
447
448     memset( &fmt_out, 0, sizeof(video_format_t) );
449
450     start = clock();
451
452     //do scale / color conversion according to p_sys config
453     if ((p_sys->f_scale != 1) || (p_sys->i_internal_chroma != CINPUT))
454     {
455         fmt_out = p_in->format;
456
457         //calc the scaled video size
458         fmt_out.i_width = p_in->format.i_width * p_sys->f_scale;
459         fmt_out.i_height = p_in->format.i_height * p_sys->f_scale;
460
461         if (p_sys->i_internal_chroma == RGB)
462         {
463             //rgb2 gives 3 separate planes, this gives 1 interleaved plane
464             //rv24 gives is about 20% faster but gives r&b the wrong way round
465             //and I cant think of an easy way to fix this
466             fmt_out.i_chroma = VLC_CODEC_RGB32;
467         }
468         else if (p_sys->i_internal_chroma == GREY)
469         {
470             //take the I (gray) plane (video seems to commonly be in this fmt so usually the
471             //conversion does nothing)
472             fmt_out.i_chroma = VLC_CODEC_I420;
473         }
474
475         //convert from the input image
476         p_sys->p_proc_image = image_Convert( p_sys->p_image, p_in,
477                                      &(p_in->format), &fmt_out );
478
479         if (!p_sys->p_proc_image)
480         {
481             msg_Err(p_vout, "can't convert (unsupported formats?), aborting...");
482             return;
483         }
484
485         p_sys->p_to_be_freed = p_sys->p_proc_image;    //remember this so we can free it later
486
487     }
488     else    //((p_sys->f_scale != 1) || (p_sys->i_internal_chroma != CINPUT))
489     {
490         //use the input image without conversion
491         p_sys->p_proc_image = p_in;
492     }
493
494     //Convert to the IplImage array that is to be processed.
495     //If there are multiple planes in p_sys->p_proc_image, then 1 IplImage
496     //is created for each plane.
497     planes = p_sys->p_proc_image->i_planes;
498     p_sys->i_cv_image_size = planes;
499     for ( i = 0; i < planes; i++ )
500     {
501         sz = cvSize(abs(p_sys->p_proc_image->p[i].i_visible_pitch /
502             p_sys->p_proc_image->p[i].i_pixel_pitch),
503             abs(p_sys->p_proc_image->p[i].i_visible_lines));
504
505         p_sys->p_cv_image[i] = cvCreateImageHeader(sz, IPL_DEPTH_8U,
506             p_sys->p_proc_image->p[i].i_pixel_pitch);
507
508         cvSetData( p_sys->p_cv_image[i],
509             (char*)(p_sys->p_proc_image->p[i].p_pixels), p_sys->p_proc_image->p[i].i_pitch );
510     }
511
512     //Hack the above opencv image array into a picture_t so that it can be sent to
513     //another video filter
514     p_sys->hacked_pic.p_data_orig = p_sys->p_cv_image;
515     p_sys->hacked_pic.i_planes = planes;
516     p_sys->hacked_pic.format.i_chroma = fmt_out.i_chroma;
517
518     //calculate duration of conversion
519     finish = clock();
520     duration = (double)(finish - start) / CLOCKS_PER_SEC;
521     if (p_sys->i_verbosity > VERB_WARN)
522         msg_Dbg( p_vout, "VlcPictureToIplImageRgb took %2.4f seconds", duration );
523 }
524
525 /*****************************************************************************
526  * Render: displays previously rendered output
527  *****************************************************************************
528  * This function send the currently rendered image to the internal opencv
529  * filter for processing.
530  *****************************************************************************/
531 static void Render( vout_thread_t *p_vout, picture_t *p_pic )
532 {
533     picture_t *p_outpic = NULL;
534     clock_t start, finish;
535     double  duration;
536
537     while( ( p_outpic = vout_CreatePicture( p_vout->p_sys->p_vout, 0, 0, 0 ) )
538               == NULL )
539     {
540         if( !vlc_object_alive (p_vout) || p_vout->b_error )
541         {   return; }
542         msleep( VOUT_OUTMEM_SLEEP );
543     }
544
545     vout_LinkPicture( p_vout->p_sys->p_vout, p_outpic );
546
547     start = clock();
548
549     if (p_vout->p_sys->i_wrapper_output == VINPUT)  //output = input video
550     {
551         //This copy is a bit unfortunate but image_Convert can't write into an existing image so it is better to copy the
552         //(say) 16bit YUV image here than a 32bit RGB image somehwere else.
553         //It is also not that expensive in time.
554         picture_Copy( p_outpic, p_pic );
555         VlcPictureToIplImage( p_vout, p_pic);
556         //pass the image to the internal opencv filter for processing
557         if ((p_vout->p_sys->p_opencv) && (p_vout->p_sys->p_opencv->p_module))
558             p_vout->p_sys->p_opencv->pf_video_filter( p_vout->p_sys->p_opencv, &(p_vout->p_sys->hacked_pic));
559     }
560     else    //output = processed video (NONE option not working yet)
561     {
562         VlcPictureToIplImage( p_vout, p_pic);
563         //pass the image to the internal opencv filter for processing
564         if ((p_vout->p_sys->p_opencv) && (p_vout->p_sys->p_opencv->p_module))
565             p_vout->p_sys->p_opencv->pf_video_filter( p_vout->p_sys->p_opencv, &(p_vout->p_sys->hacked_pic));
566         //copy the processed image into the output image
567         if ((p_vout->p_sys->p_proc_image) && (p_vout->p_sys->p_proc_image->p_data))
568             picture_Copy( p_outpic, p_vout->p_sys->p_proc_image );
569     }
570
571     //calculate duration
572     finish = clock();
573     duration = (double)(finish - start) / CLOCKS_PER_SEC;
574     if (p_vout->p_sys->i_verbosity > VERB_WARN)
575         msg_Dbg( p_vout, "Render took %2.4f seconds", duration );
576
577     ReleaseImages(p_vout);
578     p_outpic->date  = p_pic->date;
579     
580     vout_UnlinkPicture( p_vout->p_sys->p_vout, p_outpic );
581     vout_DisplayPicture( p_vout->p_sys->p_vout, p_outpic );
582 }
583