]> git.sesse.net Git - vlc/blob - modules/video_filter/opencv_wrapper.c
Use var_InheritString for --decklink-video-connection.
[vlc] / modules / video_filter / opencv_wrapper.c
1 /*****************************************************************************
2  * opencv_wrapper.c : OpenCV wrapper video filter
3  *****************************************************************************
4  * Copyright (C) 2006 the VideoLAN team
5  *
6  * Authors: Dugal Harris <dugalh@protoclea.co.za>
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16  * GNU General Public License for more details.
17  *
18  * You should have received a copy of the GNU General Public License
19  * along with this program; if not, write to the Free Software
20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
21  *****************************************************************************/
22
23 /*****************************************************************************
24  * Preamble
25  *****************************************************************************/
26
27 #ifdef HAVE_CONFIG_H
28 # include "config.h"
29 #endif
30
31 #include <vlc_common.h>
32 #include <vlc_plugin.h>
33 #include <vlc_vout.h>
34 #include <vlc_modules.h>
35
36 #include <math.h>
37 #include <time.h>
38
39 #include <vlc_filter.h>
40 #include <vlc_image.h>
41 #include <vlc_input.h>
42
43 #include <cxcore.h>
44 #include <cv.h>
45
46
47 /*****************************************************************************
48  * Local prototypes
49  *****************************************************************************/
50 static int  Create    ( vlc_object_t * );
51 static void Destroy   ( vlc_object_t * );
52
53 static int  Init      ( vout_thread_t * );
54 static void End       ( vout_thread_t * );
55 static void Render    ( vout_thread_t *, picture_t * );
56
57 static void ReleaseImages( vout_thread_t *p_vout );
58 static void VlcPictureToIplImage( vout_thread_t *p_vout, picture_t *p_in );
59
60 /*****************************************************************************
61  * Module descriptor
62  *****************************************************************************/
63
64 static const char *const chroma_list[] = { "input", "I420", "RGB32"};
65 static const char *const chroma_list_text[] = { N_("Use input chroma unaltered"),
66   N_("I420 - first plane is greyscale"), N_("RGB32")};
67
68 static const char *const output_list[] = { "none", "input", "processed"};
69 static const char *const output_list_text[] = { N_("Don't display any video"),
70   N_("Display the input video"), N_("Display the processed video")};
71
72 static const char *const verbosity_list[] = { "error", "warning", "debug"};
73 static const char *const verbosity_list_text[] = { N_("Show only errors"),
74   N_("Show errors and warnings"), N_("Show everything including debug messages")};
75
76 vlc_module_begin ()
77     set_description( N_("OpenCV video filter wrapper") )
78     set_shortname( N_("OpenCV" ))
79     set_category( CAT_VIDEO )
80     set_subcategory( SUBCAT_VIDEO_VFILTER )
81     set_capability( "video filter", 0 )
82     add_shortcut( "opencv_wrapper" )
83     set_callbacks( Create, Destroy )
84     add_float_with_range( "opencv-scale", 1.0, 0.1, 2.0, NULL,
85                           N_("Scale factor (0.1-2.0)"),
86                           N_("Ammount by which to scale the picture before sending it to the internal OpenCV filter"),
87                           false )
88     add_string( "opencv-chroma", "input", NULL,
89                           N_("OpenCV filter chroma"),
90                           N_("Chroma to convert picture to before sending it to the internal OpenCV filter"), false);
91         change_string_list( chroma_list, chroma_list_text, 0);
92     add_string( "opencv-output", "input", NULL,
93                           N_("Wrapper filter output"),
94                           N_("Determines what (if any) video is displayed by the wrapper filter"), false);
95         change_string_list( output_list, output_list_text, 0);
96     add_string( "opencv-verbosity", "error", NULL,
97                           N_("Wrapper filter verbosity"),
98                           N_("Determines wrapper filter verbosity level"), false);
99         change_string_list( verbosity_list, verbosity_list_text, 0);
100     add_string( "opencv-filter-name", "none", NULL,
101                           N_("OpenCV internal filter name"),
102                           N_("Name of internal OpenCV plugin filter to use"), false);
103 vlc_module_end ()
104
105
106 /*****************************************************************************
107  * wrapper_output_t: what video is output
108  *****************************************************************************/
109 enum wrapper_output_t
110 {
111    NONE,    //not working yet
112    VINPUT,
113    PROCESSED
114 };
115
116 /*****************************************************************************
117  * internal_chroma_t: what chroma is sent to the internal opencv filter
118  *****************************************************************************/
119 enum internal_chroma_t
120 {
121    CINPUT,
122    GREY,
123    RGB
124 };
125
126 /*****************************************************************************
127  * verbosity_t:
128  *****************************************************************************/
129 enum verbosity_t
130 {
131    VERB_ERROR,
132    VERB_WARN,
133    VERB_DEBUG
134 };
135
136 /*****************************************************************************
137  * vout_sys_t: opencv_wrapper video output method descriptor
138  *****************************************************************************
139  * This structure is part of the video output thread descriptor.
140  * It describes the opencv_wrapper specific properties of an output thread.
141  *****************************************************************************/
142 struct vout_sys_t
143 {
144     vout_thread_t *p_vout;
145
146     image_handler_t *p_image;
147
148     int i_cv_image_size;
149
150     picture_t *p_proc_image;
151     picture_t *p_to_be_freed;
152
153     float f_scale;
154
155     int i_wrapper_output;
156     int i_internal_chroma;
157     int i_verbosity;
158
159     IplImage *p_cv_image[VOUT_MAX_PLANES];
160
161     filter_t *p_opencv;
162     char* psz_inner_name;
163
164     picture_t hacked_pic;
165 };
166
167 /*****************************************************************************
168  * Control: control facility for the vout (forwards to child vout)
169  *****************************************************************************/
170 static int Control( vout_thread_t *p_vout, int i_query, va_list args )
171 {
172     return vout_vaControl( p_vout->p_sys->p_vout, i_query, args );
173 }
174
175 /*****************************************************************************
176  * Create: allocates opencv_wrapper video thread output method
177  *****************************************************************************
178  * This function allocates and initializes a opencv_wrapper vout method.
179  *****************************************************************************/
180 static int Create( vlc_object_t *p_this )
181 {
182     vout_thread_t *p_vout = (vout_thread_t *)p_this;
183     char *psz_chroma, *psz_output, *psz_verbosity;
184     int i = 0;
185
186     /* Allocate structure */
187     p_vout->p_sys = malloc( sizeof( vout_sys_t ) );
188     if( p_vout->p_sys == NULL )
189         return VLC_ENOMEM;
190
191     /* Init structure */
192     p_vout->p_sys->p_image = image_HandlerCreate( p_vout );
193     for (i = 0; i < VOUT_MAX_PLANES; i++)
194         p_vout->p_sys->p_cv_image[i] = NULL;
195     p_vout->p_sys->p_proc_image = NULL;
196     p_vout->p_sys->p_to_be_freed = NULL;
197     p_vout->p_sys->i_cv_image_size = 0;
198
199     p_vout->pf_init = Init;
200     p_vout->pf_end = End;
201     p_vout->pf_manage = NULL;
202     p_vout->pf_render = Render;
203     p_vout->pf_display = NULL;
204     p_vout->pf_control = Control;
205
206     /* Retrieve and apply config */
207     psz_chroma = var_InheritString( p_vout, "opencv-chroma" );
208     if( psz_chroma == NULL )
209     {
210         msg_Err( p_vout, "configuration variable %s empty, using 'grey'",
211                          "opencv-chroma" );
212         p_vout->p_sys->i_internal_chroma = GREY;
213     }
214     else
215     {
216         if( !strcmp( psz_chroma, "input" ) )
217             p_vout->p_sys->i_internal_chroma = CINPUT;
218         else if( !strcmp( psz_chroma, "I420" ) )
219             p_vout->p_sys->i_internal_chroma = GREY;
220         else if( !strcmp( psz_chroma, "RGB32" ) )
221             p_vout->p_sys->i_internal_chroma = RGB;
222         else
223         {
224             msg_Err( p_vout, "no valid opencv-chroma provided, using 'grey'" );
225             p_vout->p_sys->i_internal_chroma = GREY;
226         }
227     }
228     free( psz_chroma);
229
230     psz_output = var_InheritString( p_vout, "opencv-output" );
231     if( psz_output == NULL )
232     {
233         msg_Err( p_vout, "configuration variable %s empty, using 'input'",
234                          "opencv-output" );
235         p_vout->p_sys->i_wrapper_output = VINPUT;
236     }
237     else
238     {
239         if( !strcmp( psz_output, "none" ) )
240             p_vout->p_sys->i_wrapper_output = NONE;
241         else if( !strcmp( psz_output, "input" ) )
242             p_vout->p_sys->i_wrapper_output = VINPUT;
243         else if( !strcmp( psz_output, "processed" ) )
244             p_vout->p_sys->i_wrapper_output = PROCESSED;
245         else
246         {
247             msg_Err( p_vout, "no valid opencv-output provided, using 'input'" );
248             p_vout->p_sys->i_wrapper_output = VINPUT;
249         }
250     }
251     free( psz_output);
252
253     psz_verbosity = var_InheritString( p_vout, "opencv-verbosity" );
254     if( psz_verbosity == NULL )
255     {
256         msg_Err( p_vout, "configuration variable %s empty, using 'input'",
257                          "opencv-verbosity" );
258         p_vout->p_sys->i_verbosity = VERB_ERROR;
259     }
260     else
261     {
262         if( !strcmp( psz_verbosity, "error" ) )
263             p_vout->p_sys->i_verbosity = VERB_ERROR;
264         else if( !strcmp( psz_verbosity, "warning" ) )
265             p_vout->p_sys->i_verbosity = VERB_WARN;
266         else if( !strcmp( psz_verbosity, "debug" ) )
267             p_vout->p_sys->i_verbosity = VERB_DEBUG;
268         else
269         {
270             msg_Err( p_vout, "no valid opencv-verbosity provided, using 'error'" );
271             p_vout->p_sys->i_verbosity = VERB_ERROR;
272         }
273     }
274     free( psz_verbosity);
275
276     p_vout->p_sys->psz_inner_name =
277         var_InheritString( p_vout, "opencv-filter-name" );
278     p_vout->p_sys->f_scale =
279         var_InheritFloat( p_vout, "opencv-scale" );
280
281     if (p_vout->p_sys->i_verbosity > VERB_WARN)
282         msg_Info(p_vout, "Configuration: opencv-scale: %f, opencv-chroma: %d, "
283             "opencv-output: %d, opencv-verbosity %d, opencv-filter %s",
284             p_vout->p_sys->f_scale,
285             p_vout->p_sys->i_internal_chroma,
286             p_vout->p_sys->i_wrapper_output,
287             p_vout->p_sys->i_verbosity,
288             p_vout->p_sys->psz_inner_name);
289
290     return VLC_SUCCESS;
291 }
292
293 /*****************************************************************************
294  * Init: initialize opencv_wrapper video thread output method
295  *****************************************************************************/
296 static int Init( vout_thread_t *p_vout )
297 {
298     video_format_t fmt;
299     vout_sys_t *p_sys = p_vout->p_sys;
300     I_OUTPUTPICTURES = 0;
301
302     /* Initialize the output video format */
303     memset( &fmt, 0, sizeof(video_format_t) );
304     p_vout->output.i_chroma = p_vout->render.i_chroma;
305     p_vout->output.i_width  = p_vout->render.i_width;
306     p_vout->output.i_height = p_vout->render.i_height;
307     p_vout->output.i_aspect = p_vout->render.i_aspect;
308     p_vout->fmt_out = p_vout->fmt_in;           //set to input video format
309
310     fmt = p_vout->fmt_out;
311     if (p_sys->i_wrapper_output == PROCESSED)   //set to processed video format
312     {
313         fmt.i_width = fmt.i_width * p_sys->f_scale;
314         fmt.i_height = fmt.i_height * p_sys->f_scale;
315         fmt.i_visible_width = fmt.i_visible_width * p_sys->f_scale;
316         fmt.i_visible_height = fmt.i_visible_height * p_sys->f_scale;
317         fmt.i_x_offset = fmt.i_x_offset * p_sys->f_scale;
318         fmt.i_y_offset = fmt.i_y_offset * p_sys->f_scale;
319
320         if (p_sys->i_internal_chroma == GREY)
321             fmt.i_chroma = VLC_CODEC_I420;
322         else if (p_sys->i_internal_chroma == RGB)
323             fmt.i_chroma = VLC_CODEC_RGB32;
324     }
325
326     /* Load the internal opencv filter */
327     /* We don't need to set up video formats for this filter as it not actually using a picture_t */
328     p_sys->p_opencv = vlc_object_create( p_vout, sizeof(filter_t) );
329     vlc_object_attach( p_sys->p_opencv, p_vout );
330
331     if (p_vout->p_sys->psz_inner_name)
332         p_sys->p_opencv->p_module =
333             module_need( p_sys->p_opencv, p_sys->psz_inner_name, NULL, false );
334
335     if( !p_sys->p_opencv->p_module )
336     {
337         msg_Err( p_vout, "can't open internal opencv filter: %s", p_vout->p_sys->psz_inner_name );
338         p_vout->p_sys->psz_inner_name = NULL;
339         vlc_object_release( p_sys->p_opencv );
340         p_sys->p_opencv = NULL;
341     }
342
343     /* Try to open the real video output */
344     if (p_sys->i_verbosity > VERB_WARN)
345         msg_Dbg( p_vout, "spawning the real video output" );
346
347     p_vout->p_sys->p_vout = vout_Create( p_vout, &fmt );
348
349     /* Everything failed */
350     if( p_vout->p_sys->p_vout == NULL )
351     {
352         msg_Err( p_vout, "can't open vout, aborting" );
353         return VLC_EGENERIC;
354     }
355
356     vout_filter_AllocateDirectBuffers( p_vout, VOUT_MAX_PICTURES );
357
358     vout_filter_AddChild( p_vout, p_vout->p_sys->p_vout, NULL );
359
360     return VLC_SUCCESS;
361 }
362
363 /*****************************************************************************
364  * End: terminate opencv_wrapper video thread output method
365  *****************************************************************************/
366 static void End( vout_thread_t *p_vout )
367 {
368     vout_sys_t *p_sys = p_vout->p_sys;
369
370     vout_filter_DelChild( p_vout, p_sys->p_vout, NULL );
371     vout_CloseAndRelease( p_sys->p_vout );
372
373     vout_filter_ReleaseDirectBuffers( p_vout );
374
375     if( p_sys->p_opencv )
376     {
377         //release the internal opencv filter
378         if( p_sys->p_opencv->p_module )
379             module_unneed( p_sys->p_opencv, p_sys->p_opencv->p_module );
380         vlc_object_release( p_sys->p_opencv );
381         p_sys->p_opencv = NULL;
382     }
383 }
384
385 /*****************************************************************************
386  * Destroy: destroy opencv_wrapper video thread output method
387  *****************************************************************************
388  * Terminate an output method created by opencv_wrapperCreateOutputMethod
389  *****************************************************************************/
390 static void Destroy( vlc_object_t *p_this )
391 {
392     vout_thread_t *p_vout = (vout_thread_t *)p_this;
393
394     ReleaseImages(p_vout);
395
396     if( p_vout->p_sys->p_image )
397         image_HandlerDelete( p_vout->p_sys->p_image );
398
399     free( p_vout->p_sys );
400 }
401
402 /*****************************************************************************
403  * ReleaseImages: Release OpenCV images in vout_sys_t.
404  *****************************************************************************/
405 static void ReleaseImages(vout_thread_t *p_vout)
406 {
407     int i = 0;
408     if (p_vout->p_sys->p_cv_image)
409     {
410         for (i = 0; i < VOUT_MAX_PLANES; i++)
411         {
412             if (p_vout->p_sys->p_cv_image[i])
413                 cvReleaseImageHeader(&(p_vout->p_sys->p_cv_image[i]));
414             p_vout->p_sys->p_cv_image[i] = NULL;
415         }
416     }
417     p_vout->p_sys->i_cv_image_size = 0;
418
419     /* Release temp picture_t if it exists */
420     if (p_vout->p_sys->p_to_be_freed)
421     {
422         picture_Release( p_vout->p_sys->p_to_be_freed );
423         p_vout->p_sys->p_to_be_freed = NULL;
424     }
425     if (p_vout->p_sys->i_verbosity > VERB_WARN)
426         msg_Dbg( p_vout, "images released" );
427 }
428
429 /*****************************************************************************
430  * VlcPictureToIplImage: Convert picture_t to IplImage
431  *****************************************************************************
432  * Converts given picture_t into IplImage(s) according to module config.
433  * IplImage(s) are stored in vout_sys_t.
434  *****************************************************************************/
435 static void VlcPictureToIplImage( vout_thread_t *p_vout, picture_t *p_in )
436 {
437     int planes = p_in->i_planes;    //num input video planes
438     // input video size
439     CvSize sz = cvSize(abs(p_in->format.i_width), abs(p_in->format.i_height));
440     video_format_t fmt_out;
441     clock_t start, finish;  //performance measures
442     double  duration;
443     int i = 0;
444     vout_sys_t* p_sys = p_vout->p_sys;
445
446     memset( &fmt_out, 0, sizeof(video_format_t) );
447
448     start = clock();
449
450     //do scale / color conversion according to p_sys config
451     if ((p_sys->f_scale != 1) || (p_sys->i_internal_chroma != CINPUT))
452     {
453         fmt_out = p_in->format;
454
455         //calc the scaled video size
456         fmt_out.i_width = p_in->format.i_width * p_sys->f_scale;
457         fmt_out.i_height = p_in->format.i_height * p_sys->f_scale;
458
459         if (p_sys->i_internal_chroma == RGB)
460         {
461             //rgb2 gives 3 separate planes, this gives 1 interleaved plane
462             //rv24 gives is about 20% faster but gives r&b the wrong way round
463             //and I cant think of an easy way to fix this
464             fmt_out.i_chroma = VLC_CODEC_RGB32;
465         }
466         else if (p_sys->i_internal_chroma == GREY)
467         {
468             //take the I (gray) plane (video seems to commonly be in this fmt so usually the
469             //conversion does nothing)
470             fmt_out.i_chroma = VLC_CODEC_I420;
471         }
472
473         //convert from the input image
474         p_sys->p_proc_image = image_Convert( p_sys->p_image, p_in,
475                                      &(p_in->format), &fmt_out );
476
477         if (!p_sys->p_proc_image)
478         {
479             msg_Err(p_vout, "can't convert (unsupported formats?), aborting...");
480             return;
481         }
482
483         p_sys->p_to_be_freed = p_sys->p_proc_image;    //remember this so we can free it later
484
485     }
486     else    //((p_sys->f_scale != 1) || (p_sys->i_internal_chroma != CINPUT))
487     {
488         //use the input image without conversion
489         p_sys->p_proc_image = p_in;
490     }
491
492     //Convert to the IplImage array that is to be processed.
493     //If there are multiple planes in p_sys->p_proc_image, then 1 IplImage
494     //is created for each plane.
495     planes = p_sys->p_proc_image->i_planes;
496     p_sys->i_cv_image_size = planes;
497     for ( i = 0; i < planes; i++ )
498     {
499         sz = cvSize(abs(p_sys->p_proc_image->p[i].i_visible_pitch /
500             p_sys->p_proc_image->p[i].i_pixel_pitch),
501             abs(p_sys->p_proc_image->p[i].i_visible_lines));
502
503         p_sys->p_cv_image[i] = cvCreateImageHeader(sz, IPL_DEPTH_8U,
504             p_sys->p_proc_image->p[i].i_pixel_pitch);
505
506         cvSetData( p_sys->p_cv_image[i],
507             (char*)(p_sys->p_proc_image->p[i].p_pixels), p_sys->p_proc_image->p[i].i_pitch );
508     }
509
510     //Hack the above opencv image array into a picture_t so that it can be sent to
511     //another video filter
512     p_sys->hacked_pic.p_data_orig = p_sys->p_cv_image;
513     p_sys->hacked_pic.i_planes = planes;
514     p_sys->hacked_pic.format.i_chroma = fmt_out.i_chroma;
515
516     //calculate duration of conversion
517     finish = clock();
518     duration = (double)(finish - start) / CLOCKS_PER_SEC;
519     if (p_sys->i_verbosity > VERB_WARN)
520         msg_Dbg( p_vout, "VlcPictureToIplImageRgb took %2.4f seconds", duration );
521 }
522
523 /*****************************************************************************
524  * Render: displays previously rendered output
525  *****************************************************************************
526  * This function send the currently rendered image to the internal opencv
527  * filter for processing.
528  *****************************************************************************/
529 static void Render( vout_thread_t *p_vout, picture_t *p_pic )
530 {
531     picture_t *p_outpic = NULL;
532     clock_t start, finish;
533     double  duration;
534
535     while( ( p_outpic = vout_CreatePicture( p_vout->p_sys->p_vout, 0, 0, 0 ) )
536               == NULL )
537     {
538         if( !vlc_object_alive (p_vout) || p_vout->b_error )
539         {   return; }
540         msleep( VOUT_OUTMEM_SLEEP );
541     }
542
543     vout_LinkPicture( p_vout->p_sys->p_vout, p_outpic );
544
545     start = clock();
546
547     if (p_vout->p_sys->i_wrapper_output == VINPUT)  //output = input video
548     {
549         //This copy is a bit unfortunate but image_Convert can't write into an existing image so it is better to copy the
550         //(say) 16bit YUV image here than a 32bit RGB image somehwere else.
551         //It is also not that expensive in time.
552         picture_Copy( p_outpic, p_pic );
553         VlcPictureToIplImage( p_vout, p_pic);
554         //pass the image to the internal opencv filter for processing
555         if ((p_vout->p_sys->p_opencv) && (p_vout->p_sys->p_opencv->p_module))
556             p_vout->p_sys->p_opencv->pf_video_filter( p_vout->p_sys->p_opencv, &(p_vout->p_sys->hacked_pic));
557     }
558     else    //output = processed video (NONE option not working yet)
559     {
560         VlcPictureToIplImage( p_vout, p_pic);
561         //pass the image to the internal opencv filter for processing
562         if ((p_vout->p_sys->p_opencv) && (p_vout->p_sys->p_opencv->p_module))
563             p_vout->p_sys->p_opencv->pf_video_filter( p_vout->p_sys->p_opencv, &(p_vout->p_sys->hacked_pic));
564         //copy the processed image into the output image
565         if ((p_vout->p_sys->p_proc_image) && (p_vout->p_sys->p_proc_image->i_planes > 0))
566             picture_Copy( p_outpic, p_vout->p_sys->p_proc_image );
567     }
568
569     //calculate duration
570     finish = clock();
571     duration = (double)(finish - start) / CLOCKS_PER_SEC;
572     if (p_vout->p_sys->i_verbosity > VERB_WARN)
573         msg_Dbg( p_vout, "Render took %2.4f seconds", duration );
574
575     ReleaseImages(p_vout);
576     p_outpic->date  = p_pic->date;
577     
578     vout_UnlinkPicture( p_vout->p_sys->p_vout, p_outpic );
579     vout_DisplayPicture( p_vout->p_sys->p_vout, p_outpic );
580 }
581