]> git.sesse.net Git - vlc/blob - modules/video_filter/remoteosd.c
OpenCV: include config.h first
[vlc] / modules / video_filter / remoteosd.c
1 /*****************************************************************************
2  * remoteosd.c: remote osd over vnc filter module
3  *****************************************************************************
4  * Copyright (C) 2007-2008 Matthias Bauer
5  * $Id$
6  *
7  * Authors: Matthias Bauer <matthias dot bauer #_at_# gmx dot ch>
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License as published by
11  * the Free Software Foundation; either version 2 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implid warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  * You should have received a copy of the GNU General Public License
20  * along with this program; if not, write to the Free Software
21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
22  *****************************************************************************/
23
24 /*****************************************************************************
25  * RemoteOSD uses the RFB-Protocol of VNC to display an On-Screen-Display
26  * menu generated by a streaming server as overlay for the streamed video.
27  *
28  * The streaming server that implements this is the ffnetdev plugin for VDR.
29  * VDR (VideoDiskRecorder) is an Linux based OpenSource harddisk recorder
30  * software.
31  * The VDR ffnetdev plugin emulates the hardware MPEG decoder and streams the
32  * video over the network instead of hardware video outputs.
33  * The OSD menu of VDR is offered with the RFB protocol to a VNC client.
34  *
35  * In fact this video-filter is a simple VNC client that could be also used to
36  * connect to a real VNC host.
37  * Only 8-bit color is supported at the moment.
38  * Using password protected VNC hosts is supported but not recommended, because
39  * you need to insert the used password in the plugin configuration page of
40  * VLC configuration in plain text and it's saved in plain text.
41  *****************************************************************************/
42
43 //#define VNC_DEBUG
44
45 /*****************************************************************************
46  * Preamble
47  *****************************************************************************/
48
49 #ifdef HAVE_CONFIG_H
50 # include "config.h"
51 #endif
52
53 #include <vlc_common.h>
54 #include <vlc_plugin.h>
55 #include <vlc_vout.h>
56
57 #include <vlc_filter.h>
58 #include "filter_common.h"
59 #include <vlc_image.h>
60 #include <vlc_osd.h>
61 #include <vlc_keys.h>
62
63 #include <vlc_network.h>
64 #include <gcrypt.h>              /* to encrypt password */
65 #include <vlc_gcrypt.h>
66
67 #include "remoteosd_rfbproto.h" /* type definitions of the RFB protocol for VNC */
68
69 /*****************************************************************************
70  * Module descriptor
71  *****************************************************************************/
72 #define READ_BUFFER_SIZE 1000000
73
74 #define RMTOSD_HOST_TEXT N_("VNC Host")
75 #define RMTOSD_HOST_LONGTEXT N_( \
76     "VNC hostname or IP address." )
77
78 #define RMTOSD_PORT_TEXT N_("VNC Port")
79 #define RMTOSD_PORT_LONGTEXT N_( \
80     "VNC portnumber." )
81
82 #define RMTOSD_PASSWORD_TEXT N_("VNC Password")
83 #define RMTOSD_PASSWORD_LONGTEXT N_( \
84     "VNC password." )
85
86 #define RMTOSD_UPDATE_TEXT N_("VNC poll interval" )
87 #define RMTOSD_UPDATE_LONGTEXT N_( \
88     "In this interval an update from VNC is requested, default every 300 ms. ")
89
90 #define RMTOSD_POLL_TEXT N_("VNC polling")
91 #define RMTOSD_POLL_LONGTEXT N_( \
92     "Activate VNC polling. Do NOT activate for use as VDR ffnetdev client." )
93
94 #define RMTOSD_MOUSE_TEXT N_("Mouse events")
95 #define RMTOSD_MOUSE_LONGTEXT N_( \
96     "Send mouse events to VNC host. Not needed for use as VDR ffnetdev client." )
97
98 #define RMTOSD_KEYS_TEXT N_("Key events")
99 #define RMTOSD_KEYS_LONGTEXT N_( \
100     "Send key events to VNC host." )
101
102 #define RMTOSD_ALPHA_TEXT N_("Alpha transparency value (default 255)")
103 #define RMTOSD_ALPHA_LONGTEXT N_( \
104     "The transparency of the OSD VNC can be changed by giving a value " \
105     "between 0 and 255. A lower value specifies more transparency a higher " \
106     "means less transparency. The default is being not transparent " \
107     "(value 255) the minimum is fully transparent (value 0)." )
108
109 #define RMTOSD_CFG "rmtosd-"
110
111 #define RMTOSD_UPDATE_MIN     200
112 #define RMTOSD_UPDATE_DEFAULT 1000
113 #define RMTOSD_UPDATE_MAX     300
114
115 static int  CreateFilter ( vlc_object_t * );
116 static void DestroyFilter( vlc_object_t * );
117
118 vlc_module_begin ()
119     set_description( N_("Remote-OSD over VNC") )
120     set_capability( "sub filter", 100 )
121     set_shortname( N_("Remote-OSD") )
122     set_category( CAT_VIDEO )
123     set_subcategory( SUBCAT_VIDEO_SUBPIC )
124     add_shortcut( "rmtosd" )
125     set_callbacks( CreateFilter, DestroyFilter )
126
127     add_string( RMTOSD_CFG "host", "myvdr", NULL, RMTOSD_HOST_TEXT,
128         RMTOSD_HOST_LONGTEXT, false )
129     add_integer_with_range( RMTOSD_CFG "port", 20001, 1, 0xFFFF, NULL,
130         RMTOSD_PORT_TEXT, RMTOSD_PORT_LONGTEXT, false )
131     add_password( RMTOSD_CFG "password", "", NULL, RMTOSD_PASSWORD_TEXT,
132         RMTOSD_PASSWORD_LONGTEXT, false )
133     add_integer_with_range( RMTOSD_CFG "update", RMTOSD_UPDATE_DEFAULT,
134         RMTOSD_UPDATE_MIN, RMTOSD_UPDATE_MAX, NULL, RMTOSD_UPDATE_TEXT,
135         RMTOSD_UPDATE_LONGTEXT, true )
136     add_bool( RMTOSD_CFG "vnc-polling", false, NULL,
137               RMTOSD_POLL_TEXT , RMTOSD_POLL_LONGTEXT, false )
138     add_bool( RMTOSD_CFG "mouse-events", false, NULL,
139               RMTOSD_MOUSE_TEXT , RMTOSD_MOUSE_LONGTEXT, false )
140     add_bool( RMTOSD_CFG "key-events", false, NULL,
141               RMTOSD_KEYS_TEXT , RMTOSD_KEYS_LONGTEXT, false )
142     add_integer_with_range( RMTOSD_CFG "alpha", 255, 0, 255, NULL,
143         RMTOSD_ALPHA_TEXT, RMTOSD_ALPHA_LONGTEXT, true )
144
145 vlc_module_end ()
146
147
148 /*****************************************************************************
149  * Local prototypes
150  *****************************************************************************/
151 #define CHALLENGESIZE 16
152 #define MAX_VNC_SERVER_NAME_LENGTH 255
153
154 /* subfilter functions */
155 static subpicture_t *Filter( filter_t *, mtime_t );
156
157 static int MouseEvent ( vlc_object_t *p_this, char const *psz_var,
158                         vlc_value_t oldval, vlc_value_t newval, void *p_data );
159
160 static int KeyEvent( vlc_object_t *p_this, char const *psz_var,
161                      vlc_value_t oldval, vlc_value_t newval, void *p_data );
162
163 static void stop_osdvnc ( filter_t *p_filter );
164
165 static void* vnc_worker_thread ( vlc_object_t *p_thread_obj );
166
167 static void* update_request_thread( vlc_object_t *p_thread_obj );
168
169 static bool open_vnc_connection ( filter_t *p_filter );
170
171 static bool handshaking ( filter_t *p_filter );
172
173 static bool process_server_message ( filter_t *p_filter,
174                                      rfbServerToClientMsg *msg );
175
176 static inline void rgb_to_yuv( uint8_t *y, uint8_t *u, uint8_t *v,
177                                int r, int g, int b );
178
179 static inline bool fill_rect( filter_sys_t* p_sys,
180                               uint16_t i_x, uint16_t i_y,
181                               uint16_t i_w, uint16_t i_h,
182                               uint8_t i_color );
183 static inline bool copy_rect( filter_sys_t* p_sys,
184                               uint16_t i_x, uint16_t i_y,
185                               uint16_t i_w, uint16_t i_h,
186                               uint16_t i_sx, uint16_t i_sy );
187
188
189 static inline bool raw_line(  filter_sys_t* p_sys,
190                               uint16_t i_x, uint16_t i_y,
191                               uint16_t i_w );
192
193 static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd );
194
195
196 /*****************************************************************************
197  * Sub filter code
198  *****************************************************************************/
199
200 /*****************************************************************************
201  * Local prototypes
202  *****************************************************************************/
203 struct filter_sys_t
204 {
205     bool          b_need_update;       /* VNC picture is updated, do update the OSD*/
206     mtime_t       i_vnc_poll_interval; /* Update the OSD menu every n ms */
207
208     uint8_t       i_alpha;             /* alpha transparency value */
209
210     char          *psz_host;           /* VNC host */
211     int           i_port;
212
213     char          *psz_passwd;         /* VNC password */
214
215     bool          b_vnc_poll;          /* Activate VNC polling ? */
216     bool          b_vnc_mouse_events;  /* Send MouseEvents ? */
217     bool          b_vnc_key_events;    /* Send KeyEvents ? */
218
219     bool          b_connection_active; /* Handshaking finished ? */
220
221     vlc_mutex_t   lock;                /* To lock for read/write on picture */
222
223     picture_t     *p_pic;              /* The picture with OSD data from VNC */
224
225     vout_thread_t *p_vout;             /* Pointer to video-out thread */
226
227     int           i_socket;            /* Socket used for VNC */
228
229     uint16_t      i_vnc_width;          /* The with of the VNC screen */
230     uint16_t      i_vnc_height;         /* The height of the VNC screen */
231         uint32_t      i_vnc_pixels;         /* The pixels of the VNC screen */
232
233     bool    b_alpha_from_vnc;    /* Special ffnetdev alpha feature enabled ? */
234
235     char          read_buffer[READ_BUFFER_SIZE];
236
237     bool          b_continue;
238
239     vlc_object_t* p_worker_thread;
240
241     uint8_t       ar_color_table_yuv[256][4];
242 };
243
244 /*****************************************************************************
245  * CreateFilter: Create the filter and open the definition file
246  *****************************************************************************/
247 static int CreateFilter ( vlc_object_t *p_this )
248 {
249     filter_t *p_filter = (filter_t *)p_this;
250     filter_sys_t *p_sys = NULL;
251
252     msg_Dbg( p_filter, "Creating vnc osd filter..." );
253
254     p_filter->p_sys = p_sys = calloc( 1, sizeof(*p_sys) );
255     if( !p_filter->p_sys )
256         return VLC_ENOMEM;
257
258     /* Populating struct */
259     vlc_mutex_init( &p_sys->lock );
260     p_sys->b_continue = true;
261     p_sys->i_socket = -1;
262     p_sys->p_pic = NULL;
263
264     p_sys->psz_host = var_CreateGetString( p_this, RMTOSD_CFG "host" );
265     if( EMPTY_STR(p_sys->psz_host) )
266     {
267         msg_Err( p_filter, "unable to get vnc host" );
268         goto error;
269     }
270
271     p_sys->psz_passwd = var_CreateGetString( p_this, RMTOSD_CFG "password" );
272     if( !p_sys->psz_passwd )
273     {
274         msg_Err( p_filter, "unable to get vnc password" );
275         goto error;
276     }
277
278     p_sys->i_port = var_CreateGetIntegerCommand( p_this, RMTOSD_CFG "port" );
279
280     p_sys->i_alpha = var_CreateGetIntegerCommand( p_this, RMTOSD_CFG "alpha" );
281
282     /* in milliseconds, 0 disables polling, should not be lower than 100 */
283     p_sys->i_vnc_poll_interval  = var_CreateGetIntegerCommand( p_this,
284                                                        RMTOSD_CFG "update" );
285     if ( p_sys->i_vnc_poll_interval < 100)
286     {
287        p_sys->i_vnc_poll_interval = 100;
288     }
289
290     for ( int i = 0; i < 256; i++ )
291     {
292         p_sys->ar_color_table_yuv[i][0] = 255;
293         p_sys->ar_color_table_yuv[i][1] = 255;
294         p_sys->ar_color_table_yuv[i][2] = 255;
295         p_sys->ar_color_table_yuv[i][3] = 255;
296     }
297
298     p_sys->b_vnc_poll = var_CreateGetBoolCommand( p_this,
299                                             RMTOSD_CFG "vnc-polling" );
300     p_sys->b_vnc_mouse_events = var_CreateGetBoolCommand( p_this,
301                                             RMTOSD_CFG "mouse-events" );
302     p_sys->b_vnc_key_events = var_CreateGetBoolCommand( p_this,
303                                             RMTOSD_CFG "key-events" );
304
305     /* Keep track of OSD Events */
306     p_sys->b_need_update  = false;
307
308     /* Attach subpicture filter callback */
309     p_filter->pf_sub_filter = Filter;
310
311     p_sys->p_vout = vlc_object_find( p_this, VLC_OBJECT_VOUT, FIND_PARENT );
312
313     if( p_sys->p_vout )
314     {
315         var_AddCallback( p_sys->p_vout, "mouse-moved",
316                          MouseEvent, p_this );
317         var_AddCallback( p_sys->p_vout, "mouse-button-down",
318                          MouseEvent, p_this );
319         var_AddCallback( p_sys->p_vout->p_libvlc, "key-pressed",
320                          KeyEvent, p_this );
321     }
322
323     es_format_Init( &p_filter->fmt_out, SPU_ES, VLC_CODEC_SPU );
324     p_filter->fmt_out.i_priority = 0;
325
326     vlc_gcrypt_init();
327
328     /* create the vnc worker thread */
329     p_sys->p_worker_thread = vlc_object_create( p_this,
330                                                 sizeof( vlc_object_t ) );
331     vlc_object_attach( p_sys->p_worker_thread, p_this );
332     if( vlc_thread_create( p_sys->p_worker_thread, "vnc worker thread",
333                            vnc_worker_thread, VLC_THREAD_PRIORITY_LOW ) )
334     {
335         vlc_object_release( p_sys->p_worker_thread );
336         msg_Err( p_filter, "cannot spawn vnc message reader thread" );
337         goto error;
338     }
339
340     msg_Dbg( p_filter, "osdvnc filter started" );
341
342     return VLC_SUCCESS;
343
344 error:
345     msg_Err( p_filter, "osdvnc filter discarded" );
346
347     stop_osdvnc( p_filter );
348
349     vlc_mutex_destroy( &p_sys->lock );
350     free( p_sys->psz_host );
351     free( p_sys->psz_passwd );
352     free( p_sys );
353
354     return VLC_EGENERIC;
355 }
356
357 /*****************************************************************************
358  * DestroyFilter: Make a clean exit of this plugin
359  *****************************************************************************/
360 static void DestroyFilter( vlc_object_t *p_this )
361 {
362     filter_t     *p_filter = (filter_t*)p_this;
363     filter_sys_t *p_sys = p_filter->p_sys;
364
365     msg_Dbg( p_filter, "DestroyFilter called." );
366
367     stop_osdvnc( p_filter );
368
369     if( p_sys->p_vout )
370     {
371         var_DelCallback( p_sys->p_vout, "mouse-moved",
372                          MouseEvent, p_this );
373         var_DelCallback( p_sys->p_vout, "mouse-button-down",
374                          MouseEvent, p_this );
375         var_DelCallback( p_sys->p_vout->p_libvlc, "key-pressed",
376                          KeyEvent, p_this );
377
378         vlc_object_release( p_sys->p_vout );
379     }
380
381     var_Destroy( p_this, RMTOSD_CFG "host" );
382     var_Destroy( p_this, RMTOSD_CFG "port" );
383     var_Destroy( p_this, RMTOSD_CFG "password" );
384     var_Destroy( p_this, RMTOSD_CFG "update" );
385     var_Destroy( p_this, RMTOSD_CFG "vnc-polling" );
386     var_Destroy( p_this, RMTOSD_CFG "mouse-events" );
387     var_Destroy( p_this, RMTOSD_CFG "key-events" );
388     var_Destroy( p_this, RMTOSD_CFG "alpha" );
389
390     vlc_mutex_destroy( &p_sys->lock );
391     free( p_sys->psz_host );
392     free( p_sys->psz_passwd );
393     free( p_sys );
394 }
395
396 static void stop_osdvnc ( filter_t *p_filter )
397 {
398     filter_sys_t *p_sys = p_filter->p_sys;
399
400     /* It will unlock socket reading */
401     vlc_object_kill( p_filter );
402
403     /* */
404     if( p_sys->p_worker_thread )
405     {
406         msg_Dbg( p_filter, "joining worker_thread" );
407         vlc_object_kill( p_sys->p_worker_thread );
408         vlc_thread_join( p_sys->p_worker_thread );
409         vlc_object_release( p_sys->p_worker_thread );
410         msg_Dbg( p_filter, "released worker_thread" );
411     }
412
413     msg_Dbg( p_filter, "osdvnc stopped" );
414 }
415
416 static bool read_exact( filter_t *p_filter,
417                         int i_socket,
418                         char* p_readbuf,
419                         int i_bytes )
420 {
421     return i_bytes == net_Read( p_filter, i_socket, NULL,
422                                   (unsigned char*)p_readbuf,
423                                   i_bytes, true );
424 }
425
426
427 static bool write_exact( filter_t *p_filter,
428                          int i_socket,
429                          char* p_writebuf,
430                          int i_bytes )
431 {
432     return i_bytes == net_Write( p_filter, i_socket, NULL,
433                                   (unsigned char*)p_writebuf, i_bytes );
434 }
435
436 static bool open_vnc_connection ( filter_t *p_filter )
437 {
438     filter_sys_t *p_sys = p_filter->p_sys;
439
440     msg_Dbg( p_filter, "Open socket to vnc server on %s:%u.",
441               p_sys->psz_host, p_sys->i_port );
442
443     p_sys->i_socket = net_ConnectTCP( p_filter, p_sys->psz_host, p_sys->i_port );
444
445     if( p_sys->i_socket < 0 )
446     {
447         msg_Err( p_filter, "Could not open socket" );
448         return false;
449     }
450
451     msg_Dbg( p_filter, "socket is open." );
452
453     return true;
454 }
455
456 static bool handshaking ( filter_t *p_filter )
457 {
458     filter_sys_t *p_sys = p_filter->p_sys;
459
460     msg_Dbg( p_filter, "Reading protocol version" );
461
462     rfbProtocolVersionMsg pv;
463     if ( !read_exact( p_filter, p_sys->i_socket, pv,
464                       sz_rfbProtocolVersionMsg ) )
465     {
466         msg_Err( p_filter, "Could not read version message" );
467         return false;
468     }
469     pv[sz_rfbProtocolVersionMsg] = '\0'; /* pv size is sz_rfbProtocolVersionMsg+1 */
470
471     msg_Dbg( p_filter, "Server version is %s", pv );
472
473     strncpy(pv, "RFB 003.003\n", sz_rfbProtocolVersionMsg);
474
475     if( !write_exact(p_filter, p_sys->i_socket, pv,
476                      sz_rfbProtocolVersionMsg) )
477     {
478         msg_Err( p_filter, "Could not write version message" );
479         return false;
480     }
481
482     msg_Dbg( p_filter, "Reading authentication scheme" );
483     uint32_t i_authScheme;
484     if( !read_exact( p_filter, p_sys->i_socket, (char*)&i_authScheme, 4 ) )
485     {
486         msg_Err( p_filter, "Could not read authentication scheme" );
487         return false;
488     }
489     i_authScheme = htonl(i_authScheme);
490
491     msg_Dbg( p_filter, "Authentication scheme = %x", i_authScheme );
492     if ( i_authScheme == rfbConnFailed )
493     {
494         msg_Err( p_filter, "Connection rejected by server" );
495         return false;
496     }
497     if (i_authScheme == rfbVncAuth)
498     {
499         unsigned char challenge[CHALLENGESIZE];
500         if ( !read_exact( p_filter, p_sys->i_socket,
501                           (char*)challenge, CHALLENGESIZE ) )
502         {
503             msg_Err( p_filter, "Could not read password challenge" );
504             return false;
505         }
506
507         vnc_encrypt_bytes( challenge, p_sys->psz_passwd );
508
509         if( !write_exact(p_filter, p_sys->i_socket,
510                          (char*)challenge, CHALLENGESIZE ) )
511         {
512             msg_Err( p_filter, "Could not write password" );
513             return false;
514         }
515         uint32_t i_authResult;
516         if( !read_exact( p_filter, p_sys->i_socket, (char*)&i_authResult, 4 ) )
517         {
518             msg_Err( p_filter, "Could not read authentication result" );
519             return false;
520         }
521         i_authResult = htonl(i_authResult);
522         if (i_authResult != rfbVncAuthOK)
523         {
524             msg_Err( p_filter, "VNC authentication failed" );
525             return false;
526         }
527     }
528
529     msg_Dbg( p_filter, "Writing client init message" );
530     rfbClientInitMsg ci;
531     ci.shared = 1;
532     if( !write_exact( p_filter, p_sys->i_socket,
533                       (char*)&ci, sz_rfbClientInitMsg ) )
534     {
535         msg_Err( p_filter, "Could not write client init message" );
536         return false;
537     }
538
539     msg_Dbg( p_filter, "Reading server init message" );
540     rfbServerInitMsg si;
541     if( !read_exact( p_filter, p_sys->i_socket,
542                      (char*)&si, sz_rfbServerInitMsg ) )
543     {
544         msg_Err( p_filter, "Could not read server init message" );
545         return false;
546     }
547     si.framebufferWidth = htons(si.framebufferWidth);
548     si.framebufferHeight = htons(si.framebufferHeight);
549     si.format.redMax = htons(si.format.redMax);
550     si.format.greenMax = htons(si.format.greenMax);
551     si.format.blueMax = htons(si.format.blueMax);
552
553     p_sys->i_vnc_width = si.framebufferWidth;
554     p_sys->i_vnc_height = si.framebufferHeight;
555
556     msg_Dbg( p_filter, "Servers preferred pixelformat: "
557                         "%ux%u, R(%u),G(%u),B(%u), %u bit, depht=%u, %s",
558                         si.framebufferWidth,
559                         si.framebufferHeight,
560                         si.format.redMax,
561                         si.format.greenMax,
562                         si.format.blueMax,
563                         si.format.bitsPerPixel,
564                         si.format.depth,
565                         si.format.trueColour ? "TrueColor" : "Not-TrueColor");
566
567     uint32_t i_nameLength = htonl(si.nameLength);
568     if( i_nameLength > MAX_VNC_SERVER_NAME_LENGTH )
569     {
570         msg_Err( p_filter, "Server name too long" );
571         return false;
572     }
573     char s_ServerName[MAX_VNC_SERVER_NAME_LENGTH+1];
574
575     msg_Dbg( p_filter, "Reading server name with size = %u", i_nameLength );
576     if( !read_exact( p_filter, p_sys->i_socket, s_ServerName, i_nameLength ) )
577     {
578         msg_Err( p_filter, "Could not read server name" );
579         return false;
580     }
581     s_ServerName[i_nameLength] = '\0';
582
583     if( strcmp( s_ServerName, "VDR-OSD") == 0 )
584     {
585         msg_Dbg( p_filter, "Server is a VDR" );
586         p_sys->b_alpha_from_vnc = true;
587     }
588     else
589     {
590         msg_Dbg( p_filter, "Server is a normal VNC" );
591         p_sys->b_alpha_from_vnc = false;
592     }
593
594
595     msg_Dbg( p_filter, "Server init message read properly" );
596     msg_Dbg( p_filter, "Server name is %s", s_ServerName );
597
598     msg_Dbg( p_filter, "Writing SetPixelFormat message" );
599
600     rfbSetPixelFormatMsg sp;
601     sp.type = rfbSetPixelFormat;
602     sp.pad1 = sp.pad2 = 0;
603     sp.format.bitsPerPixel = 8;
604     sp.format.depth = 8 ;
605     sp.format.bigEndian = 1;
606     sp.format.trueColour = 0;
607     sp.format.redMax = htons(31);
608     sp.format.greenMax = htons(31);
609     sp.format.blueMax = htons(31);
610     sp.format.redShift = 10;
611     sp.format.greenShift = 5;
612     sp.format.blueShift = 0;
613     sp.format.pad1 = sp.format.pad2 = 0;
614
615     if( !write_exact( p_filter, p_sys->i_socket,
616                       (char*)&sp, sz_rfbSetPixelFormatMsg) )
617     {
618         msg_Err( p_filter, "Could not write SetPixelFormat message" );
619         return false;
620     }
621
622     msg_Dbg( p_filter, "Writing SetEncodings message" );
623
624     rfbSetEncodingsMsg se;
625     se.type = rfbSetEncodings;
626     se.pad = 0;
627     se.nEncodings = htons( p_sys->b_alpha_from_vnc ? 3 : 2 );
628
629     if( !write_exact( p_filter, p_sys->i_socket,
630                       (char*)&se, sz_rfbSetEncodingsMsg) )
631     {
632         msg_Err( p_filter, "Could not write SetEncodings message begin" );
633         return false;
634     }
635
636     uint32_t i_encoding;
637
638     msg_Dbg( p_filter, "Writing SetEncodings rfbEncodingCopyRect" );
639     i_encoding = htonl(rfbEncodingCopyRect);
640     if( !write_exact( p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
641     {
642         msg_Err( p_filter, "Could not write encoding type rfbEncodingCopyRect." );
643         return false;
644     }
645
646     msg_Dbg( p_filter, "Writing SetEncodings rfbEncodingRRE" );
647     i_encoding = htonl(rfbEncodingRRE);
648     if( !write_exact(p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
649     {
650         msg_Err( p_filter, "Could not write encoding type rfbEncodingRRE." );
651         return false;
652     }
653
654     if( p_sys->b_alpha_from_vnc )
655     {
656         msg_Dbg( p_filter, "Writing SetEncodings rfbEncSpecialUseAlpha" );
657         i_encoding = 0x00F0FFFF; /* rfbEncSpecialUseAlpha is 0xFFFFF000
658                                   * before we swap it */
659         if( !write_exact(p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
660         {
661             msg_Err( p_filter, "Could not write encoding type rfbEncSpecialUseAlpha." );
662             return false;
663         }
664     }
665     return true;
666
667 }
668
669 static void* vnc_worker_thread( vlc_object_t *p_thread_obj )
670 {
671     filter_t* p_filter = (filter_t*)(p_thread_obj->p_parent);
672     filter_sys_t *p_sys = p_filter->p_sys;
673     vlc_object_t *p_update_request_thread;
674     int canc = vlc_savecancel ();
675
676     msg_Dbg( p_filter, "VNC worker thread started" );
677
678     if( !open_vnc_connection ( p_filter ) )
679     {
680         msg_Err( p_filter, "Could not connect to vnc host" );
681         goto exit;
682     }
683
684     if( !handshaking ( p_filter ) )
685     {
686         msg_Err( p_filter, "Error occured while handshaking vnc host" );
687         goto exit;
688     }
689
690     p_sys->b_connection_active = true; /* to enable sending key
691                                             * and mouse events to host */
692
693     /* Create an empty picture for VNC the data */
694     vlc_mutex_lock( &p_sys->lock );
695     p_sys->p_pic = picture_New( VLC_CODEC_YUVA,
696                                 p_sys->i_vnc_width, p_sys->i_vnc_height, 1, 1 );
697     if( !p_sys->p_pic )
698     {
699         vlc_mutex_unlock( &p_sys->lock );
700         goto exit;
701     }
702         p_sys->i_vnc_pixels = p_sys->i_vnc_width * p_sys->i_vnc_height;
703
704     vlc_mutex_unlock( &p_sys->lock );
705
706     /* create the update request thread */
707     p_update_request_thread = vlc_object_create( p_filter,
708                                                  sizeof( vlc_object_t ) );
709     vlc_object_attach( p_update_request_thread, p_filter );
710     if( vlc_thread_create( p_update_request_thread,
711                            "vnc update request thread",
712                            update_request_thread, VLC_THREAD_PRIORITY_LOW ) )
713     {
714         vlc_object_release( p_update_request_thread );
715         msg_Err( p_filter, "cannot spawn vnc update request thread" );
716         goto exit;
717     }
718
719     /* connection is initialized, now read and handle server messages */
720     while( vlc_object_alive( p_thread_obj ) )
721     {
722         rfbServerToClientMsg msg;
723         int i_msgSize;
724
725         memset( &msg, 0, sizeof(msg) );
726
727         if( !read_exact(p_filter, p_sys->i_socket, (char*)&msg, 1 ) )
728         {
729             msg_Err( p_filter, "Error while waiting for next server message");
730             break;
731         }
732         switch (msg.type)
733         {
734         case rfbFramebufferUpdate:
735             i_msgSize = sz_rfbFramebufferUpdateMsg;
736             break;
737         case rfbSetColourMapEntries:
738             i_msgSize = sz_rfbSetColourMapEntriesMsg;
739             break;
740         case rfbBell:
741             i_msgSize = sz_rfbBellMsg;
742             break;
743         case rfbServerCutText:
744             i_msgSize = sz_rfbServerCutTextMsg;
745             break;
746         case rfbReSizeFrameBuffer:
747             i_msgSize = sz_rfbReSizeFrameBufferMsg;
748             break;
749         default:
750             i_msgSize = 0;
751             msg_Err( p_filter, "Invalid message %u received", msg.type );
752             break;
753         }
754
755         if( i_msgSize <= 0 )
756             break;
757
758         if( --i_msgSize > 0 )
759         {
760             if ( !read_exact( p_filter, p_sys->i_socket,
761                               ((char*)&msg)+1, i_msgSize ) )
762             {
763                 msg_Err( p_filter, "Error while reading message of type %u",
764                          msg.type );
765                 break;
766             }
767         }
768         process_server_message( p_filter, &msg);
769     }
770
771     msg_Dbg( p_filter, "joining update_request_thread" );
772     vlc_object_kill( p_update_request_thread );
773     vlc_thread_join( p_update_request_thread );
774     vlc_object_release( p_update_request_thread );
775     msg_Dbg( p_filter, "released update_request_thread" );
776
777 exit:
778
779     vlc_mutex_lock( &p_sys->lock );
780     p_sys->b_connection_active = false;
781
782     if (p_sys->i_socket >= 0)
783         net_Close(p_sys->i_socket);
784
785     if( p_sys->p_pic )
786         picture_Release( p_sys->p_pic );
787
788     /* It will hide the subtitle */
789     p_sys->b_continue = false;
790     p_sys->b_need_update = true;
791     vlc_mutex_unlock( &p_sys->lock );
792
793     msg_Dbg( p_filter, "VNC message reader thread ended" );
794     vlc_restorecancel (canc);
795     return NULL;
796 }
797
798 static void* update_request_thread( vlc_object_t *p_thread_obj )
799 {
800     filter_t* p_filter = (filter_t*)(p_thread_obj->p_parent);
801     filter_sys_t *p_sys = p_filter->p_sys;
802     int canc = vlc_savecancel ();
803
804     msg_Dbg( p_filter, "VNC update request thread started" );
805
806     rfbFramebufferUpdateRequestMsg udr;
807     udr.type = rfbFramebufferUpdateRequest;
808     udr.incremental = 0;
809     udr.x = 0;
810     udr.y = 0;
811     udr.w = htons(p_sys->i_vnc_width);
812     udr.h = htons(p_sys->i_vnc_height);
813
814     if( write_exact(p_filter, p_sys->i_socket, (char*)&udr,
815            sz_rfbFramebufferUpdateRequestMsg) == false)
816     {
817         msg_Err( p_filter, "Could not write rfbFramebufferUpdateRequestMsg." );
818         p_sys->b_continue = false;
819         return NULL;
820     }
821
822     udr.incremental = 1;
823     mtime_t i_poll_interval_microsec = p_sys->i_vnc_poll_interval * 1000;
824
825     if( p_sys->b_vnc_poll)
826     {
827         while( vlc_object_alive( p_thread_obj ) )
828         {
829             msleep( i_poll_interval_microsec );
830             if( write_exact(p_filter, p_sys->i_socket, (char*)&udr,
831                    sz_rfbFramebufferUpdateRequestMsg) == false)
832             {
833                 msg_Err( p_filter, "Could not write rfbFramebufferUpdateRequestMsg." );
834                 break;
835             }
836         }
837         p_sys->b_continue = false;
838     }
839     else
840     {
841         msg_Dbg( p_filter, "VNC polling disabled." );
842     }
843
844     msg_Dbg( p_filter, "VNC update request thread ended" );
845     vlc_restorecancel (canc);
846     return NULL;
847 }
848
849 static bool process_server_message ( filter_t *p_filter,
850                                      rfbServerToClientMsg *msg )
851 {
852     filter_sys_t *p_sys = p_filter->p_sys;
853
854     switch (msg->type)
855     {
856     case rfbFramebufferUpdate:
857         {
858             msg->fu.nRects = htons(msg->fu.nRects);
859             rfbFramebufferUpdateRectHeader hdr;
860
861             for (int i_rect = 0; i_rect < msg->fu.nRects; i_rect++)
862             {
863                 if (!read_exact(p_filter, p_sys->i_socket, (char*)&hdr,
864                     sz_rfbFramebufferUpdateRectHeader ) )
865                 {
866                     msg_Err( p_filter, "Could not read FrameBufferUpdate header" );
867                     return false;
868                 }
869                 hdr.r.x = htons(hdr.r.x);
870                 hdr.r.y = htons(hdr.r.y);
871                 hdr.r.w = htons(hdr.r.w);
872                 hdr.r.h = htons(hdr.r.h);
873                 hdr.encoding = htonl(hdr.encoding);
874
875                 switch (hdr.encoding)
876                 {
877                 case rfbEncodingRaw:
878                     {
879                         int i_line;
880                         for (i_line = 0; i_line < hdr.r.h; i_line++)
881                         {
882                             if ( !read_exact( p_filter, p_sys->i_socket,
883                                     p_sys->read_buffer, hdr.r.w ) )
884                             {
885                                 msg_Err( p_filter,
886                                  "Could not read FrameBufferUpdate line data" );
887                                return false;
888                             }
889                             vlc_mutex_lock( &p_sys->lock );
890                             if ( !raw_line( p_sys, hdr.r.x,
891                                             hdr.r.y + i_line,
892                                             hdr.r.w ) )
893                             {
894                                 msg_Err( p_filter, "raw_line failed." );
895                                 vlc_mutex_unlock( &p_sys->lock );
896                                 return false;
897                             }
898                             vlc_mutex_unlock( &p_sys->lock );
899                         }
900                     }
901                     break;
902
903                 case rfbEncodingCopyRect:
904                     {
905                         rfbCopyRect rect;
906
907                         if ( !read_exact( p_filter, p_sys->i_socket,
908                                           (char*)&rect,
909                                           sz_rfbCopyRect ) )
910                         {
911                             msg_Err( p_filter, "Could not read rfbCopyRect" );
912                             return false;
913                         }
914                         rect.srcX = htons( rect.srcX );
915                         rect.srcY = htons( rect.srcY );
916
917                         vlc_mutex_lock( &p_sys->lock );
918                         if ( !copy_rect( p_sys,
919                                          hdr.r.x,   hdr.r.y,
920                                          hdr.r.w,   hdr.r.h,
921                                          rect.srcX, rect.srcY ) )
922                         {
923                             msg_Err( p_filter, "copy_rect failed." );
924                             vlc_mutex_unlock( &p_sys->lock );
925                             return false;
926                         }
927                         vlc_mutex_unlock( &p_sys->lock );
928                     }
929                     break;
930
931                 case rfbEncodingRRE:
932                     {
933                         rfbRREHeader rrehdr;
934                         if ( !read_exact( p_filter, p_sys->i_socket,
935                                           (char*)&rrehdr,
936                                           sz_rfbRREHeader ) )
937                         {
938                             msg_Err( p_filter, "Could not read rfbRREHeader" );
939                             return false;
940                         }
941                         uint8_t i_pixcolor;
942                         if ( !read_exact(p_filter, p_sys->i_socket,
943                                          (char*)&i_pixcolor, 1 ) )
944                         {
945                             msg_Err( p_filter, "Could not read RRE pixcolor" );
946                             return false;
947                         }
948
949                         vlc_mutex_lock( &p_sys->lock );
950                         if ( !fill_rect( p_sys,
951                                         hdr.r.x, hdr.r.y,
952                                         hdr.r.w, hdr.r.h,
953                                         i_pixcolor) )
954                         {
955                             msg_Err( p_filter, "main fill_rect failed." );
956                             vlc_mutex_unlock( &p_sys->lock );
957                             return false;
958                         }
959                         vlc_mutex_unlock( &p_sys->lock );
960
961                         rrehdr.nSubrects = htonl(rrehdr.nSubrects);
962
963                         int i_datasize = rrehdr.nSubrects *
964                                      ( sizeof(i_pixcolor) + sz_rfbRectangle ) ;
965                         if ( i_datasize > READ_BUFFER_SIZE )
966                         {
967                             msg_Err( p_filter, "Buffer too small, "
968                                      "need %u bytes", i_datasize );
969                             return false;
970                         }
971                         if ( !read_exact( p_filter, p_sys->i_socket,
972                                        p_sys->read_buffer, i_datasize ) )
973                         {
974                             msg_Err( p_filter,
975                                      "Could not read RRE subrect data" );
976                             return false;
977                         }
978
979                         uint32_t i_subrect;
980                         rfbRectangle* p_subrect;
981                         int i_offset = 0;
982                         vlc_mutex_lock( &p_sys->lock );
983                         for ( i_subrect = 0;
984                               i_subrect < rrehdr.nSubrects; i_subrect++)
985                         {
986                             i_pixcolor = p_sys->read_buffer[i_offset];
987                             i_offset += sizeof(i_pixcolor);
988                             p_subrect =
989                                (rfbRectangle*)(p_sys->read_buffer + i_offset);
990                             i_offset += sz_rfbRectangle;
991
992                             if (!fill_rect( p_sys,
993                                             htons(p_subrect->x) + hdr.r.x,
994                                             htons(p_subrect->y) + hdr.r.y,
995                                             htons(p_subrect->w),
996                                             htons(p_subrect->h),
997                                             i_pixcolor) )
998                             {
999                                 msg_Err( p_filter,
1000                                     "subrect %u fill_rect failed.", i_subrect );
1001                                 vlc_mutex_unlock( &p_sys->lock );
1002                                 return false;
1003                             }
1004                         }
1005                         vlc_mutex_unlock( &p_sys->lock );
1006                     }
1007                     break;
1008
1009                 }
1010
1011             }
1012             vlc_mutex_lock( &p_sys->lock );
1013             p_sys->b_need_update = true;
1014             vlc_mutex_unlock( &p_sys->lock );
1015         }
1016         return true;
1017
1018     case rfbSetColourMapEntries:
1019         {
1020             msg->scme.nColours = htons(msg->scme.nColours);
1021             msg->scme.firstColour = htons(msg->scme.firstColour);
1022             int i_datasize;
1023             if ( p_sys->b_alpha_from_vnc == true )
1024             {
1025                 i_datasize = 2 * msg->scme.nColours * 4;
1026             }
1027             else
1028             {
1029                 i_datasize = 2 * msg->scme.nColours * 3;
1030             }
1031             if ( i_datasize > READ_BUFFER_SIZE )
1032             {
1033                 msg_Err( p_filter, "Buffer too small, need %u bytes",
1034                                    i_datasize );
1035                 return false;
1036             }
1037
1038             if ( !read_exact( p_filter, p_sys->i_socket,
1039                               p_sys->read_buffer, i_datasize ) )
1040             {
1041                 msg_Err( p_filter, "Could not read color map data" );
1042                 return false;
1043             }
1044
1045             uint8_t i_red, i_green, i_blue, i_alpha, i_color_index;
1046             uint16_t i_offset = 0;
1047             i_alpha = 255;
1048
1049             for (int i = 0; i < msg->scme.nColours; i++)
1050             {
1051                 i_color_index = i+msg->scme.firstColour;
1052                 if ( p_sys->b_alpha_from_vnc == true )
1053                 {
1054                     i_alpha = p_sys->read_buffer[i_offset];
1055                     i_offset += 2;
1056                 }
1057                 i_red   = p_sys->read_buffer[i_offset];
1058                 i_offset += 2;
1059                 i_green = p_sys->read_buffer[i_offset];
1060                 i_offset += 2;
1061                 i_blue  = p_sys->read_buffer[i_offset];
1062                 i_offset += 2;
1063                 rgb_to_yuv( &p_sys->ar_color_table_yuv[i_color_index][0],
1064                             &p_sys->ar_color_table_yuv[i_color_index][1],
1065                             &p_sys->ar_color_table_yuv[i_color_index][2],
1066                             i_red,
1067                             i_green,
1068                             i_blue );
1069                 p_sys->ar_color_table_yuv[i][3] = i_alpha;
1070             }
1071         }
1072         return true;
1073
1074     case rfbBell:
1075         msg_Err( p_filter, "rfbBell received" );
1076         return true;
1077
1078     case rfbServerCutText:
1079         msg->sct.length = htons(msg->sct.length);
1080         if ( msg->sct.length > READ_BUFFER_SIZE )
1081         {
1082             msg_Err( p_filter, "Buffer too small, need %u bytes", msg->sct.length );
1083             return false;
1084         }
1085         if ( !read_exact(p_filter, p_sys->i_socket,
1086                          p_sys->read_buffer, msg->sct.length ) )
1087         {
1088             msg_Err( p_filter, "Could not read Reading rfbServerCutText data" );
1089             return false;
1090         }
1091         return true;
1092
1093     case rfbReSizeFrameBuffer:
1094         msg_Err( p_filter, "Reading rfbReSizeFrameBuffer not implemented" );
1095         return false;
1096
1097     default:
1098         msg_Err( p_filter, "Invalid message %u received", msg->type );
1099         return false;
1100     }
1101     return false;
1102 }
1103
1104 /****************************************************************************
1105  * Filter: the whole thing
1106  ****************************************************************************
1107  * This function outputs subpictures at regular time intervals.
1108  ****************************************************************************/
1109 static subpicture_t *Filter( filter_t *p_filter, mtime_t date )
1110 {
1111     filter_sys_t *p_sys = p_filter->p_sys;
1112     subpicture_t *p_spu;
1113     subpicture_region_t *p_region;
1114     video_format_t fmt;
1115     picture_t *p_pic;
1116
1117     if( !p_sys->b_need_update )
1118     {
1119         return NULL;
1120     }
1121
1122     vlc_mutex_lock( &p_sys->lock );
1123
1124     p_pic = p_sys->p_pic;
1125
1126     if( !p_pic )
1127     {
1128         vlc_mutex_unlock( &p_sys->lock );
1129         return NULL;
1130     }
1131
1132     /* Allocate the subpicture internal data. */
1133     p_spu = filter_NewSubpicture( p_filter );
1134     if( !p_spu )
1135     {
1136         vlc_mutex_unlock( &p_sys->lock );
1137         return NULL;
1138     }
1139
1140     p_spu->b_absolute = false;
1141     p_spu->i_start = date;
1142     p_spu->i_stop = 0;
1143     p_spu->b_ephemer = true;
1144
1145     if( !p_sys->b_continue )
1146         p_spu->i_stop = p_spu->i_start + 1;
1147
1148     /* Create new SPU region */
1149     memset( &fmt, 0, sizeof(video_format_t) );
1150     fmt.i_chroma = VLC_CODEC_YUVA;
1151     fmt.i_sar_num = fmt.i_sar_den = 1;
1152     fmt.i_width = fmt.i_visible_width = p_pic->p[Y_PLANE].i_visible_pitch;
1153     fmt.i_height = fmt.i_visible_height = p_pic->p[Y_PLANE].i_visible_lines;
1154     fmt.i_x_offset = fmt.i_y_offset = 0;
1155     p_region = subpicture_region_New( &fmt );
1156     if( !p_region )
1157     {
1158         msg_Err( p_filter, "cannot allocate SPU region" );
1159         p_filter->pf_sub_buffer_del( p_filter, p_spu );
1160         vlc_mutex_unlock( &p_sys->lock );
1161         return NULL;
1162     }
1163
1164     /* FIXME the copy is probably not needed anymore */
1165     picture_Copy( p_region->p_picture, p_pic );
1166
1167     p_sys->b_need_update = false;
1168
1169     vlc_mutex_unlock( &p_sys->lock );
1170
1171     /* set to one of the 9 relative locations */
1172     p_region->i_align = 0; /* Center */
1173     p_spu->b_absolute = false;
1174
1175
1176     p_spu->i_original_picture_width = 0; /*Let vout core do the horizontal scaling */
1177     p_spu->i_original_picture_height = fmt.i_height;
1178
1179     p_spu->p_region = p_region;
1180
1181     p_spu->i_alpha = ( p_sys->i_alpha );
1182
1183     return p_spu;
1184 }
1185
1186
1187 static inline void rgb_to_yuv( uint8_t *y, uint8_t *u, uint8_t *v,
1188                                int r, int g, int b )
1189 {
1190     *y = ( ( (  66 * r + 129 * g +  25 * b + 128 ) >> 8 ) + 16 );
1191     *u =   ( ( -38 * r -  74 * g + 112 * b + 128 ) >> 8 ) + 128 ;
1192     *v =   ( ( 112 * r -  94 * g -  18 * b + 128 ) >> 8 ) + 128 ;
1193 }
1194
1195 static inline bool fill_rect( filter_sys_t* p_sys,
1196                               uint16_t i_x, uint16_t i_y,
1197                               uint16_t i_w, uint16_t i_h,
1198                               uint8_t i_color)
1199 {
1200     plane_t *p_outY = p_sys->p_pic->p+Y_PLANE;
1201     plane_t *p_outU = p_sys->p_pic->p+U_PLANE;
1202     plane_t *p_outV = p_sys->p_pic->p+V_PLANE;
1203     plane_t *p_outA = p_sys->p_pic->p+A_PLANE;
1204     int i_pitch = p_outY->i_pitch;
1205     int i_lines = p_outY->i_lines;
1206     if ( i_x + i_w > i_pitch)
1207         return false;
1208     if ( i_y + i_h > i_lines)
1209         return false;
1210     int i_line_offset = i_y * i_pitch;
1211     uint8_t i_yuv_y = p_sys->ar_color_table_yuv[i_color][0];
1212     uint8_t i_yuv_u = p_sys->ar_color_table_yuv[i_color][1];
1213     uint8_t i_yuv_v = p_sys->ar_color_table_yuv[i_color][2];
1214     uint8_t i_alpha = p_sys->ar_color_table_yuv[i_color][3];
1215     for( int i_line = 0; i_line < i_h; i_line++ )
1216     {
1217         for( int i_column = 0; i_column < i_w; i_column++ )
1218         {
1219             int i_total_offset = i_line_offset + i_x + i_column;
1220             p_outY->p_pixels[ i_total_offset ] = i_yuv_y;
1221             p_outU->p_pixels[ i_total_offset ] = i_yuv_u;
1222             p_outV->p_pixels[ i_total_offset ] = i_yuv_v;
1223             p_outA->p_pixels[ i_total_offset ] = i_alpha;
1224         }
1225         i_line_offset += i_pitch;
1226     }
1227     return true;
1228 }
1229
1230 static inline bool copy_rect( filter_sys_t* p_sys,
1231                               uint16_t i_x, uint16_t i_y,
1232                               uint16_t i_w, uint16_t i_h,
1233                               uint16_t i_sx, uint16_t i_sy )
1234 {
1235     plane_t *p_Y = p_sys->p_pic->p+Y_PLANE;
1236     plane_t *p_U = p_sys->p_pic->p+U_PLANE;
1237     plane_t *p_V = p_sys->p_pic->p+V_PLANE;
1238     plane_t *p_A = p_sys->p_pic->p+A_PLANE;
1239
1240     int i_pitch = p_Y->i_pitch;
1241     int i_lines = p_Y->i_lines;
1242
1243     fprintf( stderr, "copy_rect: (%d,%d)+(%d,%d) -> (%d,%d)\n", i_x, i_y, i_w, i_h, i_sx, i_sy );
1244
1245     if( i_x + i_w > i_pitch || i_sx + i_w > i_pitch )
1246         return false;
1247     if( i_y + i_h > i_lines || i_sy + i_h > i_lines)
1248         return false;
1249
1250     if( i_w <= 0 || i_h <= 0 )
1251         return true;
1252
1253     uint8_t *pb_buffer = calloc( i_w * i_h, 4 );
1254     if( !pb_buffer )
1255         return false;
1256
1257     for( int i_line = 0; i_line < i_h; i_line++ )
1258     {
1259         for( int i_column = 0; i_column < i_w; i_column++ )
1260         {
1261             const int i_src_offset = ( i_sy + i_line ) * i_pitch + i_sx + i_column;
1262             const int i_tmp_offset = (    0 + i_line ) *     i_w +    0 + i_column;
1263
1264             pb_buffer[4*i_tmp_offset + 0] = p_Y->p_pixels[i_src_offset];
1265             pb_buffer[4*i_tmp_offset + 1] = p_U->p_pixels[i_src_offset];
1266             pb_buffer[4*i_tmp_offset + 2] = p_V->p_pixels[i_src_offset];
1267             pb_buffer[4*i_tmp_offset + 3] = p_A->p_pixels[i_src_offset];
1268         }
1269     }
1270
1271     for( int i_line = 0; i_line < i_h; i_line++ )
1272     {
1273         for( int i_column = 0; i_column < i_w; i_column++ )
1274         {
1275             const int i_tmp_offset = (   0 + i_line ) *     i_w +   0 + i_column;
1276             const int i_dst_offset = ( i_y + i_line ) * i_pitch + i_x + i_column;
1277
1278             p_Y->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 0];
1279             p_U->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 1];
1280             p_V->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 2];
1281             p_A->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 3];
1282         }
1283     }
1284     free( pb_buffer );
1285     return true;
1286
1287 }
1288
1289 static inline bool raw_line(  filter_sys_t* p_sys,
1290                               uint16_t i_x, uint16_t i_y,
1291                               uint16_t i_w )
1292 {
1293     plane_t *p_outY = p_sys->p_pic->p+Y_PLANE;
1294     plane_t *p_outU = p_sys->p_pic->p+U_PLANE;
1295     plane_t *p_outV = p_sys->p_pic->p+V_PLANE;
1296     plane_t *p_outA = p_sys->p_pic->p+A_PLANE;
1297     int i_pitch = p_outY->i_pitch;
1298     int i_lines = p_outY->i_lines;
1299     if ( i_x + i_w > i_pitch)
1300         return false;
1301     if ( i_y > i_lines)
1302         return false;
1303
1304     int i_line_offset = i_y * i_pitch + i_x;
1305
1306     for( int i_column = 0; i_column < i_w; i_column++ )
1307     {
1308         int i_offset = i_line_offset + i_column;
1309         uint8_t i_color = p_sys->read_buffer[i_column];
1310         p_outY->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][0];
1311         p_outU->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][1];
1312         p_outV->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][2];
1313         p_outA->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][3];
1314     }
1315
1316     return true;
1317 }
1318
1319
1320 /*****************************************************************************
1321  * MouseEvent: callback for mouse events
1322  *****************************************************************************/
1323 static int MouseEvent( vlc_object_t *p_this, char const *psz_var,
1324                        vlc_value_t oldval, vlc_value_t newval, void *p_data )
1325 {
1326     VLC_UNUSED(oldval); VLC_UNUSED(newval); VLC_UNUSED(psz_var);
1327
1328     filter_t *p_filter = (filter_t *)p_data;
1329     filter_sys_t *p_sys = p_filter->p_sys;
1330
1331     if( !p_sys->b_vnc_mouse_events )
1332         return VLC_SUCCESS;
1333
1334     vout_thread_t *p_vout = (vout_thread_t*)p_sys->p_vout;
1335     int i_x, i_y;
1336     int i_v;
1337
1338     i_v = var_GetInteger( p_sys->p_vout, "mouse-button-down" );
1339     var_GetCoords( p_sys->p_vout, "mouse-moved", &i_x, &i_y );
1340
1341     vlc_mutex_lock( &p_sys->lock );
1342
1343     const int v_h = p_vout->fmt_in.i_visible_height;
1344     const int v_w = p_sys->i_vnc_width * v_h / p_sys->i_vnc_height;
1345     const int v_x = (p_vout->fmt_in.i_visible_width-v_w)/2;
1346
1347     i_x -= v_x;
1348
1349     if( i_y < 0 || i_x < 0 || i_y >= v_h || i_x >= v_w )
1350     {
1351         vlc_mutex_unlock( &p_sys->lock );
1352         msg_Dbg( p_this, "invalid mouse event? x=%d y=%d btn=%x", i_x, i_y, i_v );
1353         return VLC_SUCCESS;
1354     }
1355     if( !p_sys->b_connection_active )
1356     {
1357         vlc_mutex_unlock( &p_sys->lock );
1358         return VLC_SUCCESS;
1359     }
1360
1361 #ifdef VNC_DEBUG
1362     msg_Dbg( p_this, "mouse event x=%d y=%d btn=%x", i_x, i_y, i_v );
1363 #endif
1364
1365     /* */
1366     i_x = i_x * p_sys->i_vnc_width / v_w;
1367     i_y = i_y * p_sys->i_vnc_height / v_h;
1368
1369     /* buttonMask bits 0-7 are buttons 1-8, 0=up, 1=down */
1370     rfbPointerEventMsg ev;
1371     ev.type = rfbPointerEvent;
1372     ev.buttonMask = i_v;
1373     ev.x = htons(i_x);
1374     ev.y = htons(i_y);
1375
1376     write_exact( p_filter, p_sys->i_socket,
1377                  (char*)&ev, sz_rfbPointerEventMsg);
1378
1379     vlc_mutex_unlock( &p_sys->lock );
1380
1381     return VLC_SUCCESS;
1382 }
1383
1384 /*****************************************************************************
1385  * KeyEvent: callback for keyboard events
1386  *****************************************************************************/
1387 static int KeyEvent( vlc_object_t *p_this, char const *psz_var,
1388                      vlc_value_t oldval, vlc_value_t newval, void *p_data )
1389 {
1390     VLC_UNUSED(psz_var); VLC_UNUSED(oldval);
1391
1392     filter_t *p_filter = (filter_t *)p_data;
1393     filter_sys_t *p_sys = p_filter->p_sys;
1394
1395     if( !p_sys->b_vnc_key_events )
1396         return VLC_SUCCESS;
1397
1398     msg_Dbg( p_this, "key pressed (%d) ", newval.i_int );
1399
1400     if ( !newval.i_int )
1401     {
1402         msg_Err( p_this, "Received invalid key event %d", newval.i_int );
1403         return VLC_EGENERIC;
1404     }
1405
1406     vlc_mutex_lock( &p_sys->lock );
1407     if( !p_sys->b_connection_active )
1408     {
1409         vlc_mutex_unlock( &p_sys->lock );
1410         return VLC_SUCCESS;
1411     }
1412
1413     uint32_t i_key32 = newval.i_int;
1414     i_key32 = htonl(i_key32);
1415     rfbKeyEventMsg ev;
1416
1417     ev.type = rfbKeyEvent;
1418     ev.down = 1;
1419     ev.pad = 0;
1420
1421     /* first key-down for modifier-keys */
1422     if (newval.i_int & KEY_MODIFIER_CTRL)
1423     {
1424         ev.key = 0xffe3;
1425         write_exact( p_filter, p_sys->i_socket,
1426                      (char*)&ev, sz_rfbKeyEventMsg);
1427     }
1428     if (newval.i_int & KEY_MODIFIER_SHIFT)
1429     {
1430         ev.key = 0xffe1;
1431         write_exact( p_filter, p_sys->i_socket,
1432                      (char*)&ev, sz_rfbKeyEventMsg);
1433     }
1434     if (newval.i_int & KEY_MODIFIER_ALT)
1435     {
1436         ev.key = 0xffe9;
1437         write_exact( p_filter, p_sys->i_socket,
1438                      (char*)&ev, sz_rfbKeyEventMsg);
1439     }
1440
1441     /* then key-down for the pressed key */
1442     ev.key = i_key32;
1443     write_exact( p_filter, p_sys->i_socket,
1444                  (char*)&ev, sz_rfbKeyEventMsg);
1445
1446     ev.down = 0;
1447
1448     /* then key-up for the pressed key */
1449     write_exact( p_filter, p_sys->i_socket,
1450                  (char*)&ev, sz_rfbKeyEventMsg);
1451
1452     /* last key-down for modifier-keys */
1453     if (newval.i_int & KEY_MODIFIER_CTRL)
1454     {
1455         ev.key = 0xffe3;
1456         write_exact( p_filter, p_sys->i_socket,
1457                      (char*)&ev, sz_rfbKeyEventMsg);
1458     }
1459     if (newval.i_int & KEY_MODIFIER_SHIFT)
1460     {
1461         ev.key = 0xffe1;
1462         write_exact( p_filter, p_sys->i_socket,
1463                      (char*)&ev, sz_rfbKeyEventMsg);
1464     }
1465     if (newval.i_int & KEY_MODIFIER_ALT)
1466     {
1467        ev.key = 0xffe9;
1468        write_exact( p_filter, p_sys->i_socket,
1469                     (char*)&ev, sz_rfbKeyEventMsg);
1470     }
1471     vlc_mutex_unlock( &p_sys->lock );
1472
1473     return VLC_SUCCESS;
1474 }
1475
1476 static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd )
1477 {
1478     unsigned char key[8];
1479     unsigned int i;
1480
1481     for (i = 0; i < 8; i++)
1482         key[i] = i < strlen( passwd ) ? passwd[i] : '\0';
1483
1484     gcry_cipher_hd_t ctx;
1485     gcry_cipher_open( &ctx, GCRY_CIPHER_DES, GCRY_CIPHER_MODE_ECB,0);
1486
1487     /* reverse bits of the key */
1488     for( i = 0 ; i < 8 ; i ++ )
1489         key[i] =
1490             (key[i] >> 7) +
1491             (((key[i] >> 6) & 0x01 ) << 1 ) +
1492             (((key[i] >> 5) & 0x01 ) << 2 ) +
1493             (((key[i] >> 4) & 0x01 ) << 3 ) +
1494             (((key[i] >> 3) & 0x01 ) << 4 ) +
1495             (((key[i] >> 2) & 0x01 ) << 5 ) +
1496             (((key[i] >> 1) & 0x01 ) << 6 ) +
1497             ((key[i] & 0x01) << 7 );
1498
1499     gcry_cipher_setkey( ctx, key, 8 );
1500     gcry_cipher_encrypt( ctx, bytes, CHALLENGESIZE, bytes, CHALLENGESIZE );
1501     gcry_cipher_close( ctx );
1502 }
1503