git.sesse.net Git - vlc/blob - modules/video_filter/remoteosd.c

   1 /*****************************************************************************
   2  * remoteosd.c: remote osd over vnc filter module
   3  *****************************************************************************
   4  * Copyright (C) 2007-2008 Matthias Bauer
   5  * $Id$
   6  *
   7  * Authors: Matthias Bauer <matthias dot bauer #_at_# gmx dot ch>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implid warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  22  *****************************************************************************/
  23
  24 /*****************************************************************************
  25  * RemoteOSD uses the RFB-Protocol of VNC to display an On-Screen-Display
  26  * menu generated by a streaming server as overlay for the streamed video.
  27  *
  28  * The streaming server that implements this is the ffnetdev plugin for VDR.
  29  * VDR (VideoDiskRecorder) is an Linux based OpenSource harddisk recorder
  30  * software.
  31  * The VDR ffnetdev plugin emulates the hardware MPEG decoder and streams the
  32  * video over the network instead of hardware video outputs.
  33  * The OSD menu of VDR is offered with the RFB protocol to a VNC client.
  34  *
  35  * In fact this video-filter is a simple VNC client that could be also used to
  36  * connect to a real VNC host.
  37  * Only 8-bit color is supported at the moment.
  38  * Using password protected VNC hosts is supported but not recommended, because
  39  * you need to insert the used password in the plugin configuration page of
  40  * VLC configuration in plain text and it's saved in plain text.
  41  *****************************************************************************/
  42
  43 //#define VNC_DEBUG
  44
  45 /*****************************************************************************
  46  * Preamble
  47  *****************************************************************************/
  48
  49 #ifdef HAVE_CONFIG_H
  50 # include "config.h"
  51 #endif
  52
  53 #include <vlc_common.h>
  54 #include <vlc_plugin.h>
  55 #include <vlc_vout.h>
  56
  57 #include "vlc_filter.h"
  58 #include "filter_common.h"
  59 #include "vlc_image.h"
  60 #include "vlc_osd.h"
  61 #include "vlc_keys.h"
  62
  63 #include <vlc_network.h>
  64 #include <gcrypt.h>              /* to encrypt password */
  65 #include <vlc_gcrypt.h>
  66
  67 #define CHALLENGESIZE 16
  68 #define MAX_VNC_SERVER_NAME_LENGTH 255
  69
  70 #include "remoteosd_rfbproto.h" /* type definitions of the RFB protocol for VNC */
  71
  72 /*****************************************************************************
  73  * Local prototypes
  74  *****************************************************************************/
  75 /* subfilter functions */
  76 static int  CreateFilter ( vlc_object_t * );
  77 static void DestroyFilter( vlc_object_t * );
  78 static subpicture_t *Filter( filter_t *, mtime_t );
  79
  80 static int MouseEvent ( vlc_object_t *p_this, char const *psz_var,
  81                         vlc_value_t oldval, vlc_value_t newval, void *p_data );
  82
  83 static int KeyEvent( vlc_object_t *p_this, char const *psz_var,
  84                      vlc_value_t oldval, vlc_value_t newval, void *p_data );
  85
  86 static void stop_osdvnc ( filter_t *p_filter );
  87
  88 static void vnc_worker_thread ( vlc_object_t *p_thread_obj );
  89
  90 static void update_request_thread( vlc_object_t *p_thread_obj );
  91
  92 static bool open_vnc_connection ( filter_t *p_filter );
  93
  94 static bool handshaking ( filter_t *p_filter );
  95
  96 static bool process_server_message ( filter_t *p_filter,
  97                                      rfbServerToClientMsg *msg );
  98
  99 static bool read_exact( filter_t *p_filter,
 100                         int i_socket,
 101                         char* p_readbuf,
 102                         int i_bytes );
 103
 104 static bool write_exact( filter_t *p_filter,
 105                          int i_socket,
 106                          char* p_writebuf,
 107                          int i_bytes );
 108
 109 static inline void rgb_to_yuv( uint8_t *y, uint8_t *u, uint8_t *v,
 110                                int r, int g, int b );
 111
 112 static inline bool fill_rect( filter_sys_t* p_sys,
 113                               uint16_t i_x, uint16_t i_y,
 114                               uint16_t i_w, uint16_t i_h,
 115                               uint8_t i_color );
 116
 117 static inline bool raw_line(  filter_sys_t* p_sys,
 118                               uint16_t i_x, uint16_t i_y,
 119                               uint16_t i_w );
 120
 121 static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd );
 122
 123
 124 /*****************************************************************************
 125  * Module descriptor
 126  *****************************************************************************/
 127 #define READ_BUFFER_SIZE 1000000
 128
 129 #define RMTOSD_HOST_TEXT N_("VNC Host")
 130 #define RMTOSD_HOST_LONGTEXT N_( \
 131     "VNC hostname or IP address." )
 132
 133 #define RMTOSD_PORT_TEXT N_("VNC Port")
 134 #define RMTOSD_PORT_LONGTEXT N_( \
 135     "VNC portnumber." )
 136
 137 #define RMTOSD_PASSWORD_TEXT N_("VNC Password")
 138 #define RMTOSD_PASSWORD_LONGTEXT N_( \
 139     "VNC password." )
 140
 141 #define RMTOSD_UPDATE_TEXT N_("VNC poll interval" )
 142 #define RMTOSD_UPDATE_LONGTEXT N_( \
 143     "In this interval an update from VNC is requested, default every 300 ms. ")
 144
 145 #define RMTOSD_POLL_TEXT N_("VNC polling")
 146 #define RMTOSD_POLL_LONGTEXT N_( \
 147     "Activate VNC polling. Do NOT activate for use as VDR ffnetdev client." )
 148
 149 #define RMTOSD_MOUSE_TEXT N_("Mouse events")
 150 #define RMTOSD_MOUSE_LONGTEXT N_( \
 151     "Send mouse events to VNC host. Not needed for use as VDR ffnetdev client." )
 152
 153 #define RMTOSD_KEYS_TEXT N_("Key events")
 154 #define RMTOSD_KEYS_LONGTEXT N_( \
 155     "Send key events to VNC host." )
 156
 157 #define RMTOSD_ALPHA_TEXT N_("Alpha transparency value (default 255)")
 158 #define RMTOSD_ALPHA_LONGTEXT N_( \
 159     "The transparency of the OSD VNC can be changed by giving a value " \
 160     "between 0 and 255. A lower value specifies more transparency a higher " \
 161     "means less transparency. The default is being not transparent " \
 162     "(value 255) the minimum is fully transparent (value 0)." )
 163
 164 #define RMTOSD_CFG "rmtosd-"
 165
 166 #define RMTOSD_UPDATE_MIN     200
 167 #define RMTOSD_UPDATE_DEFAULT 1000
 168 #define RMTOSD_UPDATE_MAX     300
 169
 170
 171 vlc_module_begin();
 172     set_description( N_("Remote-OSD over VNC") );
 173     set_capability( "sub filter", 100 );
 174     set_shortname( N_("Remote-OSD") );
 175     set_category( CAT_VIDEO );
 176     set_subcategory( SUBCAT_VIDEO_SUBPIC );
 177     add_shortcut( "rmtosd" );
 178     set_callbacks( CreateFilter, DestroyFilter );
 179
 180     add_string( RMTOSD_CFG "host", "myvdr", NULL, RMTOSD_HOST_TEXT,
 181         RMTOSD_HOST_LONGTEXT, false );
 182     add_integer_with_range( RMTOSD_CFG "port", 20001, 1, 0xFFFF, NULL,
 183         RMTOSD_PORT_TEXT, RMTOSD_PORT_LONGTEXT, false );
 184     add_password( RMTOSD_CFG "password", "", NULL, RMTOSD_PASSWORD_TEXT,
 185         RMTOSD_PASSWORD_LONGTEXT, false );
 186     add_integer_with_range( RMTOSD_CFG "update", RMTOSD_UPDATE_DEFAULT,
 187         RMTOSD_UPDATE_MIN, RMTOSD_UPDATE_MAX, NULL, RMTOSD_UPDATE_TEXT,
 188         RMTOSD_UPDATE_LONGTEXT, true );
 189     add_bool( RMTOSD_CFG "vnc-polling", 0, NULL,
 190               RMTOSD_POLL_TEXT , RMTOSD_POLL_LONGTEXT, false );
 191     add_bool( RMTOSD_CFG "mouse-events", 0, NULL,
 192               RMTOSD_MOUSE_TEXT , RMTOSD_MOUSE_LONGTEXT, false );
 193     add_bool( RMTOSD_CFG "key-events", 0, NULL,
 194               RMTOSD_KEYS_TEXT , RMTOSD_KEYS_LONGTEXT, false );
 195     add_integer_with_range( RMTOSD_CFG "alpha", 255, 0, 255, NULL,
 196         RMTOSD_ALPHA_TEXT, RMTOSD_ALPHA_LONGTEXT, true );
 197
 198 vlc_module_end();
 199
 200 /*****************************************************************************
 201  * Sub filter code
 202  *****************************************************************************/
 203
 204 /*****************************************************************************
 205  * Local prototypes
 206  *****************************************************************************/
 207 struct filter_sys_t
 208 {
 209     VLC_COMMON_MEMBERS
 210
 211     bool          b_need_update;       /* VNC picture is updated, do update the OSD*/
 212     mtime_t       i_vnc_poll_interval; /* Update the OSD menu every n ms */
 213
 214     uint8_t       i_alpha;             /* alpha transparency value */
 215
 216     char          *psz_host;           /* VNC host */
 217     int           i_port;
 218
 219     char          *psz_passwd;         /* VNC password */
 220
 221     bool          b_vnc_poll;          /* Activate VNC polling ? */
 222     bool          b_vnc_mouse_events;  /* Send MouseEvents ? */
 223     bool          b_vnc_key_events;    /* Send KeyEvents ? */
 224
 225     bool          b_connection_active; /* Handshaking finished ? */
 226
 227     vlc_mutex_t   lock;                /* To lock for read/write on picture */
 228
 229     picture_t     *p_pic;              /* The picture with OSD data from VNC */
 230
 231     vout_thread_t *p_vout;             /* Pointer to video-out thread */
 232
 233     int           i_socket;            /* Socket used for VNC */
 234
 235     uint16_t      i_vnc_width;          /* The with of the VNC screen */
 236     uint16_t      i_vnc_height;         /* The height of the VNC screen */
 237         uint32_t      i_vnc_pixels;         /* The pixels of the VNC screen */
 238
 239     bool    b_alpha_from_vnc;    /* Special ffnetdev alpha feature enabled ? */
 240
 241     char          read_buffer[READ_BUFFER_SIZE];
 242
 243     bool          b_continue;
 244
 245     vlc_object_t* p_worker_thread;
 246     vlc_object_t* p_update_request_thread;
 247
 248     uint8_t       ar_color_table_yuv[256][4];
 249 };
 250
 251 /*****************************************************************************
 252  * CreateFilter: Create the filter and open the definition file
 253  *****************************************************************************/
 254 static int CreateFilter ( vlc_object_t *p_this )
 255 {
 256     filter_t *p_filter = (filter_t *)p_this;
 257     filter_sys_t *p_sys = NULL;
 258
 259     msg_Dbg( p_filter, "Creating vnc osd filter..." );
 260
 261     p_filter->p_sys = p_sys = (filter_sys_t *) malloc( sizeof(filter_sys_t) );
 262     if( !p_filter->p_sys )
 263     {
 264         msg_Err( p_filter, "out of memory" );
 265         return VLC_ENOMEM;
 266     }
 267     memset( p_sys, 0, sizeof(filter_sys_t) );
 268
 269     /* Populating struct */
 270     vlc_mutex_init( &p_sys->lock );
 271     p_sys->b_continue = true;
 272     p_sys->i_socket = -1;
 273
 274     p_sys->psz_host = var_CreateGetString( p_this, RMTOSD_CFG "host" );
 275     if( EMPTY_STR(p_sys->psz_host) )
 276     {
 277         msg_Err( p_filter, "unable to get vnc host" );
 278         goto error;
 279     }
 280
 281     p_sys->psz_passwd = var_CreateGetString( p_this, RMTOSD_CFG "password" );
 282     if( !p_sys->psz_passwd )
 283     {
 284         msg_Err( p_filter, "unable to get vnc password" );
 285         goto error;
 286     }
 287
 288     p_sys->i_port = var_CreateGetIntegerCommand( p_this, RMTOSD_CFG "port" );
 289
 290     p_sys->i_alpha = var_CreateGetIntegerCommand( p_this, RMTOSD_CFG "alpha" );
 291
 292     /* in miliseconds, 0 disables polling, should not be lower than 100 */
 293     p_sys->i_vnc_poll_interval  = var_CreateGetIntegerCommand( p_this,
 294                                                        RMTOSD_CFG "update" );
 295     if ( p_sys->i_vnc_poll_interval < 100)
 296     {
 297        p_sys->i_vnc_poll_interval = 100;
 298     }
 299
 300     for ( int i = 0; i < 256; i++ )
 301     {
 302         p_sys->ar_color_table_yuv[i][0] = 255;
 303         p_sys->ar_color_table_yuv[i][1] = 255;
 304         p_sys->ar_color_table_yuv[i][2] = 255;
 305         p_sys->ar_color_table_yuv[i][3] = 255;
 306     }
 307
 308     p_sys->b_vnc_poll = var_CreateGetBoolCommand( p_this,
 309                                             RMTOSD_CFG "vnc-polling" );
 310     p_sys->b_vnc_mouse_events = var_CreateGetBoolCommand( p_this,
 311                                             RMTOSD_CFG "mouse-events" );
 312     p_sys->b_vnc_key_events = var_CreateGetBoolCommand( p_this,
 313                                             RMTOSD_CFG "key-events" );
 314
 315     /* Keep track of OSD Events */
 316     p_sys->b_need_update  = false;
 317
 318     /* Attach subpicture filter callback */
 319     p_filter->pf_sub_filter = Filter;
 320
 321     p_sys->p_vout = vlc_object_find( p_this, VLC_OBJECT_VOUT, FIND_PARENT );
 322
 323     if( p_sys->p_vout )
 324     {
 325         var_AddCallback( p_sys->p_vout, "mouse-moved",
 326                          MouseEvent, p_this );
 327         var_AddCallback( p_sys->p_vout, "mouse-button-down",
 328                          MouseEvent, p_this );
 329         var_AddCallback( p_sys->p_vout->p_libvlc, "key-pressed",
 330                          KeyEvent, p_this );
 331     }
 332
 333     es_format_Init( &p_filter->fmt_out, SPU_ES, VLC_FOURCC( 's','p','u',' ' ) );
 334     p_filter->fmt_out.i_priority = 0;
 335
 336     vlc_gcrypt_init();
 337
 338     /* create the vnc worker thread */
 339     p_sys->p_worker_thread = vlc_object_create( p_this, VLC_OBJECT_GENERIC );
 340     vlc_object_attach( p_sys->p_worker_thread, p_this );
 341     if( vlc_thread_create( p_sys->p_worker_thread, "vnc worker thread",
 342                            vnc_worker_thread,
 343                            VLC_THREAD_PRIORITY_LOW, false ) )
 344     {
 345         vlc_object_detach( p_sys->p_worker_thread );
 346         vlc_object_release( p_sys->p_worker_thread );
 347         p_sys->p_worker_thread = NULL;
 348         msg_Err( p_filter, "cannot spawn vnc message reader thread" );
 349         goto error;
 350     }
 351
 352     msg_Dbg( p_filter, "osdvnc filter started" );
 353
 354     return VLC_SUCCESS;
 355
 356 error:
 357     msg_Err( p_filter, "osdvnc filter discarded" );
 358
 359     p_sys->b_continue = false;
 360
 361     stop_osdvnc( p_filter );
 362
 363     free( p_sys->psz_host );
 364     free( p_sys->psz_passwd );
 365     free( p_sys );
 366
 367     return VLC_EGENERIC;
 368 }
 369
 370 /*****************************************************************************
 371  * DestroyFilter: Make a clean exit of this plugin
 372  *****************************************************************************/
 373 static void DestroyFilter( vlc_object_t *p_this )
 374 {
 375     filter_t     *p_filter = (filter_t*)p_this;
 376     filter_sys_t *p_sys = p_filter->p_sys;
 377
 378     msg_Dbg( p_filter, "DestroyFilter called." );
 379
 380     stop_osdvnc( p_filter );
 381
 382     if( p_sys->p_vout )
 383     {
 384         var_DelCallback( p_sys->p_vout, "mouse-moved",
 385                          MouseEvent, p_this );
 386         var_DelCallback( p_sys->p_vout, "mouse-button-down",
 387                          MouseEvent, p_this );
 388         var_DelCallback( p_sys->p_vout->p_libvlc, "key-pressed",
 389                          KeyEvent, p_this );
 390
 391         vlc_object_release( p_sys->p_vout );
 392         p_sys->p_vout = NULL;
 393     }
 394
 395     var_Destroy( p_this, RMTOSD_CFG "host" );
 396     var_Destroy( p_this, RMTOSD_CFG "port" );
 397     var_Destroy( p_this, RMTOSD_CFG "password" );
 398     var_Destroy( p_this, RMTOSD_CFG "update" );
 399     var_Destroy( p_this, RMTOSD_CFG "vnc-polling" );
 400     var_Destroy( p_this, RMTOSD_CFG "mouse-events" );
 401     var_Destroy( p_this, RMTOSD_CFG "key-events" );
 402     var_Destroy( p_this, RMTOSD_CFG "alpha" );
 403
 404     free( p_sys->psz_host );
 405     free( p_sys->psz_passwd );
 406     free( p_sys );
 407 }
 408
 409 static void stop_osdvnc ( filter_t *p_filter )
 410 {
 411     filter_sys_t *p_sys = p_filter->p_sys;
 412
 413     if (p_sys->i_socket >= 0)
 414     {
 415         net_Close(p_sys->i_socket);
 416     }
 417     p_sys->b_continue = false; /* this causes the threads to stop */
 418
 419     if ( p_sys->p_worker_thread )
 420     {
 421         msg_Dbg( p_filter, "joining worker_thread" );
 422         vlc_thread_join( p_sys->p_worker_thread );
 423         vlc_object_detach( p_sys->p_worker_thread );
 424         vlc_object_release( p_sys->p_worker_thread );
 425         msg_Dbg( p_filter, "released worker_thread" );
 426     }
 427
 428     if ( p_sys->p_update_request_thread )
 429     {
 430         msg_Dbg( p_filter, "joining update_request_thread" );
 431         vlc_thread_join( p_sys->p_update_request_thread );
 432         vlc_object_detach( p_sys->p_update_request_thread );
 433         vlc_object_release( p_sys->p_update_request_thread );
 434         msg_Dbg( p_filter, "released update_request_thread" );
 435     }
 436
 437     msg_Dbg( p_filter, "osdvnc stopped" );
 438 }
 439
 440 static bool read_exact( filter_t *p_filter,
 441                         int i_socket,
 442                         char* p_readbuf,
 443                         int i_bytes )
 444 {
 445     return i_bytes == net_Read( p_filter, i_socket, NULL,
 446                                   (unsigned char*)p_readbuf,
 447                                   i_bytes, true );
 448 }
 449
 450
 451 static bool write_exact( filter_t *p_filter,
 452                          int i_socket,
 453                          char* p_writebuf,
 454                          int i_bytes )
 455 {
 456     return i_bytes == net_Write( p_filter, i_socket, NULL,
 457                                   (unsigned char*)p_writebuf, i_bytes );
 458 }
 459
 460 static bool open_vnc_connection ( filter_t *p_filter )
 461 {
 462     filter_sys_t *p_sys = p_filter->p_sys;
 463
 464     msg_Dbg( p_filter, "Open socket to vnc server on %s:%u.",
 465               p_sys->psz_host, p_sys->i_port );
 466
 467     p_sys->i_socket = net_ConnectTCP( p_filter, p_sys->psz_host, p_sys->i_port );
 468
 469     if( p_sys->i_socket < 0 )
 470     {
 471         msg_Err( p_filter, "Could not open socket" );
 472         return false;
 473     }
 474
 475     msg_Dbg( p_filter, "socket is open." );
 476
 477     return true;
 478 }
 479
 480 static bool handshaking ( filter_t *p_filter )
 481 {
 482     filter_sys_t *p_sys = p_filter->p_sys;
 483
 484     msg_Dbg( p_filter, "Reading protocol version" );
 485
 486     rfbProtocolVersionMsg pv;
 487     if ( !read_exact( p_filter, p_sys->i_socket, pv,
 488                       sz_rfbProtocolVersionMsg ) )
 489     {
 490         msg_Err( p_filter, "Could not read version message" );
 491         return false;
 492     }
 493     pv[sz_rfbProtocolVersionMsg] = '\0'; /* pv size is sz_rfbProtocolVersionMsg+1 */
 494
 495     msg_Dbg( p_filter, "Server version is %s", pv );
 496
 497     strncpy(pv, "RFB 003.003\n", sz_rfbProtocolVersionMsg);
 498
 499     if( !write_exact(p_filter, p_sys->i_socket, pv,
 500                      sz_rfbProtocolVersionMsg) )
 501     {
 502         msg_Err( p_filter, "Could not write version message" );
 503         return false;
 504     }
 505
 506     msg_Dbg( p_filter, "Reading authentication scheme" );
 507     uint32_t i_authScheme;
 508     if( !read_exact( p_filter, p_sys->i_socket, (char*)&i_authScheme, 4 ) )
 509     {
 510         msg_Err( p_filter, "Could not read authentication scheme" );
 511         return false;
 512     }
 513     i_authScheme = htonl(i_authScheme);
 514
 515     msg_Dbg( p_filter, "Authentication scheme = %x", i_authScheme );
 516     if ( i_authScheme == rfbConnFailed )
 517     {
 518         msg_Err( p_filter, "Connection rejected by server" );
 519         return false;
 520     }
 521     if (i_authScheme == rfbVncAuth)
 522     {
 523         unsigned char challenge[CHALLENGESIZE];
 524         if ( !read_exact( p_filter, p_sys->i_socket,
 525                           (char*)challenge, CHALLENGESIZE ) )
 526         {
 527             msg_Err( p_filter, "Could not read password challenge" );
 528             return false;
 529         }
 530
 531         vnc_encrypt_bytes( challenge, p_sys->psz_passwd );
 532
 533         if( !write_exact(p_filter, p_sys->i_socket,
 534                          (char*)challenge, CHALLENGESIZE ) )
 535         {
 536             msg_Err( p_filter, "Could not write password" );
 537             return false;
 538         }
 539         uint32_t i_authResult;
 540         if( !read_exact( p_filter, p_sys->i_socket, (char*)&i_authResult, 4 ) )
 541         {
 542             msg_Err( p_filter, "Could not read authentication result" );
 543             return false;
 544         }
 545         i_authResult = htonl(i_authResult);
 546         if (i_authResult != rfbVncAuthOK)
 547         {
 548             msg_Err( p_filter, "VNC authentication failed" );
 549             return false;
 550         }
 551     }
 552
 553     msg_Dbg( p_filter, "Writing client init message" );
 554     rfbClientInitMsg ci;
 555     ci.shared = 1;
 556     if( !write_exact( p_filter, p_sys->i_socket,
 557                       (char*)&ci, sz_rfbClientInitMsg ) )
 558     {
 559         msg_Err( p_filter, "Could not write client init message" );
 560         return false;
 561     }
 562
 563     msg_Dbg( p_filter, "Reading server init message" );
 564     rfbServerInitMsg si;
 565     if( !read_exact( p_filter, p_sys->i_socket,
 566                      (char*)&si, sz_rfbServerInitMsg ) )
 567     {
 568         msg_Err( p_filter, "Could not read server init message" );
 569         return false;
 570     }
 571     si.framebufferWidth = htons(si.framebufferWidth);
 572     si.framebufferHeight = htons(si.framebufferHeight);
 573     si.format.redMax = htons(si.format.redMax);
 574     si.format.greenMax = htons(si.format.greenMax);
 575     si.format.blueMax = htons(si.format.blueMax);
 576
 577     p_sys->i_vnc_width = si.framebufferWidth;
 578     p_sys->i_vnc_height = si.framebufferHeight;
 579
 580     msg_Dbg( p_filter, "Servers preferred pixelformat: "
 581                         "%ux%u, R(%u),G(%u),B(%u), %u bit, depht=%u, %s",
 582                         si.framebufferWidth,
 583                         si.framebufferHeight,
 584                         si.format.redMax,
 585                         si.format.greenMax,
 586                         si.format.blueMax,
 587                         si.format.bitsPerPixel,
 588                         si.format.depth,
 589                         si.format.trueColour ? "TrueColor" : "Not-TrueColor");
 590
 591     uint32_t i_nameLength = htonl(si.nameLength);
 592     if( i_nameLength > MAX_VNC_SERVER_NAME_LENGTH )
 593     {
 594         msg_Err( p_filter, "Server name too long" );
 595         return false;
 596     }
 597     char s_ServerName[MAX_VNC_SERVER_NAME_LENGTH+1];
 598
 599     msg_Dbg( p_filter, "Reading server name with size = %u", i_nameLength );
 600     if( !read_exact( p_filter, p_sys->i_socket, s_ServerName, i_nameLength ) )
 601     {
 602         msg_Err( p_filter, "Could not read server name" );
 603         return false;
 604     }
 605     s_ServerName[i_nameLength] = '\0';
 606
 607     if( strcmp( s_ServerName, "VDR-OSD") == 0 )
 608     {
 609         msg_Dbg( p_filter, "Server is a VDR" );
 610         p_sys->b_alpha_from_vnc = true;
 611     }
 612     else
 613     {
 614         msg_Dbg( p_filter, "Server is a normal VNC" );
 615         p_sys->b_alpha_from_vnc = false;
 616     }
 617
 618
 619     msg_Dbg( p_filter, "Server init message read properly" );
 620     msg_Dbg( p_filter, "Server name is %s", s_ServerName );
 621
 622     msg_Dbg( p_filter, "Writing SetPixelFormat message" );
 623
 624     rfbSetPixelFormatMsg sp;
 625     sp.type = rfbSetPixelFormat;
 626     sp.format.bitsPerPixel = 8;
 627     sp.format.depth = 8 ;
 628     sp.format.bigEndian = 1;
 629     sp.format.trueColour = 0;
 630     sp.format.redMax = htons(31);
 631     sp.format.greenMax = htons(31);
 632     sp.format.blueMax = htons(31);
 633     sp.format.redShift = 10;
 634     sp.format.greenShift = 5;
 635     sp.format.blueShift = 0;
 636
 637     if( !write_exact( p_filter, p_sys->i_socket,
 638                       (char*)&sp, sz_rfbSetPixelFormatMsg) )
 639     {
 640         msg_Err( p_filter, "Could not write SetPixelFormat message" );
 641         return false;
 642     }
 643
 644     msg_Dbg( p_filter, "Writing SetEncodings message" );
 645
 646     rfbSetEncodingsMsg se;
 647     se.type = rfbSetEncodings;
 648     se.nEncodings = htons( p_sys->b_alpha_from_vnc ? 3 : 2 );
 649
 650     if( !write_exact( p_filter, p_sys->i_socket,
 651                       (char*)&se, sz_rfbSetEncodingsMsg) )
 652     {
 653         msg_Err( p_filter, "Could not write SetEncodings message begin" );
 654         return false;
 655     }
 656
 657     uint32_t i_encoding;
 658
 659     msg_Dbg( p_filter, "Writing SetEncodings rfbEncodingCopyRect" );
 660     i_encoding = htonl(rfbEncodingCopyRect);
 661     if( !write_exact( p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
 662     {
 663         msg_Err( p_filter, "Could not write encoding type rfbEncodingCopyRect." );
 664         return false;
 665     }
 666
 667     msg_Dbg( p_filter, "Writing SetEncodings rfbEncodingRRE" );
 668     i_encoding = htonl(rfbEncodingRRE);
 669     if( !write_exact(p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
 670     {
 671         msg_Err( p_filter, "Could not write encoding type rfbEncodingRRE." );
 672         return false;
 673     }
 674
 675     if( p_sys->b_alpha_from_vnc )
 676     {
 677         msg_Dbg( p_filter, "Writing SetEncodings rfbEncSpecialUseAlpha" );
 678         i_encoding = 0x00F0FFFF; /* rfbEncSpecialUseAlpha is 0xFFFFF000
 679                                   * before we swap it */
 680         if( !write_exact(p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
 681         {
 682             msg_Err( p_filter, "Could not write encoding type rfbEncSpecialUseAlpha." );
 683             return false;
 684         }
 685     }
 686     return true;
 687
 688 }
 689
 690 static void vnc_worker_thread( vlc_object_t *p_thread_obj )
 691 {
 692     filter_t* p_filter = (filter_t*)(p_thread_obj->p_parent);
 693     filter_sys_t *p_sys = p_filter->p_sys;
 694
 695     msg_Dbg( p_filter, "VNC worker thread started" );
 696
 697     if ( open_vnc_connection ( p_filter ) == false )
 698     {
 699         msg_Err( p_filter, "Could not connect to vnc host" );
 700         return;
 701     }
 702
 703     if ( handshaking ( p_filter ) == false )
 704     {
 705         msg_Err( p_filter, "Error occured while handshaking vnc host" );
 706         return;
 707     }
 708
 709     p_sys->b_connection_active = true; /* to enable sending key
 710                                             * and mouse events to host */
 711
 712     /* Create an empty picture for VNC the data */
 713     vlc_mutex_lock( &p_sys->lock );
 714     p_sys->p_pic = malloc( sizeof(picture_t) );
 715     if( !p_sys->p_pic )
 716     {
 717         vlc_mutex_unlock( &p_sys->lock );
 718         return;
 719     }
 720     vout_AllocatePicture( VLC_OBJECT(p_filter), p_sys->p_pic,
 721                           VLC_FOURCC('Y','U','V','A'),
 722                           p_sys->i_vnc_width,
 723                           p_sys->i_vnc_height,
 724                           VOUT_ASPECT_FACTOR );
 725     if( !p_sys->p_pic->i_planes )
 726     {
 727         free( p_sys->p_pic );
 728         p_sys->p_pic = NULL;
 729         vlc_mutex_unlock( &p_sys->lock );
 730         return;
 731     }
 732         p_sys->i_vnc_pixels = p_sys->i_vnc_width * p_sys->i_vnc_height;
 733
 734     vlc_mutex_unlock( &p_sys->lock );
 735
 736     /* create the update request thread */
 737     p_sys->p_update_request_thread = vlc_object_create( p_filter,
 738                                                         VLC_OBJECT_GENERIC );
 739     vlc_object_attach( p_sys->p_update_request_thread, p_filter );
 740     if( vlc_thread_create( p_sys->p_update_request_thread,
 741                            "vnc update request thread",
 742                            update_request_thread,
 743                            VLC_THREAD_PRIORITY_LOW, false ) )
 744     {
 745         vlc_object_detach( p_sys->p_update_request_thread );
 746         vlc_object_release( p_sys->p_update_request_thread );
 747         p_sys->p_update_request_thread = NULL;
 748         msg_Err( p_filter, "cannot spawn vnc update request thread" );
 749         return;
 750     }
 751
 752     /* connection is initialized, now read and handle server messages */
 753
 754     int i_msgSize;
 755
 756     rfbServerToClientMsg msg;
 757
 758     while (p_sys->b_continue)
 759     {
 760        msg.type = 0;
 761        if ( !read_exact(p_filter, p_sys->i_socket, (char*)&msg, 1 ) )
 762        {
 763            msg_Err( p_filter, "Error while waiting for next server message");
 764            p_sys->b_continue = false;
 765        }
 766        else
 767        {
 768            switch (msg.type)
 769            {
 770               case rfbFramebufferUpdate:
 771                 i_msgSize = sz_rfbFramebufferUpdateMsg;
 772                 break;
 773               case rfbSetColourMapEntries:
 774                 i_msgSize = sz_rfbSetColourMapEntriesMsg;
 775                 break;
 776               case rfbBell:
 777                 i_msgSize = sz_rfbBellMsg;
 778                 break;
 779               case rfbServerCutText:
 780                 i_msgSize = sz_rfbServerCutTextMsg;
 781                 break;
 782               case rfbReSizeFrameBuffer:
 783                 i_msgSize = sz_rfbReSizeFrameBufferMsg;
 784                 break;
 785               default:
 786                 i_msgSize = 0;
 787                 msg_Err( p_filter, "Invalid message %u received", msg.type );
 788                 p_sys->b_continue = false;
 789            }
 790            if (p_sys->b_continue == true)
 791            {
 792                if (--i_msgSize > 0)
 793                {
 794                    if ( !read_exact( p_filter, p_sys->i_socket,
 795                                      ((char*)&msg)+1, i_msgSize ) )
 796                    {
 797                        msg_Err( p_filter, "Error while reading message of type %u",
 798                                 msg.type );
 799                        p_sys->b_continue = false;
 800                    }
 801                    else
 802                    {
 803                        process_server_message( p_filter, &msg);
 804                    }
 805                }
 806                else
 807                {
 808                    process_server_message( p_filter, &msg);
 809                }
 810            }
 811
 812        }
 813
 814        if (p_sys->p_worker_thread->b_die ||
 815            p_sys->p_worker_thread->b_error)
 816        {
 817            p_sys->b_continue = false;
 818        }
 819
 820     }
 821
 822     msg_Dbg( p_filter, "VNC message reader thread ended" );
 823
 824 }
 825
 826 static void update_request_thread( vlc_object_t *p_thread_obj )
 827 {
 828     filter_t* p_filter = (filter_t*)(p_thread_obj->p_parent);
 829     filter_sys_t *p_sys = p_filter->p_sys;
 830
 831     msg_Dbg( p_filter, "VNC update request thread started" );
 832
 833     rfbFramebufferUpdateRequestMsg udr;
 834     udr.type = rfbFramebufferUpdateRequest;
 835     udr.incremental = 0;
 836     udr.x = 0;
 837     udr.y = 0;
 838     udr.w = htons(p_sys->i_vnc_width);
 839     udr.h = htons(p_sys->i_vnc_height);
 840
 841     if( write_exact(p_filter, p_sys->i_socket, (char*)&udr,
 842            sz_rfbFramebufferUpdateRequestMsg) == false)
 843     {
 844         msg_Err( p_filter, "Could not write rfbFramebufferUpdateRequestMsg." );
 845         p_sys->b_continue = false;
 846     }
 847
 848     udr.incremental = 1;
 849     mtime_t i_poll_interval_microsec = p_sys->i_vnc_poll_interval * 1000;
 850
 851     if (p_sys->b_vnc_poll)
 852     {
 853         while ( p_sys->b_continue == true )
 854         {
 855             msleep( i_poll_interval_microsec );
 856             if( write_exact(p_filter, p_sys->i_socket, (char*)&udr,
 857                    sz_rfbFramebufferUpdateRequestMsg) == false)
 858             {
 859                 msg_Err( p_filter, "Could not write rfbFramebufferUpdateRequestMsg." );
 860                 p_sys->b_continue = false;
 861             }
 862             if (p_sys->p_update_request_thread->b_die ||
 863                 p_sys->p_update_request_thread->b_error)
 864             {
 865                 p_sys->b_continue = false;
 866             }
 867         }
 868     }
 869     else
 870     {
 871         msg_Dbg( p_filter, "VNC polling disabled." );
 872     }
 873     msg_Dbg( p_filter, "VNC update request thread ended" );
 874 }
 875
 876 static bool process_server_message ( filter_t *p_filter,
 877                                      rfbServerToClientMsg *msg )
 878 {
 879     filter_sys_t *p_sys = p_filter->p_sys;
 880
 881     switch (msg->type)
 882     {
 883     case rfbFramebufferUpdate:
 884         {
 885             msg->fu.nRects = htons(msg->fu.nRects);
 886             rfbFramebufferUpdateRectHeader hdr;
 887
 888             for (int i_rect = 0; i_rect < msg->fu.nRects; i_rect++)
 889             {
 890                 if (!read_exact(p_filter, p_sys->i_socket, (char*)&hdr,
 891                     sz_rfbFramebufferUpdateRectHeader ) )
 892                 {
 893                     msg_Err( p_filter, "Could not read FrameBufferUpdate header" );
 894                     return false;
 895                 }
 896                 hdr.r.x = htons(hdr.r.x);
 897                 hdr.r.y = htons(hdr.r.y);
 898                 hdr.r.w = htons(hdr.r.w);
 899                 hdr.r.h = htons(hdr.r.h);
 900                 hdr.encoding = htonl(hdr.encoding);
 901
 902                 switch (hdr.encoding)
 903                 {
 904                 case rfbEncodingRaw:
 905                     {
 906                         int i_line;
 907                         for (i_line = 0; i_line < hdr.r.h; i_line++)
 908                         {
 909                             if ( !read_exact( p_filter, p_sys->i_socket,
 910                                     p_sys->read_buffer, hdr.r.w ) )
 911                             {
 912                                 msg_Err( p_filter,
 913                                  "Could not read FrameBufferUpdate line data" );
 914                                return false;
 915                             }
 916                             vlc_mutex_lock( &p_sys->lock );
 917                             if ( !raw_line( p_sys, hdr.r.x,
 918                                             hdr.r.y + i_line,
 919                                             hdr.r.w ) )
 920                             {
 921                                 msg_Err( p_filter, "raw_line failed." );
 922                                 vlc_mutex_unlock( &p_sys->lock );
 923                                 return false;
 924                             }
 925                             vlc_mutex_unlock( &p_sys->lock );
 926                         }
 927                     }
 928                     break;
 929
 930                 case rfbEncodingCopyRect:
 931                     {
 932                         msg_Err( p_filter,
 933                           "Rect in unsupported encoding rfbEncodingCopyRect" );
 934                         return false;
 935                     }
 936
 937                 case rfbEncodingRRE:
 938                     {
 939                         rfbRREHeader rrehdr;
 940                         if ( !read_exact( p_filter, p_sys->i_socket,
 941                                           (char*)&rrehdr,
 942                                           sz_rfbRREHeader ) )
 943                         {
 944                             msg_Err( p_filter, "Could not read rfbRREHeader" );
 945                             return false;
 946                         }
 947                         uint8_t i_pixcolor;
 948                         if ( !read_exact(p_filter, p_sys->i_socket,
 949                                          (char*)&i_pixcolor, 1 ) )
 950                         {
 951                             msg_Err( p_filter, "Could not read RRE pixcolor" );
 952                             return false;
 953                         }
 954
 955                         vlc_mutex_lock( &p_sys->lock );
 956                         if ( !fill_rect( p_sys,
 957                                         hdr.r.x, hdr.r.y,
 958                                         hdr.r.w, hdr.r.h,
 959                                         i_pixcolor) )
 960                         {
 961                             msg_Err( p_filter, "main fill_rect failed." );
 962                             vlc_mutex_unlock( &p_sys->lock );
 963                             return false;
 964                         }
 965                         vlc_mutex_unlock( &p_sys->lock );
 966
 967                         rrehdr.nSubrects = htonl(rrehdr.nSubrects);
 968
 969                         int i_datasize = rrehdr.nSubrects *
 970                                      ( sizeof(i_pixcolor) + sz_rfbRectangle ) ;
 971                         if ( i_datasize > READ_BUFFER_SIZE )
 972                         {
 973                             msg_Err( p_filter, "Buffer too small, "
 974                                      "need %u bytes", i_datasize );
 975                             return false;
 976                         }
 977                         if ( !read_exact( p_filter, p_sys->i_socket,
 978                                        p_sys->read_buffer, i_datasize ) )
 979                         {
 980                             msg_Err( p_filter,
 981                                      "Could not read RRE subrect data" );
 982                             return false;
 983                         }
 984
 985                         uint32_t i_subrect;
 986                         rfbRectangle* p_subrect;
 987                         int i_offset = 0;
 988                         vlc_mutex_lock( &p_sys->lock );
 989                         for ( i_subrect = 0;
 990                               i_subrect < rrehdr.nSubrects; i_subrect++)
 991                         {
 992                             i_pixcolor = p_sys->read_buffer[i_offset];
 993                             i_offset += sizeof(i_pixcolor);
 994                             p_subrect =
 995                                (rfbRectangle*)(p_sys->read_buffer + i_offset);
 996                             i_offset += sz_rfbRectangle;
 997
 998                             if (!fill_rect( p_sys,
 999                                             htons(p_subrect->x) + hdr.r.x,
1000                                             htons(p_subrect->y) + hdr.r.y,
1001                                             htons(p_subrect->w),
1002                                             htons(p_subrect->h),
1003                                             i_pixcolor) )
1004                             {
1005                                 msg_Err( p_filter,
1006                                     "subrect %u fill_rect failed.", i_subrect );
1007                                 vlc_mutex_unlock( &p_sys->lock );
1008                                 return false;
1009                             }
1010                         }
1011                         vlc_mutex_unlock( &p_sys->lock );
1012                     }
1013                     break;
1014
1015                 }
1016
1017             }
1018             vlc_mutex_lock( &p_sys->lock );
1019             p_sys->b_need_update = true;
1020             vlc_mutex_unlock( &p_sys->lock );
1021         }
1022         return true;
1023
1024     case rfbSetColourMapEntries:
1025         {
1026             msg->scme.nColours = htons(msg->scme.nColours);
1027             msg->scme.firstColour = htons(msg->scme.firstColour);
1028             int i_datasize;
1029             if ( p_sys->b_alpha_from_vnc == true )
1030             {
1031                 i_datasize = 2 * msg->scme.nColours * 4;
1032             }
1033             else
1034             {
1035                 i_datasize = 2 * msg->scme.nColours * 3;
1036             }
1037             if ( i_datasize > READ_BUFFER_SIZE )
1038             {
1039                 msg_Err( p_filter, "Buffer too small, need %u bytes",
1040                                    i_datasize );
1041                 return false;
1042             }
1043
1044             if ( !read_exact( p_filter, p_sys->i_socket,
1045                               p_sys->read_buffer, i_datasize ) )
1046             {
1047                 msg_Err( p_filter, "Could not read color map data" );
1048                 return false;
1049             }
1050
1051             uint8_t i_red, i_green, i_blue, i_alpha, i_color_index;
1052             uint16_t i_offset = 0;
1053             i_alpha = 255;
1054
1055             for (int i = 0; i < msg->scme.nColours; i++)
1056             {
1057                 i_color_index = i+msg->scme.firstColour;
1058                 if ( p_sys->b_alpha_from_vnc == true )
1059                 {
1060                     i_alpha = p_sys->read_buffer[i_offset];
1061                     i_offset += 2;
1062                 }
1063                 i_red   = p_sys->read_buffer[i_offset];
1064                 i_offset += 2;
1065                 i_green = p_sys->read_buffer[i_offset];
1066                 i_offset += 2;
1067                 i_blue  = p_sys->read_buffer[i_offset];
1068                 i_offset += 2;
1069                 rgb_to_yuv( &p_sys->ar_color_table_yuv[i_color_index][0],
1070                             &p_sys->ar_color_table_yuv[i_color_index][1],
1071                             &p_sys->ar_color_table_yuv[i_color_index][2],
1072                             i_red,
1073                             i_green,
1074                             i_blue );
1075                 p_sys->ar_color_table_yuv[i][3] = i_alpha;
1076             }
1077         }
1078         return true;
1079
1080     case rfbBell:
1081         msg_Err( p_filter, "rfbBell received" );
1082         return true;
1083
1084     case rfbServerCutText:
1085         msg->sct.length = htons(msg->sct.length);
1086         if ( msg->sct.length > READ_BUFFER_SIZE )
1087         {
1088             msg_Err( p_filter, "Buffer too small, need %u bytes", msg->sct.length );
1089             return false;
1090         }
1091         if ( !read_exact(p_filter, p_sys->i_socket,
1092                          p_sys->read_buffer, msg->sct.length ) )
1093         {
1094             msg_Err( p_filter, "Could not read Reading rfbServerCutText data" );
1095             return false;
1096         }
1097         return true;
1098
1099     case rfbReSizeFrameBuffer:
1100         msg_Err( p_filter, "Reading rfbReSizeFrameBuffer not implemented" );
1101         return false;
1102
1103     default:
1104         msg_Err( p_filter, "Invalid message %u received", msg->type );
1105         return false;
1106     }
1107     return false;
1108 }
1109
1110 /****************************************************************************
1111  * Filter: the whole thing
1112  ****************************************************************************
1113  * This function outputs subpictures at regular time intervals.
1114  ****************************************************************************/
1115 static subpicture_t *Filter( filter_t *p_filter, mtime_t date )
1116 {
1117     filter_sys_t *p_sys = p_filter->p_sys;
1118     subpicture_t *p_spu;
1119     subpicture_region_t *p_region;
1120     video_format_t fmt;
1121     picture_t *p_pic;
1122
1123     if( !p_sys->b_need_update )
1124     {
1125         return NULL;
1126     }
1127
1128     vlc_mutex_lock( &p_sys->lock );
1129
1130     p_pic = p_sys->p_pic;
1131
1132     if( !p_pic )
1133     {
1134         vlc_mutex_unlock( &p_sys->lock );
1135         return NULL;
1136     }
1137
1138     /* Allocate the subpicture internal data. */
1139     p_spu = p_filter->pf_sub_buffer_new( p_filter );
1140     if( !p_spu )
1141     {
1142         vlc_mutex_unlock( &p_sys->lock );
1143         return NULL;
1144     }
1145
1146     p_spu->b_absolute = false;
1147     p_spu->i_start = date;
1148     p_spu->i_stop = 0;
1149     p_spu->b_ephemer = true;
1150
1151     /* Create new SPU region */
1152     memset( &fmt, 0, sizeof(video_format_t) );
1153     fmt.i_chroma = VLC_FOURCC('Y','U','V','A');
1154     fmt.i_aspect = VOUT_ASPECT_FACTOR;
1155     fmt.i_sar_num = fmt.i_sar_den = 1;
1156     fmt.i_width = fmt.i_visible_width = p_pic->p[Y_PLANE].i_visible_pitch;
1157     fmt.i_height = fmt.i_visible_height = p_pic->p[Y_PLANE].i_visible_lines;
1158     fmt.i_x_offset = fmt.i_y_offset = 0;
1159     p_region = p_spu->pf_create_region( VLC_OBJECT(p_filter), &fmt );
1160     if( !p_region )
1161     {
1162         msg_Err( p_filter, "cannot allocate SPU region" );
1163         p_filter->pf_sub_buffer_del( p_filter, p_spu );
1164         vlc_mutex_unlock( &p_sys->lock );
1165         return NULL;
1166     }
1167
1168     vout_CopyPicture( p_filter, &p_region->picture, p_pic );
1169
1170     p_sys->b_need_update = false;
1171
1172     vlc_mutex_unlock( &p_sys->lock );
1173
1174     /* set to one of the 9 relative locations */
1175     p_region->i_align = 0; //=CENTER
1176     p_spu->b_absolute = false;
1177
1178
1179     p_spu->i_x = 0;
1180     p_spu->i_y = 0;
1181
1182     p_spu->p_region = p_region;
1183
1184     p_spu->i_alpha = ( p_sys->i_alpha );
1185
1186     return p_spu;
1187 }
1188
1189
1190 static inline void rgb_to_yuv( uint8_t *y, uint8_t *u, uint8_t *v,
1191                                int r, int g, int b )
1192 {
1193     *y = ( ( (  66 * r + 129 * g +  25 * b + 128 ) >> 8 ) + 16 );
1194     *u =   ( ( -38 * r -  74 * g + 112 * b + 128 ) >> 8 ) + 128 ;
1195     *v =   ( ( 112 * r -  94 * g -  18 * b + 128 ) >> 8 ) + 128 ;
1196 }
1197
1198 static inline bool fill_rect( filter_sys_t* p_sys,
1199                               uint16_t i_x, uint16_t i_y,
1200                               uint16_t i_w, uint16_t i_h,
1201                               uint8_t i_color)
1202 {
1203     plane_t *p_outY = p_sys->p_pic->p+Y_PLANE;
1204     plane_t *p_outU = p_sys->p_pic->p+U_PLANE;
1205     plane_t *p_outV = p_sys->p_pic->p+V_PLANE;
1206     plane_t *p_outA = p_sys->p_pic->p+A_PLANE;
1207     int i_pitch = p_outY->i_pitch;
1208     int i_lines = p_outY->i_lines;
1209     if ( i_x + i_w > i_pitch)
1210         return false;
1211     if ( i_y + i_h > i_lines)
1212         return false;
1213     int i_line_offset = i_y * i_pitch;
1214     uint8_t i_yuv_y = p_sys->ar_color_table_yuv[i_color][0];
1215     uint8_t i_yuv_u = p_sys->ar_color_table_yuv[i_color][1];
1216     uint8_t i_yuv_v = p_sys->ar_color_table_yuv[i_color][2];
1217     uint8_t i_alpha = p_sys->ar_color_table_yuv[i_color][3];
1218     for( int i_line = 0; i_line < i_h; i_line++ )
1219     {
1220         for( int i_column = 0; i_column < i_w; i_column++ )
1221         {
1222             int i_total_offset = i_line_offset + i_x + i_column;
1223             p_outY->p_pixels[ i_total_offset ] = i_yuv_y;
1224             p_outU->p_pixels[ i_total_offset ] = i_yuv_u;
1225             p_outV->p_pixels[ i_total_offset ] = i_yuv_v;
1226             p_outA->p_pixels[ i_total_offset ] = i_alpha;
1227         }
1228         i_line_offset += i_pitch;
1229     }
1230     return true;
1231 }
1232
1233 static inline bool raw_line(  filter_sys_t* p_sys,
1234                               uint16_t i_x, uint16_t i_y,
1235                               uint16_t i_w )
1236 {
1237     plane_t *p_outY = p_sys->p_pic->p+Y_PLANE;
1238     plane_t *p_outU = p_sys->p_pic->p+U_PLANE;
1239     plane_t *p_outV = p_sys->p_pic->p+V_PLANE;
1240     plane_t *p_outA = p_sys->p_pic->p+A_PLANE;
1241     int i_pitch = p_outY->i_pitch;
1242     int i_lines = p_outY->i_lines;
1243     if ( i_x + i_w > i_pitch)
1244         return false;
1245     if ( i_y > i_lines)
1246         return false;
1247
1248     int i_line_offset = i_y * i_pitch + i_x;
1249
1250     for( int i_column = 0; i_column < i_w; i_column++ )
1251     {
1252         int i_offset = i_line_offset + i_column;
1253         uint8_t i_color = p_sys->read_buffer[i_column];
1254         p_outY->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][0];
1255         p_outU->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][1];
1256         p_outV->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][2];
1257         p_outA->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][3];
1258     }
1259
1260     return true;
1261 }
1262
1263
1264 /*****************************************************************************
1265  * MouseEvent: callback for mouse events
1266  *****************************************************************************/
1267 static int MouseEvent( vlc_object_t *p_this, char const *psz_var,
1268                        vlc_value_t oldval, vlc_value_t newval, void *p_data )
1269 {
1270     VLC_UNUSED(oldval); VLC_UNUSED(newval); VLC_UNUSED(psz_var);
1271
1272     filter_t *p_filter = (filter_t *)p_data;
1273     filter_sys_t *p_sys = p_filter->p_sys;
1274
1275     if( !p_sys->b_connection_active )
1276         return VLC_SUCCESS;
1277
1278     if( !p_sys->b_vnc_mouse_events )
1279         return VLC_SUCCESS;
1280
1281     vout_thread_t *p_vout = (vout_thread_t*)p_sys->p_vout;
1282     int i_x, i_y;
1283     int i_v;
1284
1285     int v_h = p_vout->output.i_height;
1286     int v_w = p_vout->output.i_width;
1287
1288     i_v = var_GetInteger( p_sys->p_vout, "mouse-button-down" );
1289     i_y = var_GetInteger( p_sys->p_vout, "mouse-y" );
1290     i_x = var_GetInteger( p_sys->p_vout, "mouse-x" );
1291
1292     if( i_y < 0 || i_x < 0 || i_y >= v_h || i_x >= v_w )
1293     {
1294        msg_Dbg( p_this, "invalid mouse event? x=%d y=%d btn=%x", i_x, i_y, i_v );
1295        return VLC_SUCCESS;
1296     }
1297 #ifdef VNC_DEBUG
1298     msg_Dbg( p_this, "mouse event x=%d y=%d btn=%x", i_x, i_y, i_v );
1299 #endif
1300
1301     /* FIXME: calculate x and y coordinates for scaled output */
1302
1303     /* buttonMask bits 0-7 are buttons 1-8, 0=up, 1=down */
1304     rfbPointerEventMsg ev;
1305     ev.type = rfbPointerEvent;
1306     ev.buttonMask = i_v;
1307     ev.x = htons(i_x);
1308     ev.y = htons(i_y);
1309
1310     write_exact( p_filter, p_sys->i_socket,
1311                  (char*)&ev, sz_rfbPointerEventMsg);
1312
1313     return VLC_SUCCESS;
1314 }
1315
1316 /*****************************************************************************
1317  * KeyEvent: callback for keyboard events
1318  *****************************************************************************/
1319 static int KeyEvent( vlc_object_t *p_this, char const *psz_var,
1320                      vlc_value_t oldval, vlc_value_t newval, void *p_data )
1321 {
1322     VLC_UNUSED(psz_var); VLC_UNUSED(oldval);
1323
1324     filter_t *p_filter = (filter_t *)p_data;
1325     filter_sys_t *p_sys = p_filter->p_sys;
1326
1327     if( !p_sys->b_connection_active )
1328         return VLC_SUCCESS;
1329
1330     if( !p_sys->b_vnc_key_events )
1331         return VLC_SUCCESS;
1332
1333     msg_Dbg( p_this, "key pressed (%d) ", newval.i_int );
1334
1335     if ( !newval.i_int )
1336     {
1337         msg_Err( p_this, "Received invalid key event %d", newval.i_int );
1338         return VLC_EGENERIC;
1339     }
1340
1341     uint32_t i_key32 = newval.i_int;
1342     i_key32 = htonl(i_key32);
1343     rfbKeyEventMsg ev;
1344     ev.type = rfbKeyEvent;
1345     ev.down = 1;
1346
1347     /* first key-down for modifier-keys */
1348     if (newval.i_int & KEY_MODIFIER_CTRL)
1349     {
1350         ev.key = 0xffe3;
1351         write_exact( p_filter, p_sys->i_socket,
1352                      (char*)&ev, sz_rfbKeyEventMsg);
1353     }
1354     if (newval.i_int & KEY_MODIFIER_SHIFT)
1355     {
1356         ev.key = 0xffe1;
1357         write_exact( p_filter, p_sys->i_socket,
1358                      (char*)&ev, sz_rfbKeyEventMsg);
1359     }
1360     if (newval.i_int & KEY_MODIFIER_ALT)
1361     {
1362         ev.key = 0xffe9;
1363         write_exact( p_filter, p_sys->i_socket,
1364                      (char*)&ev, sz_rfbKeyEventMsg);
1365     }
1366
1367     /* then key-down for the pressed key */
1368     ev.key = i_key32;
1369     write_exact( p_filter, p_sys->i_socket,
1370                  (char*)&ev, sz_rfbKeyEventMsg);
1371
1372     ev.down = 0;
1373
1374     /* then key-up for the pressed key */
1375     write_exact( p_filter, p_sys->i_socket,
1376                  (char*)&ev, sz_rfbKeyEventMsg);
1377
1378     /* last key-down for modifier-keys */
1379     if (newval.i_int & KEY_MODIFIER_CTRL)
1380     {
1381         ev.key = 0xffe3;
1382         write_exact( p_filter, p_sys->i_socket,
1383                      (char*)&ev, sz_rfbKeyEventMsg);
1384     }
1385     if (newval.i_int & KEY_MODIFIER_SHIFT)
1386     {
1387         ev.key = 0xffe1;
1388         write_exact( p_filter, p_sys->i_socket,
1389                      (char*)&ev, sz_rfbKeyEventMsg);
1390     }
1391     if (newval.i_int & KEY_MODIFIER_ALT)
1392     {
1393        ev.key = 0xffe9;
1394        write_exact( p_filter, p_sys->i_socket,
1395                     (char*)&ev, sz_rfbKeyEventMsg);
1396     }
1397
1398     return VLC_SUCCESS;
1399 }
1400
1401 static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd )
1402 {
1403     unsigned char key[8];
1404     unsigned int i;
1405
1406     for (i = 0; i < 8; i++)
1407         key[i] = i < strlen( passwd ) ? passwd[i] : '\0';
1408
1409     gcry_cipher_hd_t ctx;
1410     gcry_cipher_open( &ctx, GCRY_CIPHER_DES, GCRY_CIPHER_MODE_ECB,0);
1411
1412     /* reverse bits of the key */
1413     for( i = 0 ; i < 8 ; i ++ )
1414         key[i] =
1415             (key[i] >> 7) +
1416             (((key[i] >> 6) & 0x01 ) << 1 ) +
1417             (((key[i] >> 5) & 0x01 ) << 2 ) +
1418             (((key[i] >> 4) & 0x01 ) << 3 ) +
1419             (((key[i] >> 3) & 0x01 ) << 4 ) +
1420             (((key[i] >> 2) & 0x01 ) << 5 ) +
1421             (((key[i] >> 1) & 0x01 ) << 6 ) +
1422             ((key[i] & 0x01) << 7 );
1423
1424     gcry_cipher_setkey( ctx, key, 8 );
1425     gcry_cipher_encrypt( ctx, bytes, CHALLENGESIZE, bytes, CHALLENGESIZE );
1426     gcry_cipher_close( ctx );
1427 }