git.sesse.net Git - vlc/blob - modules/video_filter/remoteosd.c

   1 /*****************************************************************************
   2  * remoteosd.c: remote osd over vnc filter module
   3  *****************************************************************************
   4  * Copyright (C) 2007-2008 Matthias Bauer
   5  * $Id$
   6  *
   7  * Authors: Matthias Bauer <matthias dot bauer #_at_# gmx dot ch>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implid warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  22  *****************************************************************************/
  23
  24 /*****************************************************************************
  25  * RemoteOSD uses the RFB-Protocol of VNC to display an On-Screen-Display
  26  * menu generated by a streaming server as overlay for the streamed video.
  27  *
  28  * The streaming server that implements this is the ffnetdev plugin for VDR.
  29  * VDR (VideoDiskRecorder) is an Linux based OpenSource harddisk recorder
  30  * software.
  31  * The VDR ffnetdev plugin emulates the hardware MPEG decoder and streams the
  32  * video over the network instead of hardware video outputs.
  33  * The OSD menu of VDR is offered with the RFB protocol to a VNC client.
  34  *
  35  * In fact this video-filter is a simple VNC client that could be also used to
  36  * connect to a real VNC host.
  37  * Only 8-bit color is supported at the moment.
  38  * Using password protected VNC hosts is supported but not recommended, because
  39  * you need to insert the used password in the plugin configuration page of
  40  * VLC configuration in plain text and it's saved in plain text.
  41  *****************************************************************************/
  42
  43 //#define VNC_DEBUG
  44
  45 /*****************************************************************************
  46  * Preamble
  47  *****************************************************************************/
  48
  49 #ifdef HAVE_CONFIG_H
  50 # include "config.h"
  51 #endif
  52
  53 #include <vlc_common.h>
  54 #include <vlc_plugin.h>
  55 #include <vlc_vout.h>
  56
  57 #include "vlc_filter.h"
  58 #include "filter_common.h"
  59 #include "vlc_image.h"
  60 #include "vlc_osd.h"
  61 #include "vlc_keys.h"
  62
  63 #include <vlc_network.h>
  64 #include <gcrypt.h>              /* to encrypt password */
  65 #include <vlc_gcrypt.h>
  66
  67 #define CHALLENGESIZE 16
  68 #define MAX_VNC_SERVER_NAME_LENGTH 255
  69
  70 #include "remoteosd_rfbproto.h" /* type definitions of the RFB protocol for VNC */
  71
  72 /*****************************************************************************
  73  * Local prototypes
  74  *****************************************************************************/
  75 /* subfilter functions */
  76 static int  CreateFilter ( vlc_object_t * );
  77 static void DestroyFilter( vlc_object_t * );
  78 static subpicture_t *Filter( filter_t *, mtime_t );
  79
  80 static int MouseEvent ( vlc_object_t *p_this, char const *psz_var,
  81                         vlc_value_t oldval, vlc_value_t newval, void *p_data );
  82
  83 static int KeyEvent( vlc_object_t *p_this, char const *psz_var,
  84                      vlc_value_t oldval, vlc_value_t newval, void *p_data );
  85
  86 static void stop_osdvnc ( filter_t *p_filter );
  87
  88 static void vnc_worker_thread ( vlc_object_t *p_thread_obj );
  89
  90 static void update_request_thread( vlc_object_t *p_thread_obj );
  91
  92 static bool open_vnc_connection ( filter_t *p_filter );
  93
  94 static bool handshaking ( filter_t *p_filter );
  95
  96 static bool process_server_message ( filter_t *p_filter,
  97                                      rfbServerToClientMsg *msg );
  98
  99 static bool read_exact( filter_t *p_filter,
 100                         int i_socket,
 101                         char* p_readbuf,
 102                         int i_bytes );
 103
 104 static bool write_exact( filter_t *p_filter,
 105                          int i_socket,
 106                          char* p_writebuf,
 107                          int i_bytes );
 108
 109 static inline void rgb_to_yuv( uint8_t *y, uint8_t *u, uint8_t *v,
 110                                int r, int g, int b );
 111
 112 static inline bool fill_rect( filter_sys_t* p_sys,
 113                               uint16_t i_x, uint16_t i_y,
 114                               uint16_t i_w, uint16_t i_h,
 115                               uint8_t i_color );
 116
 117 static inline bool raw_line(  filter_sys_t* p_sys,
 118                               uint16_t i_x, uint16_t i_y,
 119                               uint16_t i_w );
 120
 121 static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd );
 122
 123
 124 /*****************************************************************************
 125  * Module descriptor
 126  *****************************************************************************/
 127 #define READ_BUFFER_SIZE 1000000
 128
 129 #define RMTOSD_HOST_TEXT N_("VNC Host")
 130 #define RMTOSD_HOST_LONGTEXT N_( \
 131     "VNC hostname or IP address." )
 132
 133 #define RMTOSD_PORT_TEXT N_("VNC Port")
 134 #define RMTOSD_PORT_LONGTEXT N_( \
 135     "VNC portnumber." )
 136
 137 #define RMTOSD_PASSWORD_TEXT N_("VNC Password")
 138 #define RMTOSD_PASSWORD_LONGTEXT N_( \
 139     "VNC password." )
 140
 141 #define RMTOSD_UPDATE_TEXT N_("VNC poll interval" )
 142 #define RMTOSD_UPDATE_LONGTEXT N_( \
 143     "In this interval an update from VNC is requested, default every 300 ms. ")
 144
 145 #define RMTOSD_POLL_TEXT N_("VNC polling")
 146 #define RMTOSD_POLL_LONGTEXT N_( \
 147     "Activate VNC polling. Do NOT activate for use as VDR ffnetdev client." )
 148
 149 #define RMTOSD_MOUSE_TEXT N_("Mouse events")
 150 #define RMTOSD_MOUSE_LONGTEXT N_( \
 151     "Send mouse events to VNC host. Not needed for use as VDR ffnetdev client." )
 152
 153 #define RMTOSD_KEYS_TEXT N_("Key events")
 154 #define RMTOSD_KEYS_LONGTEXT N_( \
 155     "Send key events to VNC host." )
 156
 157 #define RMTOSD_ALPHA_TEXT N_("Alpha transparency value (default 255)")
 158 #define RMTOSD_ALPHA_LONGTEXT N_( \
 159     "The transparency of the OSD VNC can be changed by giving a value " \
 160     "between 0 and 255. A lower value specifies more transparency a higher " \
 161     "means less transparency. The default is being not transparent " \
 162     "(value 255) the minimum is fully transparent (value 0)." )
 163
 164 #define RMTOSD_CFG "rmtosd-"
 165
 166 #define RMTOSD_UPDATE_MIN     200
 167 #define RMTOSD_UPDATE_DEFAULT 1000
 168 #define RMTOSD_UPDATE_MAX     300
 169
 170
 171 vlc_module_begin();
 172     set_description( N_("Remote-OSD over VNC") );
 173     set_capability( "sub filter", 100 );
 174     set_shortname( N_("Remote-OSD") );
 175     set_category( CAT_VIDEO );
 176     set_subcategory( SUBCAT_VIDEO_SUBPIC );
 177     add_shortcut( "rmtosd" );
 178     set_callbacks( CreateFilter, DestroyFilter );
 179
 180     add_string( RMTOSD_CFG "host", "myvdr", NULL, RMTOSD_HOST_TEXT,
 181         RMTOSD_HOST_LONGTEXT, false );
 182     add_integer_with_range( RMTOSD_CFG "port", 20001, 1, 0xFFFF, NULL,
 183         RMTOSD_PORT_TEXT, RMTOSD_PORT_LONGTEXT, false );
 184     add_password( RMTOSD_CFG "password", "", NULL, RMTOSD_PASSWORD_TEXT,
 185         RMTOSD_PASSWORD_LONGTEXT, false );
 186     add_integer_with_range( RMTOSD_CFG "update", RMTOSD_UPDATE_DEFAULT,
 187         RMTOSD_UPDATE_MIN, RMTOSD_UPDATE_MAX, NULL, RMTOSD_UPDATE_TEXT,
 188         RMTOSD_UPDATE_LONGTEXT, true );
 189     add_bool( RMTOSD_CFG "vnc-polling", 0, NULL,
 190               RMTOSD_POLL_TEXT , RMTOSD_POLL_LONGTEXT, false );
 191     add_bool( RMTOSD_CFG "mouse-events", 0, NULL,
 192               RMTOSD_MOUSE_TEXT , RMTOSD_MOUSE_LONGTEXT, false );
 193     add_bool( RMTOSD_CFG "key-events", 0, NULL,
 194               RMTOSD_KEYS_TEXT , RMTOSD_KEYS_LONGTEXT, false );
 195     add_integer_with_range( RMTOSD_CFG "alpha", 255, 0, 255, NULL,
 196         RMTOSD_ALPHA_TEXT, RMTOSD_ALPHA_LONGTEXT, true );
 197
 198 vlc_module_end();
 199
 200 /*****************************************************************************
 201  * Sub filter code
 202  *****************************************************************************/
 203
 204 /*****************************************************************************
 205  * Local prototypes
 206  *****************************************************************************/
 207 struct filter_sys_t
 208 {
 209     VLC_COMMON_MEMBERS
 210
 211     bool          b_need_update;       /* VNC picture is updated, do update the OSD*/
 212     mtime_t       i_vnc_poll_interval; /* Update the OSD menu every n ms */
 213
 214     uint8_t       i_alpha;             /* alpha transparency value */
 215
 216     char          *psz_host;           /* VNC host */
 217     int           i_port;
 218
 219     char          *psz_passwd;         /* VNC password */
 220
 221     bool          b_vnc_poll;          /* Activate VNC polling ? */
 222     bool          b_vnc_mouse_events;  /* Send MouseEvents ? */
 223     bool          b_vnc_key_events;    /* Send KeyEvents ? */
 224
 225     bool          b_connection_active; /* Handshaking finished ? */
 226
 227     vlc_mutex_t   lock;                /* To lock for read/write on picture */
 228
 229     picture_t     *p_pic;              /* The picture with OSD data from VNC */
 230
 231     vout_thread_t *p_vout;             /* Pointer to video-out thread */
 232
 233     int           i_socket;            /* Socket used for VNC */
 234
 235     uint16_t      i_vnc_width;          /* The with of the VNC screen */
 236     uint16_t      i_vnc_height;         /* The height of the VNC screen */
 237         uint32_t      i_vnc_pixels;         /* The pixels of the VNC screen */
 238
 239     bool    b_alpha_from_vnc;    /* Special ffnetdev alpha feature enabled ? */
 240
 241     char          read_buffer[READ_BUFFER_SIZE];
 242
 243     bool          b_continue;
 244
 245     vlc_object_t* p_worker_thread;
 246     vlc_object_t* p_update_request_thread;
 247
 248     uint8_t       ar_color_table_yuv[256][4];
 249 };
 250
 251 /*****************************************************************************
 252  * CreateFilter: Create the filter and open the definition file
 253  *****************************************************************************/
 254 static int CreateFilter ( vlc_object_t *p_this )
 255 {
 256     filter_t *p_filter = (filter_t *)p_this;
 257     filter_sys_t *p_sys = NULL;
 258
 259     msg_Dbg( p_filter, "Creating vnc osd filter..." );
 260
 261     p_filter->p_sys = p_sys = (filter_sys_t *) malloc( sizeof(filter_sys_t) );
 262     if( !p_filter->p_sys )
 263     {
 264         msg_Err( p_filter, "out of memory" );
 265         return VLC_ENOMEM;
 266     }
 267     memset( p_sys, 0, sizeof(filter_sys_t) );
 268
 269     /* Populating struct */
 270     vlc_mutex_init( &p_sys->lock );
 271     p_sys->b_continue = true;
 272     p_sys->i_socket = -1;
 273
 274     p_sys->psz_host = var_CreateGetString( p_this, RMTOSD_CFG "host" );
 275     if( EMPTY_STR(p_sys->psz_host) )
 276     {
 277         msg_Err( p_filter, "unable to get vnc host" );
 278         goto error;
 279     }
 280
 281     p_sys->psz_passwd = var_CreateGetString( p_this, RMTOSD_CFG "password" );
 282     if( !p_sys->psz_passwd )
 283     {
 284         msg_Err( p_filter, "unable to get vnc password" );
 285         goto error;
 286     }
 287
 288     p_sys->i_port = var_CreateGetIntegerCommand( p_this, RMTOSD_CFG "port" );
 289
 290     p_sys->i_alpha = var_CreateGetIntegerCommand( p_this, RMTOSD_CFG "alpha" );
 291
 292     /* in miliseconds, 0 disables polling, should not be lower than 100 */
 293     p_sys->i_vnc_poll_interval  = var_CreateGetIntegerCommand( p_this,
 294                                                        RMTOSD_CFG "update" );
 295     if ( p_sys->i_vnc_poll_interval < 100)
 296     {
 297        p_sys->i_vnc_poll_interval = 100;
 298     }
 299
 300     for ( int i = 0; i < 256; i++ )
 301     {
 302         p_sys->ar_color_table_yuv[i][0] = 255;
 303         p_sys->ar_color_table_yuv[i][1] = 255;
 304         p_sys->ar_color_table_yuv[i][2] = 255;
 305         p_sys->ar_color_table_yuv[i][3] = 255;
 306     }
 307
 308     p_sys->b_vnc_poll = var_CreateGetBoolCommand( p_this,
 309                                             RMTOSD_CFG "vnc-polling" );
 310     p_sys->b_vnc_mouse_events = var_CreateGetBoolCommand( p_this,
 311                                             RMTOSD_CFG "mouse-events" );
 312     p_sys->b_vnc_key_events = var_CreateGetBoolCommand( p_this,
 313                                             RMTOSD_CFG "key-events" );
 314
 315     /* Keep track of OSD Events */
 316     p_sys->b_need_update  = false;
 317
 318     /* Attach subpicture filter callback */
 319     p_filter->pf_sub_filter = Filter;
 320
 321     p_sys->p_vout = vlc_object_find( p_this, VLC_OBJECT_VOUT, FIND_PARENT );
 322
 323     if( p_sys->p_vout )
 324     {
 325         var_AddCallback( p_sys->p_vout, "mouse-moved",
 326                          MouseEvent, p_this );
 327         var_AddCallback( p_sys->p_vout, "mouse-button-down",
 328                          MouseEvent, p_this );
 329         var_AddCallback( p_sys->p_vout->p_libvlc, "key-pressed",
 330                          KeyEvent, p_this );
 331     }
 332
 333     es_format_Init( &p_filter->fmt_out, SPU_ES, VLC_FOURCC( 's','p','u',' ' ) );
 334     p_filter->fmt_out.i_priority = 0;
 335
 336     /* create the vnc worker thread */
 337     p_sys->p_worker_thread = vlc_object_create( p_this, VLC_OBJECT_GENERIC );
 338     vlc_object_attach( p_sys->p_worker_thread, p_this );
 339     if( vlc_thread_create( p_sys->p_worker_thread, "vnc worker thread",
 340                            vnc_worker_thread,
 341                            VLC_THREAD_PRIORITY_LOW, false ) )
 342     {
 343         vlc_object_detach( p_sys->p_worker_thread );
 344         vlc_object_release( p_sys->p_worker_thread );
 345         p_sys->p_worker_thread = NULL;
 346         msg_Err( p_filter, "cannot spawn vnc message reader thread" );
 347         goto error;
 348     }
 349
 350     msg_Dbg( p_filter, "osdvnc filter started" );
 351
 352     return VLC_SUCCESS;
 353
 354 error:
 355     msg_Err( p_filter, "osdvnc filter discarded" );
 356
 357     p_sys->b_continue = false;
 358
 359     stop_osdvnc( p_filter );
 360
 361     free( p_sys->psz_host );
 362     free( p_sys->psz_passwd );
 363     free( p_sys );
 364
 365     return VLC_EGENERIC;
 366 }
 367
 368 /*****************************************************************************
 369  * DestroyFilter: Make a clean exit of this plugin
 370  *****************************************************************************/
 371 static void DestroyFilter( vlc_object_t *p_this )
 372 {
 373     filter_t     *p_filter = (filter_t*)p_this;
 374     filter_sys_t *p_sys = p_filter->p_sys;
 375
 376     msg_Dbg( p_filter, "DestroyFilter called." );
 377
 378     stop_osdvnc( p_filter );
 379
 380     if( p_sys->p_vout )
 381     {
 382         var_DelCallback( p_sys->p_vout, "mouse-moved",
 383                          MouseEvent, p_this );
 384         var_DelCallback( p_sys->p_vout, "mouse-button-down",
 385                          MouseEvent, p_this );
 386         var_DelCallback( p_sys->p_vout->p_libvlc, "key-pressed",
 387                          KeyEvent, p_this );
 388
 389         vlc_object_release( p_sys->p_vout );
 390         p_sys->p_vout = NULL;
 391     }
 392
 393     var_Destroy( p_this, RMTOSD_CFG "host" );
 394     var_Destroy( p_this, RMTOSD_CFG "port" );
 395     var_Destroy( p_this, RMTOSD_CFG "password" );
 396     var_Destroy( p_this, RMTOSD_CFG "update" );
 397     var_Destroy( p_this, RMTOSD_CFG "vnc-polling" );
 398     var_Destroy( p_this, RMTOSD_CFG "mouse-events" );
 399     var_Destroy( p_this, RMTOSD_CFG "key-events" );
 400     var_Destroy( p_this, RMTOSD_CFG "alpha" );
 401
 402     free( p_sys->psz_host );
 403     free( p_sys->psz_passwd );
 404     free( p_sys );
 405 }
 406
 407 static void stop_osdvnc ( filter_t *p_filter )
 408 {
 409     filter_sys_t *p_sys = p_filter->p_sys;
 410
 411     if (p_sys->i_socket >= 0)
 412     {
 413         net_Close(p_sys->i_socket);
 414     }
 415     p_sys->b_continue = false; /* this causes the threads to stop */
 416
 417     if ( p_sys->p_worker_thread )
 418     {
 419         msg_Dbg( p_filter, "joining worker_thread" );
 420         vlc_thread_join( p_sys->p_worker_thread );
 421         vlc_object_detach( p_sys->p_worker_thread );
 422         vlc_object_release( p_sys->p_worker_thread );
 423         msg_Dbg( p_filter, "released worker_thread" );
 424     }
 425
 426     if ( p_sys->p_update_request_thread )
 427     {
 428         msg_Dbg( p_filter, "joining update_request_thread" );
 429         vlc_thread_join( p_sys->p_update_request_thread );
 430         vlc_object_detach( p_sys->p_update_request_thread );
 431         vlc_object_release( p_sys->p_update_request_thread );
 432         msg_Dbg( p_filter, "released update_request_thread" );
 433     }
 434
 435     msg_Dbg( p_filter, "osdvnc stopped" );
 436 }
 437
 438 static bool read_exact( filter_t *p_filter,
 439                         int i_socket,
 440                         char* p_readbuf,
 441                         int i_bytes )
 442 {
 443     return i_bytes == net_Read( p_filter, i_socket, NULL,
 444                                   (unsigned char*)p_readbuf,
 445                                   i_bytes, true );
 446 }
 447
 448
 449 static bool write_exact( filter_t *p_filter,
 450                          int i_socket,
 451                          char* p_writebuf,
 452                          int i_bytes )
 453 {
 454     return i_bytes == net_Write( p_filter, i_socket, NULL,
 455                                   (unsigned char*)p_writebuf, i_bytes );
 456 }
 457
 458 static bool open_vnc_connection ( filter_t *p_filter )
 459 {
 460     filter_sys_t *p_sys = p_filter->p_sys;
 461
 462     msg_Dbg( p_filter, "Open socket to vnc server on %s:%u.",
 463               p_sys->psz_host, p_sys->i_port );
 464
 465     p_sys->i_socket = net_ConnectTCP( p_filter, p_sys->psz_host, p_sys->i_port );
 466
 467     if( p_sys->i_socket < 0 )
 468     {
 469         msg_Err( p_filter, "Could not open socket" );
 470         return false;
 471     }
 472
 473     msg_Dbg( p_filter, "socket is open." );
 474
 475     return true;
 476 }
 477
 478 static bool handshaking ( filter_t *p_filter )
 479 {
 480     filter_sys_t *p_sys = p_filter->p_sys;
 481
 482     msg_Dbg( p_filter, "Reading protocol version" );
 483
 484     rfbProtocolVersionMsg pv;
 485     if ( !read_exact( p_filter, p_sys->i_socket, pv,
 486                       sz_rfbProtocolVersionMsg ) )
 487     {
 488         msg_Err( p_filter, "Could not read version message" );
 489         return false;
 490     }
 491     pv[sz_rfbProtocolVersionMsg] = '\0'; /* pv size is sz_rfbProtocolVersionMsg+1 */
 492
 493     msg_Dbg( p_filter, "Server version is %s", pv );
 494
 495     strncpy(pv, "RFB 003.003\n", sz_rfbProtocolVersionMsg);
 496
 497     if( !write_exact(p_filter, p_sys->i_socket, pv,
 498                      sz_rfbProtocolVersionMsg) )
 499     {
 500         msg_Err( p_filter, "Could not write version message" );
 501         return false;
 502     }
 503
 504     msg_Dbg( p_filter, "Reading authentication scheme" );
 505     uint32_t i_authScheme;
 506     if( !read_exact( p_filter, p_sys->i_socket, (char*)&i_authScheme, 4 ) )
 507     {
 508         msg_Err( p_filter, "Could not read authentication scheme" );
 509         return false;
 510     }
 511     i_authScheme = htonl(i_authScheme);
 512
 513     msg_Dbg( p_filter, "Authentication scheme = %x", i_authScheme );
 514     if ( i_authScheme == rfbConnFailed )
 515     {
 516         msg_Err( p_filter, "Connection rejected by server" );
 517         return false;
 518     }
 519     if (i_authScheme == rfbVncAuth)
 520     {
 521         unsigned char challenge[CHALLENGESIZE];
 522         if ( !read_exact( p_filter, p_sys->i_socket,
 523                           (char*)challenge, CHALLENGESIZE ) )
 524         {
 525             msg_Err( p_filter, "Could not read password challenge" );
 526             return false;
 527         }
 528
 529         vnc_encrypt_bytes( challenge, p_sys->psz_passwd );
 530
 531         if( !write_exact(p_filter, p_sys->i_socket,
 532                          (char*)challenge, CHALLENGESIZE ) )
 533         {
 534             msg_Err( p_filter, "Could not write password" );
 535             return false;
 536         }
 537         uint32_t i_authResult;
 538         if( !read_exact( p_filter, p_sys->i_socket, (char*)&i_authResult, 4 ) )
 539         {
 540             msg_Err( p_filter, "Could not read authentication result" );
 541             return false;
 542         }
 543         i_authResult = htonl(i_authResult);
 544         if (i_authResult != rfbVncAuthOK)
 545         {
 546             msg_Err( p_filter, "VNC authentication failed" );
 547             return false;
 548         }
 549     }
 550
 551     msg_Dbg( p_filter, "Writing client init message" );
 552     rfbClientInitMsg ci;
 553     ci.shared = 1;
 554     if( !write_exact( p_filter, p_sys->i_socket,
 555                       (char*)&ci, sz_rfbClientInitMsg ) )
 556     {
 557         msg_Err( p_filter, "Could not write client init message" );
 558         return false;
 559     }
 560
 561     msg_Dbg( p_filter, "Reading server init message" );
 562     rfbServerInitMsg si;
 563     if( !read_exact( p_filter, p_sys->i_socket,
 564                      (char*)&si, sz_rfbServerInitMsg ) )
 565     {
 566         msg_Err( p_filter, "Could not read server init message" );
 567         return false;
 568     }
 569     si.framebufferWidth = htons(si.framebufferWidth);
 570     si.framebufferHeight = htons(si.framebufferHeight);
 571     si.format.redMax = htons(si.format.redMax);
 572     si.format.greenMax = htons(si.format.greenMax);
 573     si.format.blueMax = htons(si.format.blueMax);
 574
 575     p_sys->i_vnc_width = si.framebufferWidth;
 576     p_sys->i_vnc_height = si.framebufferHeight;
 577
 578     msg_Dbg( p_filter, "Servers preferred pixelformat: "
 579                         "%ux%u, R(%u),G(%u),B(%u), %u bit, depht=%u, %s",
 580                         si.framebufferWidth,
 581                         si.framebufferHeight,
 582                         si.format.redMax,
 583                         si.format.greenMax,
 584                         si.format.blueMax,
 585                         si.format.bitsPerPixel,
 586                         si.format.depth,
 587                         si.format.trueColour ? "TrueColor" : "Not-TrueColor");
 588
 589     uint32_t i_nameLength = htonl(si.nameLength);
 590     if( i_nameLength > MAX_VNC_SERVER_NAME_LENGTH )
 591     {
 592         msg_Err( p_filter, "Server name too long" );
 593         return false;
 594     }
 595     char s_ServerName[MAX_VNC_SERVER_NAME_LENGTH+1];
 596
 597     msg_Dbg( p_filter, "Reading server name with size = %u", i_nameLength );
 598     if( !read_exact( p_filter, p_sys->i_socket, s_ServerName, i_nameLength ) )
 599     {
 600         msg_Err( p_filter, "Could not read server name" );
 601         return false;
 602     }
 603     s_ServerName[i_nameLength] = '\0';
 604
 605     if( strcmp( s_ServerName, "VDR-OSD") == 0 )
 606     {
 607         msg_Dbg( p_filter, "Server is a VDR" );
 608         p_sys->b_alpha_from_vnc = true;
 609     }
 610     else
 611     {
 612         msg_Dbg( p_filter, "Server is a normal VNC" );
 613         p_sys->b_alpha_from_vnc = false;
 614     }
 615
 616
 617     msg_Dbg( p_filter, "Server init message read properly" );
 618     msg_Dbg( p_filter, "Server name is %s", s_ServerName );
 619
 620     msg_Dbg( p_filter, "Writing SetPixelFormat message" );
 621
 622     rfbSetPixelFormatMsg sp;
 623     sp.type = rfbSetPixelFormat;
 624     sp.format.bitsPerPixel = 8;
 625     sp.format.depth = 8 ;
 626     sp.format.bigEndian = 1;
 627     sp.format.trueColour = 0;
 628     sp.format.redMax = htons(31);
 629     sp.format.greenMax = htons(31);
 630     sp.format.blueMax = htons(31);
 631     sp.format.redShift = 10;
 632     sp.format.greenShift = 5;
 633     sp.format.blueShift = 0;
 634
 635     if( !write_exact( p_filter, p_sys->i_socket,
 636                       (char*)&sp, sz_rfbSetPixelFormatMsg) )
 637     {
 638         msg_Err( p_filter, "Could not write SetPixelFormat message" );
 639         return false;
 640     }
 641
 642     msg_Dbg( p_filter, "Writing SetEncodings message" );
 643
 644     rfbSetEncodingsMsg se;
 645     se.type = rfbSetEncodings;
 646     se.nEncodings = htons( p_sys->b_alpha_from_vnc ? 3 : 2 );
 647
 648     if( !write_exact( p_filter, p_sys->i_socket,
 649                       (char*)&se, sz_rfbSetEncodingsMsg) )
 650     {
 651         msg_Err( p_filter, "Could not write SetEncodings message begin" );
 652         return false;
 653     }
 654
 655     uint32_t i_encoding;
 656
 657     msg_Dbg( p_filter, "Writing SetEncodings rfbEncodingCopyRect" );
 658     i_encoding = htonl(rfbEncodingCopyRect);
 659     if( !write_exact( p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
 660     {
 661         msg_Err( p_filter, "Could not write encoding type rfbEncodingCopyRect." );
 662         return false;
 663     }
 664
 665     msg_Dbg( p_filter, "Writing SetEncodings rfbEncodingRRE" );
 666     i_encoding = htonl(rfbEncodingRRE);
 667     if( !write_exact(p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
 668     {
 669         msg_Err( p_filter, "Could not write encoding type rfbEncodingRRE." );
 670         return false;
 671     }
 672
 673     if( p_sys->b_alpha_from_vnc )
 674     {
 675         msg_Dbg( p_filter, "Writing SetEncodings rfbEncSpecialUseAlpha" );
 676         i_encoding = 0x00F0FFFF; /* rfbEncSpecialUseAlpha is 0xFFFFF000
 677                                   * before we swap it */
 678         if( !write_exact(p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
 679         {
 680             msg_Err( p_filter, "Could not write encoding type rfbEncSpecialUseAlpha." );
 681             return false;
 682         }
 683     }
 684     return true;
 685
 686 }
 687
 688 static void vnc_worker_thread( vlc_object_t *p_thread_obj )
 689 {
 690     filter_t* p_filter = (filter_t*)(p_thread_obj->p_parent);
 691     filter_sys_t *p_sys = p_filter->p_sys;
 692
 693     msg_Dbg( p_filter, "VNC worker thread started" );
 694
 695     if ( open_vnc_connection ( p_filter ) == false )
 696     {
 697         msg_Err( p_filter, "Could not connect to vnc host" );
 698         return;
 699     }
 700
 701     if ( handshaking ( p_filter ) == false )
 702     {
 703         msg_Err( p_filter, "Error occured while handshaking vnc host" );
 704         return;
 705     }
 706
 707     p_sys->b_connection_active = true; /* to enable sending key
 708                                             * and mouse events to host */
 709
 710     /* Create an empty picture for VNC the data */
 711     vlc_mutex_lock( &p_sys->lock );
 712     p_sys->p_pic = malloc( sizeof(picture_t) );
 713     if( !p_sys->p_pic )
 714     {
 715         vlc_mutex_unlock( &p_sys->lock );
 716         return;
 717     }
 718     vout_AllocatePicture( VLC_OBJECT(p_filter), p_sys->p_pic,
 719                           VLC_FOURCC('Y','U','V','A'),
 720                           p_sys->i_vnc_width,
 721                           p_sys->i_vnc_height,
 722                           VOUT_ASPECT_FACTOR );
 723     if( !p_sys->p_pic->i_planes )
 724     {
 725         free( p_sys->p_pic );
 726         p_sys->p_pic = NULL;
 727         vlc_mutex_unlock( &p_sys->lock );
 728         return;
 729     }
 730         p_sys->i_vnc_pixels = p_sys->i_vnc_width * p_sys->i_vnc_height;
 731
 732     vlc_mutex_unlock( &p_sys->lock );
 733
 734     /* create the update request thread */
 735     p_sys->p_update_request_thread = vlc_object_create( p_filter,
 736                                                         VLC_OBJECT_GENERIC );
 737     vlc_object_attach( p_sys->p_update_request_thread, p_filter );
 738     if( vlc_thread_create( p_sys->p_update_request_thread,
 739                            "vnc update request thread",
 740                            update_request_thread,
 741                            VLC_THREAD_PRIORITY_LOW, false ) )
 742     {
 743         vlc_object_detach( p_sys->p_update_request_thread );
 744         vlc_object_release( p_sys->p_update_request_thread );
 745         p_sys->p_update_request_thread = NULL;
 746         msg_Err( p_filter, "cannot spawn vnc update request thread" );
 747         return;
 748     }
 749
 750     /* connection is initialized, now read and handle server messages */
 751
 752     int i_msgSize;
 753
 754     rfbServerToClientMsg msg;
 755
 756     while (p_sys->b_continue)
 757     {
 758        msg.type = 0;
 759        if ( !read_exact(p_filter, p_sys->i_socket, (char*)&msg, 1 ) )
 760        {
 761            msg_Err( p_filter, "Error while waiting for next server message");
 762            p_sys->b_continue = false;
 763        }
 764        else
 765        {
 766            switch (msg.type)
 767            {
 768               case rfbFramebufferUpdate:
 769                 i_msgSize = sz_rfbFramebufferUpdateMsg;
 770                 break;
 771               case rfbSetColourMapEntries:
 772                 i_msgSize = sz_rfbSetColourMapEntriesMsg;
 773                 break;
 774               case rfbBell:
 775                 i_msgSize = sz_rfbBellMsg;
 776                 break;
 777               case rfbServerCutText:
 778                 i_msgSize = sz_rfbServerCutTextMsg;
 779                 break;
 780               case rfbReSizeFrameBuffer:
 781                 i_msgSize = sz_rfbReSizeFrameBufferMsg;
 782                 break;
 783               default:
 784                 i_msgSize = 0;
 785                 msg_Err( p_filter, "Invalid message %u received", msg.type );
 786                 p_sys->b_continue = false;
 787            }
 788            if (p_sys->b_continue == true)
 789            {
 790                if (--i_msgSize > 0)
 791                {
 792                    if ( !read_exact( p_filter, p_sys->i_socket,
 793                                      ((char*)&msg)+1, i_msgSize ) )
 794                    {
 795                        msg_Err( p_filter, "Error while reading message of type %u",
 796                                 msg.type );
 797                        p_sys->b_continue = false;
 798                    }
 799                    else
 800                    {
 801                        process_server_message( p_filter, &msg);
 802                    }
 803                }
 804                else
 805                {
 806                    process_server_message( p_filter, &msg);
 807                }
 808            }
 809
 810        }
 811
 812        if (p_sys->p_worker_thread->b_die ||
 813            p_sys->p_worker_thread->b_error)
 814        {
 815            p_sys->b_continue = false;
 816        }
 817
 818     }
 819
 820     msg_Dbg( p_filter, "VNC message reader thread ended" );
 821
 822 }
 823
 824 static void update_request_thread( vlc_object_t *p_thread_obj )
 825 {
 826     filter_t* p_filter = (filter_t*)(p_thread_obj->p_parent);
 827     filter_sys_t *p_sys = p_filter->p_sys;
 828
 829     msg_Dbg( p_filter, "VNC update request thread started" );
 830
 831     rfbFramebufferUpdateRequestMsg udr;
 832     udr.type = rfbFramebufferUpdateRequest;
 833     udr.incremental = 0;
 834     udr.x = 0;
 835     udr.y = 0;
 836     udr.w = htons(p_sys->i_vnc_width);
 837     udr.h = htons(p_sys->i_vnc_height);
 838
 839     if( write_exact(p_filter, p_sys->i_socket, (char*)&udr,
 840            sz_rfbFramebufferUpdateRequestMsg) == false)
 841     {
 842         msg_Err( p_filter, "Could not write rfbFramebufferUpdateRequestMsg." );
 843         p_sys->b_continue = false;
 844     }
 845
 846     udr.incremental = 1;
 847     mtime_t i_poll_interval_microsec = p_sys->i_vnc_poll_interval * 1000;
 848
 849     if (p_sys->b_vnc_poll)
 850     {
 851         while ( p_sys->b_continue == true )
 852         {
 853             msleep( i_poll_interval_microsec );
 854             if( write_exact(p_filter, p_sys->i_socket, (char*)&udr,
 855                    sz_rfbFramebufferUpdateRequestMsg) == false)
 856             {
 857                 msg_Err( p_filter, "Could not write rfbFramebufferUpdateRequestMsg." );
 858                 p_sys->b_continue = false;
 859             }
 860             if (p_sys->p_update_request_thread->b_die ||
 861                 p_sys->p_update_request_thread->b_error)
 862             {
 863                 p_sys->b_continue = false;
 864             }
 865         }
 866     }
 867     else
 868     {
 869         msg_Dbg( p_filter, "VNC polling disabled." );
 870     }
 871     msg_Dbg( p_filter, "VNC update request thread ended" );
 872 }
 873
 874 static bool process_server_message ( filter_t *p_filter,
 875                                      rfbServerToClientMsg *msg )
 876 {
 877     filter_sys_t *p_sys = p_filter->p_sys;
 878
 879     switch (msg->type)
 880     {
 881     case rfbFramebufferUpdate:
 882         {
 883             msg->fu.nRects = htons(msg->fu.nRects);
 884             rfbFramebufferUpdateRectHeader hdr;
 885
 886             for (int i_rect = 0; i_rect < msg->fu.nRects; i_rect++)
 887             {
 888                 if (!read_exact(p_filter, p_sys->i_socket, (char*)&hdr,
 889                     sz_rfbFramebufferUpdateRectHeader ) )
 890                 {
 891                     msg_Err( p_filter, "Could not read FrameBufferUpdate header" );
 892                     return false;
 893                 }
 894                 hdr.r.x = htons(hdr.r.x);
 895                 hdr.r.y = htons(hdr.r.y);
 896                 hdr.r.w = htons(hdr.r.w);
 897                 hdr.r.h = htons(hdr.r.h);
 898                 hdr.encoding = htonl(hdr.encoding);
 899
 900                 switch (hdr.encoding)
 901                 {
 902                 case rfbEncodingRaw:
 903                     {
 904                         int i_line;
 905                         for (i_line = 0; i_line < hdr.r.h; i_line++)
 906                         {
 907                             if ( !read_exact( p_filter, p_sys->i_socket,
 908                                     p_sys->read_buffer, hdr.r.w ) )
 909                             {
 910                                 msg_Err( p_filter,
 911                                  "Could not read FrameBufferUpdate line data" );
 912                                return false;
 913                             }
 914                             vlc_mutex_lock( &p_sys->lock );
 915                             if ( !raw_line( p_sys, hdr.r.x,
 916                                             hdr.r.y + i_line,
 917                                             hdr.r.w ) )
 918                             {
 919                                 msg_Err( p_filter, "raw_line failed." );
 920                                 vlc_mutex_unlock( &p_sys->lock );
 921                                 return false;
 922                             }
 923                             vlc_mutex_unlock( &p_sys->lock );
 924                         }
 925                     }
 926                     break;
 927
 928                 case rfbEncodingCopyRect:
 929                     {
 930                         msg_Err( p_filter,
 931                           "Rect in unsupported encoding rfbEncodingCopyRect" );
 932                         return false;
 933                     }
 934
 935                 case rfbEncodingRRE:
 936                     {
 937                         rfbRREHeader rrehdr;
 938                         if ( !read_exact( p_filter, p_sys->i_socket,
 939                                           (char*)&rrehdr,
 940                                           sz_rfbRREHeader ) )
 941                         {
 942                             msg_Err( p_filter, "Could not read rfbRREHeader" );
 943                             return false;
 944                         }
 945                         uint8_t i_pixcolor;
 946                         if ( !read_exact(p_filter, p_sys->i_socket,
 947                                          (char*)&i_pixcolor, 1 ) )
 948                         {
 949                             msg_Err( p_filter, "Could not read RRE pixcolor" );
 950                             return false;
 951                         }
 952
 953                         vlc_mutex_lock( &p_sys->lock );
 954                         if ( !fill_rect( p_sys,
 955                                         hdr.r.x, hdr.r.y,
 956                                         hdr.r.w, hdr.r.h,
 957                                         i_pixcolor) )
 958                         {
 959                             msg_Err( p_filter, "main fill_rect failed." );
 960                             vlc_mutex_unlock( &p_sys->lock );
 961                             return false;
 962                         }
 963                         vlc_mutex_unlock( &p_sys->lock );
 964
 965                         rrehdr.nSubrects = htonl(rrehdr.nSubrects);
 966
 967                         int i_datasize = rrehdr.nSubrects *
 968                                      ( sizeof(i_pixcolor) + sz_rfbRectangle ) ;
 969                         if ( i_datasize > READ_BUFFER_SIZE )
 970                         {
 971                             msg_Err( p_filter, "Buffer too small, "
 972                                      "need %u bytes", i_datasize );
 973                             return false;
 974                         }
 975                         if ( !read_exact( p_filter, p_sys->i_socket,
 976                                        p_sys->read_buffer, i_datasize ) )
 977                         {
 978                             msg_Err( p_filter,
 979                                      "Could not read RRE subrect data" );
 980                             return false;
 981                         }
 982
 983                         uint32_t i_subrect;
 984                         rfbRectangle* p_subrect;
 985                         int i_offset = 0;
 986                         vlc_mutex_lock( &p_sys->lock );
 987                         for ( i_subrect = 0;
 988                               i_subrect < rrehdr.nSubrects; i_subrect++)
 989                         {
 990                             i_pixcolor = p_sys->read_buffer[i_offset];
 991                             i_offset += sizeof(i_pixcolor);
 992                             p_subrect =
 993                                (rfbRectangle*)(p_sys->read_buffer + i_offset);
 994                             i_offset += sz_rfbRectangle;
 995
 996                             if (!fill_rect( p_sys,
 997                                             htons(p_subrect->x) + hdr.r.x,
 998                                             htons(p_subrect->y) + hdr.r.y,
 999                                             htons(p_subrect->w),
1000                                             htons(p_subrect->h),
1001                                             i_pixcolor) )
1002                             {
1003                                 msg_Err( p_filter,
1004                                     "subrect %u fill_rect failed.", i_subrect );
1005                                 vlc_mutex_unlock( &p_sys->lock );
1006                                 return false;
1007                             }
1008                         }
1009                         vlc_mutex_unlock( &p_sys->lock );
1010                     }
1011                     break;
1012
1013                 }
1014
1015             }
1016             vlc_mutex_lock( &p_sys->lock );
1017             p_sys->b_need_update = true;
1018             vlc_mutex_unlock( &p_sys->lock );
1019         }
1020         return true;
1021
1022     case rfbSetColourMapEntries:
1023         {
1024             msg->scme.nColours = htons(msg->scme.nColours);
1025             msg->scme.firstColour = htons(msg->scme.firstColour);
1026             int i_datasize;
1027             if ( p_sys->b_alpha_from_vnc == true )
1028             {
1029                 i_datasize = 2 * msg->scme.nColours * 4;
1030             }
1031             else
1032             {
1033                 i_datasize = 2 * msg->scme.nColours * 3;
1034             }
1035             if ( i_datasize > READ_BUFFER_SIZE )
1036             {
1037                 msg_Err( p_filter, "Buffer too small, need %u bytes",
1038                                    i_datasize );
1039                 return false;
1040             }
1041
1042             if ( !read_exact( p_filter, p_sys->i_socket,
1043                               p_sys->read_buffer, i_datasize ) )
1044             {
1045                 msg_Err( p_filter, "Could not read color map data" );
1046                 return false;
1047             }
1048
1049             uint8_t i_red, i_green, i_blue, i_alpha, i_color_index;
1050             uint16_t i_offset = 0;
1051             i_alpha = 255;
1052
1053             for (int i = 0; i < msg->scme.nColours; i++)
1054             {
1055                 i_color_index = i+msg->scme.firstColour;
1056                 if ( p_sys->b_alpha_from_vnc == true )
1057                 {
1058                     i_alpha = p_sys->read_buffer[i_offset];
1059                     i_offset += 2;
1060                 }
1061                 i_red   = p_sys->read_buffer[i_offset];
1062                 i_offset += 2;
1063                 i_green = p_sys->read_buffer[i_offset];
1064                 i_offset += 2;
1065                 i_blue  = p_sys->read_buffer[i_offset];
1066                 i_offset += 2;
1067                 rgb_to_yuv( &p_sys->ar_color_table_yuv[i_color_index][0],
1068                             &p_sys->ar_color_table_yuv[i_color_index][1],
1069                             &p_sys->ar_color_table_yuv[i_color_index][2],
1070                             i_red,
1071                             i_green,
1072                             i_blue );
1073                 p_sys->ar_color_table_yuv[i][3] = i_alpha;
1074             }
1075         }
1076         return true;
1077
1078     case rfbBell:
1079         msg_Err( p_filter, "rfbBell received" );
1080         return true;
1081
1082     case rfbServerCutText:
1083         msg->sct.length = htons(msg->sct.length);
1084         if ( msg->sct.length > READ_BUFFER_SIZE )
1085         {
1086             msg_Err( p_filter, "Buffer too small, need %u bytes", msg->sct.length );
1087             return false;
1088         }
1089         if ( !read_exact(p_filter, p_sys->i_socket,
1090                          p_sys->read_buffer, msg->sct.length ) )
1091         {
1092             msg_Err( p_filter, "Could not read Reading rfbServerCutText data" );
1093             return false;
1094         }
1095         return true;
1096
1097     case rfbReSizeFrameBuffer:
1098         msg_Err( p_filter, "Reading rfbReSizeFrameBuffer not implemented" );
1099         return false;
1100
1101     default:
1102         msg_Err( p_filter, "Invalid message %u received", msg->type );
1103         return false;
1104     }
1105     return false;
1106 }
1107
1108 /****************************************************************************
1109  * Filter: the whole thing
1110  ****************************************************************************
1111  * This function outputs subpictures at regular time intervals.
1112  ****************************************************************************/
1113 static subpicture_t *Filter( filter_t *p_filter, mtime_t date )
1114 {
1115     filter_sys_t *p_sys = p_filter->p_sys;
1116     subpicture_t *p_spu;
1117     subpicture_region_t *p_region;
1118     video_format_t fmt;
1119     picture_t *p_pic;
1120
1121     if( !p_sys->b_need_update )
1122     {
1123         return NULL;
1124     }
1125
1126     vlc_mutex_lock( &p_sys->lock );
1127
1128     p_pic = p_sys->p_pic;
1129
1130     if( !p_pic )
1131     {
1132         vlc_mutex_unlock( &p_sys->lock );
1133         return NULL;
1134     }
1135
1136     /* Allocate the subpicture internal data. */
1137     p_spu = p_filter->pf_sub_buffer_new( p_filter );
1138     if( !p_spu )
1139     {
1140         vlc_mutex_unlock( &p_sys->lock );
1141         return NULL;
1142     }
1143
1144     p_spu->b_absolute = false;
1145     p_spu->i_start = date;
1146     p_spu->i_stop = 0;
1147     p_spu->b_ephemer = true;
1148
1149     /* Create new SPU region */
1150     memset( &fmt, 0, sizeof(video_format_t) );
1151     fmt.i_chroma = VLC_FOURCC('Y','U','V','A');
1152     fmt.i_aspect = VOUT_ASPECT_FACTOR;
1153     fmt.i_sar_num = fmt.i_sar_den = 1;
1154     fmt.i_width = fmt.i_visible_width = p_pic->p[Y_PLANE].i_visible_pitch;
1155     fmt.i_height = fmt.i_visible_height = p_pic->p[Y_PLANE].i_visible_lines;
1156     fmt.i_x_offset = fmt.i_y_offset = 0;
1157     p_region = p_spu->pf_create_region( VLC_OBJECT(p_filter), &fmt );
1158     if( !p_region )
1159     {
1160         msg_Err( p_filter, "cannot allocate SPU region" );
1161         p_filter->pf_sub_buffer_del( p_filter, p_spu );
1162         vlc_mutex_unlock( &p_sys->lock );
1163         return NULL;
1164     }
1165
1166     vout_CopyPicture( p_filter, &p_region->picture, p_pic );
1167
1168     p_sys->b_need_update = false;
1169
1170     vlc_mutex_unlock( &p_sys->lock );
1171
1172     /* set to one of the 9 relative locations */
1173     p_region->i_align = 0; //=CENTER
1174     p_spu->b_absolute = false;
1175
1176
1177     p_spu->i_x = 0;
1178     p_spu->i_y = 0;
1179
1180     p_spu->p_region = p_region;
1181
1182     p_spu->i_alpha = ( p_sys->i_alpha );
1183
1184     return p_spu;
1185 }
1186
1187
1188 static inline void rgb_to_yuv( uint8_t *y, uint8_t *u, uint8_t *v,
1189                                int r, int g, int b )
1190 {
1191     *y = ( ( (  66 * r + 129 * g +  25 * b + 128 ) >> 8 ) + 16 );
1192     *u =   ( ( -38 * r -  74 * g + 112 * b + 128 ) >> 8 ) + 128 ;
1193     *v =   ( ( 112 * r -  94 * g -  18 * b + 128 ) >> 8 ) + 128 ;
1194 }
1195
1196 static inline bool fill_rect( filter_sys_t* p_sys,
1197                               uint16_t i_x, uint16_t i_y,
1198                               uint16_t i_w, uint16_t i_h,
1199                               uint8_t i_color)
1200 {
1201     plane_t *p_outY = p_sys->p_pic->p+Y_PLANE;
1202     plane_t *p_outU = p_sys->p_pic->p+U_PLANE;
1203     plane_t *p_outV = p_sys->p_pic->p+V_PLANE;
1204     plane_t *p_outA = p_sys->p_pic->p+A_PLANE;
1205     int i_pitch = p_outY->i_pitch;
1206     int i_lines = p_outY->i_lines;
1207     if ( i_x + i_w > i_pitch)
1208         return false;
1209     if ( i_y + i_h > i_lines)
1210         return false;
1211     int i_line_offset = i_y * i_pitch;
1212     uint8_t i_yuv_y = p_sys->ar_color_table_yuv[i_color][0];
1213     uint8_t i_yuv_u = p_sys->ar_color_table_yuv[i_color][1];
1214     uint8_t i_yuv_v = p_sys->ar_color_table_yuv[i_color][2];
1215     uint8_t i_alpha = p_sys->ar_color_table_yuv[i_color][3];
1216     for( int i_line = 0; i_line < i_h; i_line++ )
1217     {
1218         for( int i_column = 0; i_column < i_w; i_column++ )
1219         {
1220             int i_total_offset = i_line_offset + i_x + i_column;
1221             p_outY->p_pixels[ i_total_offset ] = i_yuv_y;
1222             p_outU->p_pixels[ i_total_offset ] = i_yuv_u;
1223             p_outV->p_pixels[ i_total_offset ] = i_yuv_v;
1224             p_outA->p_pixels[ i_total_offset ] = i_alpha;
1225         }
1226         i_line_offset += i_pitch;
1227     }
1228     return true;
1229 }
1230
1231 static inline bool raw_line(  filter_sys_t* p_sys,
1232                               uint16_t i_x, uint16_t i_y,
1233                               uint16_t i_w )
1234 {
1235     plane_t *p_outY = p_sys->p_pic->p+Y_PLANE;
1236     plane_t *p_outU = p_sys->p_pic->p+U_PLANE;
1237     plane_t *p_outV = p_sys->p_pic->p+V_PLANE;
1238     plane_t *p_outA = p_sys->p_pic->p+A_PLANE;
1239     int i_pitch = p_outY->i_pitch;
1240     int i_lines = p_outY->i_lines;
1241     if ( i_x + i_w > i_pitch)
1242         return false;
1243     if ( i_y > i_lines)
1244         return false;
1245
1246     int i_line_offset = i_y * i_pitch + i_x;
1247
1248     for( int i_column = 0; i_column < i_w; i_column++ )
1249     {
1250         int i_offset = i_line_offset + i_column;
1251         uint8_t i_color = p_sys->read_buffer[i_column];
1252         p_outY->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][0];
1253         p_outU->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][1];
1254         p_outV->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][2];
1255         p_outA->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][3];
1256     }
1257
1258     return true;
1259 }
1260
1261
1262 /*****************************************************************************
1263  * MouseEvent: callback for mouse events
1264  *****************************************************************************/
1265 static int MouseEvent( vlc_object_t *p_this, char const *psz_var,
1266                        vlc_value_t oldval, vlc_value_t newval, void *p_data )
1267 {
1268     VLC_UNUSED(oldval); VLC_UNUSED(newval); VLC_UNUSED(psz_var);
1269
1270     filter_t *p_filter = (filter_t *)p_data;
1271     filter_sys_t *p_sys = p_filter->p_sys;
1272
1273     if( !p_sys->b_connection_active )
1274         return VLC_SUCCESS;
1275
1276     if( !p_sys->b_vnc_mouse_events )
1277         return VLC_SUCCESS;
1278
1279     vout_thread_t *p_vout = (vout_thread_t*)p_sys->p_vout;
1280     int i_x, i_y;
1281     int i_v;
1282
1283     int v_h = p_vout->output.i_height;
1284     int v_w = p_vout->output.i_width;
1285
1286     i_v = var_GetInteger( p_sys->p_vout, "mouse-button-down" );
1287     i_y = var_GetInteger( p_sys->p_vout, "mouse-y" );
1288     i_x = var_GetInteger( p_sys->p_vout, "mouse-x" );
1289
1290     if( i_y < 0 || i_x < 0 || i_y >= v_h || i_x >= v_w )
1291     {
1292        msg_Dbg( p_this, "invalid mouse event? x=%d y=%d btn=%x", i_x, i_y, i_v );
1293        return VLC_SUCCESS;
1294     }
1295 #ifdef VNC_DEBUG
1296     msg_Dbg( p_this, "mouse event x=%d y=%d btn=%x", i_x, i_y, i_v );
1297 #endif
1298
1299     /* FIXME: calculate x and y coordinates for scaled output */
1300
1301     /* buttonMask bits 0-7 are buttons 1-8, 0=up, 1=down */
1302     rfbPointerEventMsg ev;
1303     ev.type = rfbPointerEvent;
1304     ev.buttonMask = i_v;
1305     ev.x = htons(i_x);
1306     ev.y = htons(i_y);
1307
1308     write_exact( p_filter, p_sys->i_socket,
1309                  (char*)&ev, sz_rfbPointerEventMsg);
1310
1311     return VLC_SUCCESS;
1312 }
1313
1314 /*****************************************************************************
1315  * KeyEvent: callback for keyboard events
1316  *****************************************************************************/
1317 static int KeyEvent( vlc_object_t *p_this, char const *psz_var,
1318                      vlc_value_t oldval, vlc_value_t newval, void *p_data )
1319 {
1320     VLC_UNUSED(psz_var); VLC_UNUSED(oldval);
1321
1322     filter_t *p_filter = (filter_t *)p_data;
1323     filter_sys_t *p_sys = p_filter->p_sys;
1324
1325     if( !p_sys->b_connection_active )
1326         return VLC_SUCCESS;
1327
1328     if( !p_sys->b_vnc_key_events )
1329         return VLC_SUCCESS;
1330
1331     msg_Dbg( p_this, "key pressed (%d) ", newval.i_int );
1332
1333     if ( !newval.i_int )
1334     {
1335         msg_Err( p_this, "Received invalid key event %d", newval.i_int );
1336         return VLC_EGENERIC;
1337     }
1338
1339     uint32_t i_key32 = newval.i_int;
1340     i_key32 = htonl(i_key32);
1341     rfbKeyEventMsg ev;
1342     ev.type = rfbKeyEvent;
1343     ev.down = 1;
1344
1345     /* first key-down for modifier-keys */
1346     if (newval.i_int & KEY_MODIFIER_CTRL)
1347     {
1348         ev.key = 0xffe3;
1349         write_exact( p_filter, p_sys->i_socket,
1350                      (char*)&ev, sz_rfbKeyEventMsg);
1351     }
1352     if (newval.i_int & KEY_MODIFIER_SHIFT)
1353     {
1354         ev.key = 0xffe1;
1355         write_exact( p_filter, p_sys->i_socket,
1356                      (char*)&ev, sz_rfbKeyEventMsg);
1357     }
1358     if (newval.i_int & KEY_MODIFIER_ALT)
1359     {
1360         ev.key = 0xffe9;
1361         write_exact( p_filter, p_sys->i_socket,
1362                      (char*)&ev, sz_rfbKeyEventMsg);
1363     }
1364
1365     /* then key-down for the pressed key */
1366     ev.key = i_key32;
1367     write_exact( p_filter, p_sys->i_socket,
1368                  (char*)&ev, sz_rfbKeyEventMsg);
1369
1370     ev.down = 0;
1371
1372     /* then key-up for the pressed key */
1373     write_exact( p_filter, p_sys->i_socket,
1374                  (char*)&ev, sz_rfbKeyEventMsg);
1375
1376     /* last key-down for modifier-keys */
1377     if (newval.i_int & KEY_MODIFIER_CTRL)
1378     {
1379         ev.key = 0xffe3;
1380         write_exact( p_filter, p_sys->i_socket,
1381                      (char*)&ev, sz_rfbKeyEventMsg);
1382     }
1383     if (newval.i_int & KEY_MODIFIER_SHIFT)
1384     {
1385         ev.key = 0xffe1;
1386         write_exact( p_filter, p_sys->i_socket,
1387                      (char*)&ev, sz_rfbKeyEventMsg);
1388     }
1389     if (newval.i_int & KEY_MODIFIER_ALT)
1390     {
1391        ev.key = 0xffe9;
1392        write_exact( p_filter, p_sys->i_socket,
1393                     (char*)&ev, sz_rfbKeyEventMsg);
1394     }
1395
1396     return VLC_SUCCESS;
1397 }
1398
1399 static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd )
1400 {
1401     unsigned char key[8];
1402     unsigned int i;
1403
1404     for (i = 0; i < 8; i++)
1405         key[i] = i < strlen( passwd ) ? passwd[i] : '\0';
1406
1407     gcry_cipher_hd_t ctx;
1408     gcry_cipher_open( &ctx, GCRY_CIPHER_DES, GCRY_CIPHER_MODE_ECB,0);
1409
1410     /* reverse bits of the key */
1411     for( i = 0 ; i < 8 ; i ++ )
1412         key[i] =
1413             (key[i] >> 7) +
1414             (((key[i] >> 6) & 0x01 ) << 1 ) +
1415             (((key[i] >> 5) & 0x01 ) << 2 ) +
1416             (((key[i] >> 4) & 0x01 ) << 3 ) +
1417             (((key[i] >> 3) & 0x01 ) << 4 ) +
1418             (((key[i] >> 2) & 0x01 ) << 5 ) +
1419             (((key[i] >> 1) & 0x01 ) << 6 ) +
1420             ((key[i] & 0x01) << 7 );
1421
1422     gcry_cipher_setkey( ctx, key, 8 );
1423     gcry_cipher_encrypt( ctx, bytes, CHALLENGESIZE, bytes, CHALLENGESIZE );
1424     gcry_cipher_close( ctx );
1425 }