git.sesse.net Git - vlc/blob - modules/video_filter/remoteosd.c

   1 /*****************************************************************************
   2  * remoteosd.c: remote osd over vnc filter module
   3  *****************************************************************************
   4  * Copyright (C) 2007-2008 Matthias Bauer
   5  * $Id$
   6  *
   7  * Authors: Matthias Bauer <matthias dot bauer #_at_# gmx dot ch>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implid warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  22  *****************************************************************************/
  23
  24 /*****************************************************************************
  25  * RemoteOSD uses the RFB-Protocol of VNC to display an On-Screen-Display
  26  * menu generated by a streaming server as overlay for the streamed video.
  27  *
  28  * The streaming server that implements this is the ffnetdev plugin for VDR.
  29  * VDR (VideoDiskRecorder) is an Linux based OpenSource harddisk recorder
  30  * software.
  31  * The VDR ffnetdev plugin emulates the hardware MPEG decoder and streams the
  32  * video over the network instead of hardware video outputs.
  33  * The OSD menu of VDR is offered with the RFB protocol to a VNC client.
  34  *
  35  * In fact this video-filter is a simple VNC client that could be also used to
  36  * connect to a real VNC host.
  37  * Only 8-bit color is supported at the moment.
  38  * Using password protected VNC hosts is supported but not recommended, because
  39  * you need to insert the used password in the plugin configuration page of
  40  * VLC configuration in plain text and it's saved in plain text.
  41  *****************************************************************************/
  42
  43 //#define VNC_DEBUG
  44
  45 /*****************************************************************************
  46  * Preamble
  47  *****************************************************************************/
  48
  49 #ifdef HAVE_CONFIG_H
  50 # include "config.h"
  51 #endif
  52
  53 #include <vlc_common.h>
  54 #include <vlc_plugin.h>
  55 #include <vlc_vout.h>
  56
  57 #include "vlc_filter.h"
  58 #include "filter_common.h"
  59 #include "vlc_image.h"
  60 #include "vlc_osd.h"
  61 #include "vlc_keys.h"
  62
  63 #include <vlc_network.h>
  64 #include <gcrypt.h>              /* to encrypt password */
  65 #include <vlc_gcrypt.h>
  66
  67 #define CHALLENGESIZE 16
  68 #define MAX_VNC_SERVER_NAME_LENGTH 255
  69
  70 #include "remoteosd_rfbproto.h" /* type definitions of the RFB protocol for VNC */
  71
  72 /*****************************************************************************
  73  * Local prototypes
  74  *****************************************************************************/
  75 /* subfilter functions */
  76 static int  CreateFilter ( vlc_object_t * );
  77 static void DestroyFilter( vlc_object_t * );
  78 static subpicture_t *Filter( filter_t *, mtime_t );
  79
  80 static int MouseEvent ( vlc_object_t *p_this, char const *psz_var,
  81                         vlc_value_t oldval, vlc_value_t newval, void *p_data );
  82
  83 static int KeyEvent( vlc_object_t *p_this, char const *psz_var,
  84                      vlc_value_t oldval, vlc_value_t newval, void *p_data );
  85
  86 static void stop_osdvnc ( filter_t *p_filter );
  87
  88 static void vnc_worker_thread ( vlc_object_t *p_thread_obj );
  89
  90 static void update_request_thread( vlc_object_t *p_thread_obj );
  91
  92 static bool open_vnc_connection ( filter_t *p_filter );
  93
  94 static bool handshaking ( filter_t *p_filter );
  95
  96 static bool process_server_message ( filter_t *p_filter,
  97                                      rfbServerToClientMsg *msg );
  98
  99 static bool read_exact( filter_t *p_filter,
 100                         int i_socket,
 101                         char* p_readbuf,
 102                         int i_bytes );
 103
 104 static bool write_exact( filter_t *p_filter,
 105                          int i_socket,
 106                          char* p_writebuf,
 107                          int i_bytes );
 108
 109 static inline void rgb_to_yuv( uint8_t *y, uint8_t *u, uint8_t *v,
 110                                int r, int g, int b );
 111
 112 static inline bool fill_rect( filter_sys_t* p_sys,
 113                               uint16_t i_x, uint16_t i_y,
 114                               uint16_t i_w, uint16_t i_h,
 115                               uint8_t i_color );
 116
 117 static inline bool raw_line(  filter_sys_t* p_sys,
 118                               uint16_t i_x, uint16_t i_y,
 119                               uint16_t i_w );
 120
 121 static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd );
 122
 123
 124 /*****************************************************************************
 125  * Module descriptor
 126  *****************************************************************************/
 127 #define READ_BUFFER_SIZE 1000000
 128
 129 #define RMTOSD_HOST_TEXT N_("VNC Host")
 130 #define RMTOSD_HOST_LONGTEXT N_( \
 131     "VNC hostname or IP address." )
 132
 133 #define RMTOSD_PORT_TEXT N_("VNC Port")
 134 #define RMTOSD_PORT_LONGTEXT N_( \
 135     "VNC portnumber." )
 136
 137 #define RMTOSD_PASSWORD_TEXT N_("VNC Password")
 138 #define RMTOSD_PASSWORD_LONGTEXT N_( \
 139     "VNC password." )
 140
 141 #define RMTOSD_UPDATE_TEXT N_("VNC poll interval" )
 142 #define RMTOSD_UPDATE_LONGTEXT N_( \
 143     "In this interval an update from VNC is requested, default every 300 ms. ")
 144
 145 #define RMTOSD_POLL_TEXT N_("VNC polling")
 146 #define RMTOSD_POLL_LONGTEXT N_( \
 147     "Activate VNC polling. Do NOT activate for use as VDR ffnetdev client." )
 148
 149 #define RMTOSD_MOUSE_TEXT N_("Mouse events")
 150 #define RMTOSD_MOUSE_LONGTEXT N_( \
 151     "Send mouse events to VNC host. Not needed for use as VDR ffnetdev client." )
 152
 153 #define RMTOSD_KEYS_TEXT N_("Key events")
 154 #define RMTOSD_KEYS_LONGTEXT N_( \
 155     "Send key events to VNC host." )
 156
 157 #define RMTOSD_ALPHA_TEXT N_("Alpha transparency value (default 255)")
 158 #define RMTOSD_ALPHA_LONGTEXT N_( \
 159     "The transparency of the OSD VNC can be changed by giving a value " \
 160     "between 0 and 255. A lower value specifies more transparency a higher " \
 161     "means less transparency. The default is being not transparent " \
 162     "(value 255) the minimum is fully transparent (value 0)." )
 163
 164 #define RMTOSD_CFG "rmtosd-"
 165
 166 #define RMTOSD_UPDATE_MIN     200
 167 #define RMTOSD_UPDATE_DEFAULT 1000
 168 #define RMTOSD_UPDATE_MAX     300
 169
 170
 171 vlc_module_begin();
 172     set_description( N_("Remote-OSD over VNC") );
 173     set_capability( "sub filter", 100 );
 174     set_shortname( N_("Remote-OSD") );
 175     set_category( CAT_VIDEO );
 176     set_subcategory( SUBCAT_VIDEO_SUBPIC );
 177     add_shortcut( "rmtosd" );
 178     set_callbacks( CreateFilter, DestroyFilter );
 179
 180     add_string( RMTOSD_CFG "host", "myvdr", NULL, RMTOSD_HOST_TEXT,
 181         RMTOSD_HOST_LONGTEXT, false );
 182     add_integer_with_range( RMTOSD_CFG "port", 20001, 1, 0xFFFF, NULL,
 183         RMTOSD_PORT_TEXT, RMTOSD_PORT_LONGTEXT, false );
 184     add_password( RMTOSD_CFG "password", "", NULL, RMTOSD_PASSWORD_TEXT,
 185         RMTOSD_PASSWORD_LONGTEXT, false );
 186     add_integer_with_range( RMTOSD_CFG "update", RMTOSD_UPDATE_DEFAULT,
 187         RMTOSD_UPDATE_MIN, RMTOSD_UPDATE_MAX, NULL, RMTOSD_UPDATE_TEXT,
 188         RMTOSD_UPDATE_LONGTEXT, true );
 189     add_bool( RMTOSD_CFG "vnc-polling", 0, NULL,
 190               RMTOSD_POLL_TEXT , RMTOSD_POLL_LONGTEXT, false );
 191     add_bool( RMTOSD_CFG "mouse-events", 0, NULL,
 192               RMTOSD_MOUSE_TEXT , RMTOSD_MOUSE_LONGTEXT, false );
 193     add_bool( RMTOSD_CFG "key-events", 0, NULL,
 194               RMTOSD_KEYS_TEXT , RMTOSD_KEYS_LONGTEXT, false );
 195     add_integer_with_range( RMTOSD_CFG "alpha", 255, 0, 255, NULL,
 196         RMTOSD_ALPHA_TEXT, RMTOSD_ALPHA_LONGTEXT, true );
 197
 198 vlc_module_end();
 199
 200 /*****************************************************************************
 201  * Sub filter code
 202  *****************************************************************************/
 203
 204 /*****************************************************************************
 205  * Local prototypes
 206  *****************************************************************************/
 207 struct filter_sys_t
 208 {
 209     VLC_COMMON_MEMBERS
 210
 211     bool          b_need_update;       /* VNC picture is updated, do update the OSD*/
 212     mtime_t       i_vnc_poll_interval; /* Update the OSD menu every n ms */
 213
 214     uint8_t       i_alpha;             /* alpha transparency value */
 215
 216     char          *psz_host;           /* VNC host */
 217     int           i_port;
 218
 219     char          *psz_passwd;         /* VNC password */
 220
 221     bool          b_vnc_poll;          /* Activate VNC polling ? */
 222     bool          b_vnc_mouse_events;  /* Send MouseEvents ? */
 223     bool          b_vnc_key_events;    /* Send KeyEvents ? */
 224
 225     bool          b_connection_active; /* Handshaking finished ? */
 226
 227     vlc_mutex_t   lock;                /* To lock for read/write on picture */
 228
 229     picture_t     *p_pic;              /* The picture with OSD data from VNC */
 230
 231     vout_thread_t *p_vout;             /* Pointer to video-out thread */
 232
 233     int           i_socket;            /* Socket used for VNC */
 234
 235     uint16_t      i_vnc_width;          /* The with of the VNC screen */
 236     uint16_t      i_vnc_height;         /* The height of the VNC screen */
 237         uint32_t      i_vnc_pixels;         /* The pixels of the VNC screen */
 238
 239     bool    b_alpha_from_vnc;    /* Special ffnetdev alpha feature enabled ? */
 240
 241     char          read_buffer[READ_BUFFER_SIZE];
 242
 243     bool          b_continue;
 244
 245     vlc_object_t* p_worker_thread;
 246     vlc_object_t* p_update_request_thread;
 247
 248     uint8_t       ar_color_table_yuv[256][4];
 249 };
 250
 251 /*****************************************************************************
 252  * CreateFilter: Create the filter and open the definition file
 253  *****************************************************************************/
 254 static int CreateFilter ( vlc_object_t *p_this )
 255 {
 256     filter_t *p_filter = (filter_t *)p_this;
 257     filter_sys_t *p_sys = NULL;
 258
 259     msg_Dbg( p_filter, "Creating vnc osd filter..." );
 260
 261     p_filter->p_sys = p_sys = (filter_sys_t *) malloc( sizeof(filter_sys_t) );
 262     if( !p_filter->p_sys )
 263         return VLC_ENOMEM;
 264     memset( p_sys, 0, sizeof(filter_sys_t) );
 265
 266     /* Populating struct */
 267     vlc_mutex_init( &p_sys->lock );
 268     p_sys->b_continue = true;
 269     p_sys->i_socket = -1;
 270
 271     p_sys->psz_host = var_CreateGetString( p_this, RMTOSD_CFG "host" );
 272     if( EMPTY_STR(p_sys->psz_host) )
 273     {
 274         msg_Err( p_filter, "unable to get vnc host" );
 275         goto error;
 276     }
 277
 278     p_sys->psz_passwd = var_CreateGetString( p_this, RMTOSD_CFG "password" );
 279     if( !p_sys->psz_passwd )
 280     {
 281         msg_Err( p_filter, "unable to get vnc password" );
 282         goto error;
 283     }
 284
 285     p_sys->i_port = var_CreateGetIntegerCommand( p_this, RMTOSD_CFG "port" );
 286
 287     p_sys->i_alpha = var_CreateGetIntegerCommand( p_this, RMTOSD_CFG "alpha" );
 288
 289     /* in miliseconds, 0 disables polling, should not be lower than 100 */
 290     p_sys->i_vnc_poll_interval  = var_CreateGetIntegerCommand( p_this,
 291                                                        RMTOSD_CFG "update" );
 292     if ( p_sys->i_vnc_poll_interval < 100)
 293     {
 294        p_sys->i_vnc_poll_interval = 100;
 295     }
 296
 297     for ( int i = 0; i < 256; i++ )
 298     {
 299         p_sys->ar_color_table_yuv[i][0] = 255;
 300         p_sys->ar_color_table_yuv[i][1] = 255;
 301         p_sys->ar_color_table_yuv[i][2] = 255;
 302         p_sys->ar_color_table_yuv[i][3] = 255;
 303     }
 304
 305     p_sys->b_vnc_poll = var_CreateGetBoolCommand( p_this,
 306                                             RMTOSD_CFG "vnc-polling" );
 307     p_sys->b_vnc_mouse_events = var_CreateGetBoolCommand( p_this,
 308                                             RMTOSD_CFG "mouse-events" );
 309     p_sys->b_vnc_key_events = var_CreateGetBoolCommand( p_this,
 310                                             RMTOSD_CFG "key-events" );
 311
 312     /* Keep track of OSD Events */
 313     p_sys->b_need_update  = false;
 314
 315     /* Attach subpicture filter callback */
 316     p_filter->pf_sub_filter = Filter;
 317
 318     p_sys->p_vout = vlc_object_find( p_this, VLC_OBJECT_VOUT, FIND_PARENT );
 319
 320     if( p_sys->p_vout )
 321     {
 322         var_AddCallback( p_sys->p_vout, "mouse-moved",
 323                          MouseEvent, p_this );
 324         var_AddCallback( p_sys->p_vout, "mouse-button-down",
 325                          MouseEvent, p_this );
 326         var_AddCallback( p_sys->p_vout->p_libvlc, "key-pressed",
 327                          KeyEvent, p_this );
 328     }
 329
 330     es_format_Init( &p_filter->fmt_out, SPU_ES, VLC_FOURCC( 's','p','u',' ' ) );
 331     p_filter->fmt_out.i_priority = 0;
 332
 333     vlc_gcrypt_init();
 334
 335     /* create the vnc worker thread */
 336     p_sys->p_worker_thread = vlc_object_create( p_this, VLC_OBJECT_GENERIC );
 337     vlc_object_attach( p_sys->p_worker_thread, p_this );
 338     if( vlc_thread_create( p_sys->p_worker_thread, "vnc worker thread",
 339                            vnc_worker_thread,
 340                            VLC_THREAD_PRIORITY_LOW, false ) )
 341     {
 342         vlc_object_detach( p_sys->p_worker_thread );
 343         vlc_object_release( p_sys->p_worker_thread );
 344         p_sys->p_worker_thread = NULL;
 345         msg_Err( p_filter, "cannot spawn vnc message reader thread" );
 346         goto error;
 347     }
 348
 349     msg_Dbg( p_filter, "osdvnc filter started" );
 350
 351     return VLC_SUCCESS;
 352
 353 error:
 354     msg_Err( p_filter, "osdvnc filter discarded" );
 355
 356     p_sys->b_continue = false;
 357
 358     stop_osdvnc( p_filter );
 359
 360     free( p_sys->psz_host );
 361     free( p_sys->psz_passwd );
 362     free( p_sys );
 363
 364     return VLC_EGENERIC;
 365 }
 366
 367 /*****************************************************************************
 368  * DestroyFilter: Make a clean exit of this plugin
 369  *****************************************************************************/
 370 static void DestroyFilter( vlc_object_t *p_this )
 371 {
 372     filter_t     *p_filter = (filter_t*)p_this;
 373     filter_sys_t *p_sys = p_filter->p_sys;
 374
 375     msg_Dbg( p_filter, "DestroyFilter called." );
 376
 377     stop_osdvnc( p_filter );
 378
 379     if( p_sys->p_vout )
 380     {
 381         var_DelCallback( p_sys->p_vout, "mouse-moved",
 382                          MouseEvent, p_this );
 383         var_DelCallback( p_sys->p_vout, "mouse-button-down",
 384                          MouseEvent, p_this );
 385         var_DelCallback( p_sys->p_vout->p_libvlc, "key-pressed",
 386                          KeyEvent, p_this );
 387
 388         vlc_object_release( p_sys->p_vout );
 389         p_sys->p_vout = NULL;
 390     }
 391
 392     var_Destroy( p_this, RMTOSD_CFG "host" );
 393     var_Destroy( p_this, RMTOSD_CFG "port" );
 394     var_Destroy( p_this, RMTOSD_CFG "password" );
 395     var_Destroy( p_this, RMTOSD_CFG "update" );
 396     var_Destroy( p_this, RMTOSD_CFG "vnc-polling" );
 397     var_Destroy( p_this, RMTOSD_CFG "mouse-events" );
 398     var_Destroy( p_this, RMTOSD_CFG "key-events" );
 399     var_Destroy( p_this, RMTOSD_CFG "alpha" );
 400
 401     free( p_sys->psz_host );
 402     free( p_sys->psz_passwd );
 403     free( p_sys );
 404 }
 405
 406 static void stop_osdvnc ( filter_t *p_filter )
 407 {
 408     filter_sys_t *p_sys = p_filter->p_sys;
 409
 410     if (p_sys->i_socket >= 0)
 411     {
 412         net_Close(p_sys->i_socket);
 413     }
 414     p_sys->b_continue = false; /* this causes the threads to stop */
 415
 416     if ( p_sys->p_worker_thread )
 417     {
 418         msg_Dbg( p_filter, "joining worker_thread" );
 419         vlc_thread_join( p_sys->p_worker_thread );
 420         vlc_object_detach( p_sys->p_worker_thread );
 421         vlc_object_release( p_sys->p_worker_thread );
 422         msg_Dbg( p_filter, "released worker_thread" );
 423     }
 424
 425     if ( p_sys->p_update_request_thread )
 426     {
 427         msg_Dbg( p_filter, "joining update_request_thread" );
 428         vlc_thread_join( p_sys->p_update_request_thread );
 429         vlc_object_detach( p_sys->p_update_request_thread );
 430         vlc_object_release( p_sys->p_update_request_thread );
 431         msg_Dbg( p_filter, "released update_request_thread" );
 432     }
 433
 434     msg_Dbg( p_filter, "osdvnc stopped" );
 435 }
 436
 437 static bool read_exact( filter_t *p_filter,
 438                         int i_socket,
 439                         char* p_readbuf,
 440                         int i_bytes )
 441 {
 442     return i_bytes == net_Read( p_filter, i_socket, NULL,
 443                                   (unsigned char*)p_readbuf,
 444                                   i_bytes, true );
 445 }
 446
 447
 448 static bool write_exact( filter_t *p_filter,
 449                          int i_socket,
 450                          char* p_writebuf,
 451                          int i_bytes )
 452 {
 453     return i_bytes == net_Write( p_filter, i_socket, NULL,
 454                                   (unsigned char*)p_writebuf, i_bytes );
 455 }
 456
 457 static bool open_vnc_connection ( filter_t *p_filter )
 458 {
 459     filter_sys_t *p_sys = p_filter->p_sys;
 460
 461     msg_Dbg( p_filter, "Open socket to vnc server on %s:%u.",
 462               p_sys->psz_host, p_sys->i_port );
 463
 464     p_sys->i_socket = net_ConnectTCP( p_filter, p_sys->psz_host, p_sys->i_port );
 465
 466     if( p_sys->i_socket < 0 )
 467     {
 468         msg_Err( p_filter, "Could not open socket" );
 469         return false;
 470     }
 471
 472     msg_Dbg( p_filter, "socket is open." );
 473
 474     return true;
 475 }
 476
 477 static bool handshaking ( filter_t *p_filter )
 478 {
 479     filter_sys_t *p_sys = p_filter->p_sys;
 480
 481     msg_Dbg( p_filter, "Reading protocol version" );
 482
 483     rfbProtocolVersionMsg pv;
 484     if ( !read_exact( p_filter, p_sys->i_socket, pv,
 485                       sz_rfbProtocolVersionMsg ) )
 486     {
 487         msg_Err( p_filter, "Could not read version message" );
 488         return false;
 489     }
 490     pv[sz_rfbProtocolVersionMsg] = '\0'; /* pv size is sz_rfbProtocolVersionMsg+1 */
 491
 492     msg_Dbg( p_filter, "Server version is %s", pv );
 493
 494     strncpy(pv, "RFB 003.003\n", sz_rfbProtocolVersionMsg);
 495
 496     if( !write_exact(p_filter, p_sys->i_socket, pv,
 497                      sz_rfbProtocolVersionMsg) )
 498     {
 499         msg_Err( p_filter, "Could not write version message" );
 500         return false;
 501     }
 502
 503     msg_Dbg( p_filter, "Reading authentication scheme" );
 504     uint32_t i_authScheme;
 505     if( !read_exact( p_filter, p_sys->i_socket, (char*)&i_authScheme, 4 ) )
 506     {
 507         msg_Err( p_filter, "Could not read authentication scheme" );
 508         return false;
 509     }
 510     i_authScheme = htonl(i_authScheme);
 511
 512     msg_Dbg( p_filter, "Authentication scheme = %x", i_authScheme );
 513     if ( i_authScheme == rfbConnFailed )
 514     {
 515         msg_Err( p_filter, "Connection rejected by server" );
 516         return false;
 517     }
 518     if (i_authScheme == rfbVncAuth)
 519     {
 520         unsigned char challenge[CHALLENGESIZE];
 521         if ( !read_exact( p_filter, p_sys->i_socket,
 522                           (char*)challenge, CHALLENGESIZE ) )
 523         {
 524             msg_Err( p_filter, "Could not read password challenge" );
 525             return false;
 526         }
 527
 528         vnc_encrypt_bytes( challenge, p_sys->psz_passwd );
 529
 530         if( !write_exact(p_filter, p_sys->i_socket,
 531                          (char*)challenge, CHALLENGESIZE ) )
 532         {
 533             msg_Err( p_filter, "Could not write password" );
 534             return false;
 535         }
 536         uint32_t i_authResult;
 537         if( !read_exact( p_filter, p_sys->i_socket, (char*)&i_authResult, 4 ) )
 538         {
 539             msg_Err( p_filter, "Could not read authentication result" );
 540             return false;
 541         }
 542         i_authResult = htonl(i_authResult);
 543         if (i_authResult != rfbVncAuthOK)
 544         {
 545             msg_Err( p_filter, "VNC authentication failed" );
 546             return false;
 547         }
 548     }
 549
 550     msg_Dbg( p_filter, "Writing client init message" );
 551     rfbClientInitMsg ci;
 552     ci.shared = 1;
 553     if( !write_exact( p_filter, p_sys->i_socket,
 554                       (char*)&ci, sz_rfbClientInitMsg ) )
 555     {
 556         msg_Err( p_filter, "Could not write client init message" );
 557         return false;
 558     }
 559
 560     msg_Dbg( p_filter, "Reading server init message" );
 561     rfbServerInitMsg si;
 562     if( !read_exact( p_filter, p_sys->i_socket,
 563                      (char*)&si, sz_rfbServerInitMsg ) )
 564     {
 565         msg_Err( p_filter, "Could not read server init message" );
 566         return false;
 567     }
 568     si.framebufferWidth = htons(si.framebufferWidth);
 569     si.framebufferHeight = htons(si.framebufferHeight);
 570     si.format.redMax = htons(si.format.redMax);
 571     si.format.greenMax = htons(si.format.greenMax);
 572     si.format.blueMax = htons(si.format.blueMax);
 573
 574     p_sys->i_vnc_width = si.framebufferWidth;
 575     p_sys->i_vnc_height = si.framebufferHeight;
 576
 577     msg_Dbg( p_filter, "Servers preferred pixelformat: "
 578                         "%ux%u, R(%u),G(%u),B(%u), %u bit, depht=%u, %s",
 579                         si.framebufferWidth,
 580                         si.framebufferHeight,
 581                         si.format.redMax,
 582                         si.format.greenMax,
 583                         si.format.blueMax,
 584                         si.format.bitsPerPixel,
 585                         si.format.depth,
 586                         si.format.trueColour ? "TrueColor" : "Not-TrueColor");
 587
 588     uint32_t i_nameLength = htonl(si.nameLength);
 589     if( i_nameLength > MAX_VNC_SERVER_NAME_LENGTH )
 590     {
 591         msg_Err( p_filter, "Server name too long" );
 592         return false;
 593     }
 594     char s_ServerName[MAX_VNC_SERVER_NAME_LENGTH+1];
 595
 596     msg_Dbg( p_filter, "Reading server name with size = %u", i_nameLength );
 597     if( !read_exact( p_filter, p_sys->i_socket, s_ServerName, i_nameLength ) )
 598     {
 599         msg_Err( p_filter, "Could not read server name" );
 600         return false;
 601     }
 602     s_ServerName[i_nameLength] = '\0';
 603
 604     if( strcmp( s_ServerName, "VDR-OSD") == 0 )
 605     {
 606         msg_Dbg( p_filter, "Server is a VDR" );
 607         p_sys->b_alpha_from_vnc = true;
 608     }
 609     else
 610     {
 611         msg_Dbg( p_filter, "Server is a normal VNC" );
 612         p_sys->b_alpha_from_vnc = false;
 613     }
 614
 615
 616     msg_Dbg( p_filter, "Server init message read properly" );
 617     msg_Dbg( p_filter, "Server name is %s", s_ServerName );
 618
 619     msg_Dbg( p_filter, "Writing SetPixelFormat message" );
 620
 621     rfbSetPixelFormatMsg sp;
 622     sp.type = rfbSetPixelFormat;
 623     sp.format.bitsPerPixel = 8;
 624     sp.format.depth = 8 ;
 625     sp.format.bigEndian = 1;
 626     sp.format.trueColour = 0;
 627     sp.format.redMax = htons(31);
 628     sp.format.greenMax = htons(31);
 629     sp.format.blueMax = htons(31);
 630     sp.format.redShift = 10;
 631     sp.format.greenShift = 5;
 632     sp.format.blueShift = 0;
 633
 634     if( !write_exact( p_filter, p_sys->i_socket,
 635                       (char*)&sp, sz_rfbSetPixelFormatMsg) )
 636     {
 637         msg_Err( p_filter, "Could not write SetPixelFormat message" );
 638         return false;
 639     }
 640
 641     msg_Dbg( p_filter, "Writing SetEncodings message" );
 642
 643     rfbSetEncodingsMsg se;
 644     se.type = rfbSetEncodings;
 645     se.nEncodings = htons( p_sys->b_alpha_from_vnc ? 3 : 2 );
 646
 647     if( !write_exact( p_filter, p_sys->i_socket,
 648                       (char*)&se, sz_rfbSetEncodingsMsg) )
 649     {
 650         msg_Err( p_filter, "Could not write SetEncodings message begin" );
 651         return false;
 652     }
 653
 654     uint32_t i_encoding;
 655
 656     msg_Dbg( p_filter, "Writing SetEncodings rfbEncodingCopyRect" );
 657     i_encoding = htonl(rfbEncodingCopyRect);
 658     if( !write_exact( p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
 659     {
 660         msg_Err( p_filter, "Could not write encoding type rfbEncodingCopyRect." );
 661         return false;
 662     }
 663
 664     msg_Dbg( p_filter, "Writing SetEncodings rfbEncodingRRE" );
 665     i_encoding = htonl(rfbEncodingRRE);
 666     if( !write_exact(p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
 667     {
 668         msg_Err( p_filter, "Could not write encoding type rfbEncodingRRE." );
 669         return false;
 670     }
 671
 672     if( p_sys->b_alpha_from_vnc )
 673     {
 674         msg_Dbg( p_filter, "Writing SetEncodings rfbEncSpecialUseAlpha" );
 675         i_encoding = 0x00F0FFFF; /* rfbEncSpecialUseAlpha is 0xFFFFF000
 676                                   * before we swap it */
 677         if( !write_exact(p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
 678         {
 679             msg_Err( p_filter, "Could not write encoding type rfbEncSpecialUseAlpha." );
 680             return false;
 681         }
 682     }
 683     return true;
 684
 685 }
 686
 687 static void vnc_worker_thread( vlc_object_t *p_thread_obj )
 688 {
 689     filter_t* p_filter = (filter_t*)(p_thread_obj->p_parent);
 690     filter_sys_t *p_sys = p_filter->p_sys;
 691
 692     msg_Dbg( p_filter, "VNC worker thread started" );
 693
 694     if ( open_vnc_connection ( p_filter ) == false )
 695     {
 696         msg_Err( p_filter, "Could not connect to vnc host" );
 697         return;
 698     }
 699
 700     if ( handshaking ( p_filter ) == false )
 701     {
 702         msg_Err( p_filter, "Error occured while handshaking vnc host" );
 703         return;
 704     }
 705
 706     p_sys->b_connection_active = true; /* to enable sending key
 707                                             * and mouse events to host */
 708
 709     /* Create an empty picture for VNC the data */
 710     vlc_mutex_lock( &p_sys->lock );
 711     p_sys->p_pic = malloc( sizeof(picture_t) );
 712     if( !p_sys->p_pic )
 713     {
 714         vlc_mutex_unlock( &p_sys->lock );
 715         return;
 716     }
 717     vout_AllocatePicture( VLC_OBJECT(p_filter), p_sys->p_pic,
 718                           VLC_FOURCC('Y','U','V','A'),
 719                           p_sys->i_vnc_width,
 720                           p_sys->i_vnc_height,
 721                           VOUT_ASPECT_FACTOR );
 722     if( !p_sys->p_pic->i_planes )
 723     {
 724         free( p_sys->p_pic );
 725         p_sys->p_pic = NULL;
 726         vlc_mutex_unlock( &p_sys->lock );
 727         return;
 728     }
 729         p_sys->i_vnc_pixels = p_sys->i_vnc_width * p_sys->i_vnc_height;
 730
 731     vlc_mutex_unlock( &p_sys->lock );
 732
 733     /* create the update request thread */
 734     p_sys->p_update_request_thread = vlc_object_create( p_filter,
 735                                                         VLC_OBJECT_GENERIC );
 736     vlc_object_attach( p_sys->p_update_request_thread, p_filter );
 737     if( vlc_thread_create( p_sys->p_update_request_thread,
 738                            "vnc update request thread",
 739                            update_request_thread,
 740                            VLC_THREAD_PRIORITY_LOW, false ) )
 741     {
 742         vlc_object_detach( p_sys->p_update_request_thread );
 743         vlc_object_release( p_sys->p_update_request_thread );
 744         p_sys->p_update_request_thread = NULL;
 745         msg_Err( p_filter, "cannot spawn vnc update request thread" );
 746         return;
 747     }
 748
 749     /* connection is initialized, now read and handle server messages */
 750
 751     int i_msgSize;
 752
 753     rfbServerToClientMsg msg;
 754
 755     while (p_sys->b_continue)
 756     {
 757        msg.type = 0;
 758        if ( !read_exact(p_filter, p_sys->i_socket, (char*)&msg, 1 ) )
 759        {
 760            msg_Err( p_filter, "Error while waiting for next server message");
 761            p_sys->b_continue = false;
 762        }
 763        else
 764        {
 765            switch (msg.type)
 766            {
 767               case rfbFramebufferUpdate:
 768                 i_msgSize = sz_rfbFramebufferUpdateMsg;
 769                 break;
 770               case rfbSetColourMapEntries:
 771                 i_msgSize = sz_rfbSetColourMapEntriesMsg;
 772                 break;
 773               case rfbBell:
 774                 i_msgSize = sz_rfbBellMsg;
 775                 break;
 776               case rfbServerCutText:
 777                 i_msgSize = sz_rfbServerCutTextMsg;
 778                 break;
 779               case rfbReSizeFrameBuffer:
 780                 i_msgSize = sz_rfbReSizeFrameBufferMsg;
 781                 break;
 782               default:
 783                 i_msgSize = 0;
 784                 msg_Err( p_filter, "Invalid message %u received", msg.type );
 785                 p_sys->b_continue = false;
 786            }
 787            if (p_sys->b_continue == true)
 788            {
 789                if (--i_msgSize > 0)
 790                {
 791                    if ( !read_exact( p_filter, p_sys->i_socket,
 792                                      ((char*)&msg)+1, i_msgSize ) )
 793                    {
 794                        msg_Err( p_filter, "Error while reading message of type %u",
 795                                 msg.type );
 796                        p_sys->b_continue = false;
 797                    }
 798                    else
 799                    {
 800                        process_server_message( p_filter, &msg);
 801                    }
 802                }
 803                else
 804                {
 805                    process_server_message( p_filter, &msg);
 806                }
 807            }
 808
 809        }
 810
 811        if (p_sys->p_worker_thread->b_die ||
 812            p_sys->p_worker_thread->b_error)
 813        {
 814            p_sys->b_continue = false;
 815        }
 816
 817     }
 818
 819     msg_Dbg( p_filter, "VNC message reader thread ended" );
 820
 821 }
 822
 823 static void update_request_thread( vlc_object_t *p_thread_obj )
 824 {
 825     filter_t* p_filter = (filter_t*)(p_thread_obj->p_parent);
 826     filter_sys_t *p_sys = p_filter->p_sys;
 827
 828     msg_Dbg( p_filter, "VNC update request thread started" );
 829
 830     rfbFramebufferUpdateRequestMsg udr;
 831     udr.type = rfbFramebufferUpdateRequest;
 832     udr.incremental = 0;
 833     udr.x = 0;
 834     udr.y = 0;
 835     udr.w = htons(p_sys->i_vnc_width);
 836     udr.h = htons(p_sys->i_vnc_height);
 837
 838     if( write_exact(p_filter, p_sys->i_socket, (char*)&udr,
 839            sz_rfbFramebufferUpdateRequestMsg) == false)
 840     {
 841         msg_Err( p_filter, "Could not write rfbFramebufferUpdateRequestMsg." );
 842         p_sys->b_continue = false;
 843     }
 844
 845     udr.incremental = 1;
 846     mtime_t i_poll_interval_microsec = p_sys->i_vnc_poll_interval * 1000;
 847
 848     if (p_sys->b_vnc_poll)
 849     {
 850         while ( p_sys->b_continue == true )
 851         {
 852             msleep( i_poll_interval_microsec );
 853             if( write_exact(p_filter, p_sys->i_socket, (char*)&udr,
 854                    sz_rfbFramebufferUpdateRequestMsg) == false)
 855             {
 856                 msg_Err( p_filter, "Could not write rfbFramebufferUpdateRequestMsg." );
 857                 p_sys->b_continue = false;
 858             }
 859             if (p_sys->p_update_request_thread->b_die ||
 860                 p_sys->p_update_request_thread->b_error)
 861             {
 862                 p_sys->b_continue = false;
 863             }
 864         }
 865     }
 866     else
 867     {
 868         msg_Dbg( p_filter, "VNC polling disabled." );
 869     }
 870     msg_Dbg( p_filter, "VNC update request thread ended" );
 871 }
 872
 873 static bool process_server_message ( filter_t *p_filter,
 874                                      rfbServerToClientMsg *msg )
 875 {
 876     filter_sys_t *p_sys = p_filter->p_sys;
 877
 878     switch (msg->type)
 879     {
 880     case rfbFramebufferUpdate:
 881         {
 882             msg->fu.nRects = htons(msg->fu.nRects);
 883             rfbFramebufferUpdateRectHeader hdr;
 884
 885             for (int i_rect = 0; i_rect < msg->fu.nRects; i_rect++)
 886             {
 887                 if (!read_exact(p_filter, p_sys->i_socket, (char*)&hdr,
 888                     sz_rfbFramebufferUpdateRectHeader ) )
 889                 {
 890                     msg_Err( p_filter, "Could not read FrameBufferUpdate header" );
 891                     return false;
 892                 }
 893                 hdr.r.x = htons(hdr.r.x);
 894                 hdr.r.y = htons(hdr.r.y);
 895                 hdr.r.w = htons(hdr.r.w);
 896                 hdr.r.h = htons(hdr.r.h);
 897                 hdr.encoding = htonl(hdr.encoding);
 898
 899                 switch (hdr.encoding)
 900                 {
 901                 case rfbEncodingRaw:
 902                     {
 903                         int i_line;
 904                         for (i_line = 0; i_line < hdr.r.h; i_line++)
 905                         {
 906                             if ( !read_exact( p_filter, p_sys->i_socket,
 907                                     p_sys->read_buffer, hdr.r.w ) )
 908                             {
 909                                 msg_Err( p_filter,
 910                                  "Could not read FrameBufferUpdate line data" );
 911                                return false;
 912                             }
 913                             vlc_mutex_lock( &p_sys->lock );
 914                             if ( !raw_line( p_sys, hdr.r.x,
 915                                             hdr.r.y + i_line,
 916                                             hdr.r.w ) )
 917                             {
 918                                 msg_Err( p_filter, "raw_line failed." );
 919                                 vlc_mutex_unlock( &p_sys->lock );
 920                                 return false;
 921                             }
 922                             vlc_mutex_unlock( &p_sys->lock );
 923                         }
 924                     }
 925                     break;
 926
 927                 case rfbEncodingCopyRect:
 928                     {
 929                         msg_Err( p_filter,
 930                           "Rect in unsupported encoding rfbEncodingCopyRect" );
 931                         return false;
 932                     }
 933
 934                 case rfbEncodingRRE:
 935                     {
 936                         rfbRREHeader rrehdr;
 937                         if ( !read_exact( p_filter, p_sys->i_socket,
 938                                           (char*)&rrehdr,
 939                                           sz_rfbRREHeader ) )
 940                         {
 941                             msg_Err( p_filter, "Could not read rfbRREHeader" );
 942                             return false;
 943                         }
 944                         uint8_t i_pixcolor;
 945                         if ( !read_exact(p_filter, p_sys->i_socket,
 946                                          (char*)&i_pixcolor, 1 ) )
 947                         {
 948                             msg_Err( p_filter, "Could not read RRE pixcolor" );
 949                             return false;
 950                         }
 951
 952                         vlc_mutex_lock( &p_sys->lock );
 953                         if ( !fill_rect( p_sys,
 954                                         hdr.r.x, hdr.r.y,
 955                                         hdr.r.w, hdr.r.h,
 956                                         i_pixcolor) )
 957                         {
 958                             msg_Err( p_filter, "main fill_rect failed." );
 959                             vlc_mutex_unlock( &p_sys->lock );
 960                             return false;
 961                         }
 962                         vlc_mutex_unlock( &p_sys->lock );
 963
 964                         rrehdr.nSubrects = htonl(rrehdr.nSubrects);
 965
 966                         int i_datasize = rrehdr.nSubrects *
 967                                      ( sizeof(i_pixcolor) + sz_rfbRectangle ) ;
 968                         if ( i_datasize > READ_BUFFER_SIZE )
 969                         {
 970                             msg_Err( p_filter, "Buffer too small, "
 971                                      "need %u bytes", i_datasize );
 972                             return false;
 973                         }
 974                         if ( !read_exact( p_filter, p_sys->i_socket,
 975                                        p_sys->read_buffer, i_datasize ) )
 976                         {
 977                             msg_Err( p_filter,
 978                                      "Could not read RRE subrect data" );
 979                             return false;
 980                         }
 981
 982                         uint32_t i_subrect;
 983                         rfbRectangle* p_subrect;
 984                         int i_offset = 0;
 985                         vlc_mutex_lock( &p_sys->lock );
 986                         for ( i_subrect = 0;
 987                               i_subrect < rrehdr.nSubrects; i_subrect++)
 988                         {
 989                             i_pixcolor = p_sys->read_buffer[i_offset];
 990                             i_offset += sizeof(i_pixcolor);
 991                             p_subrect =
 992                                (rfbRectangle*)(p_sys->read_buffer + i_offset);
 993                             i_offset += sz_rfbRectangle;
 994
 995                             if (!fill_rect( p_sys,
 996                                             htons(p_subrect->x) + hdr.r.x,
 997                                             htons(p_subrect->y) + hdr.r.y,
 998                                             htons(p_subrect->w),
 999                                             htons(p_subrect->h),
1000                                             i_pixcolor) )
1001                             {
1002                                 msg_Err( p_filter,
1003                                     "subrect %u fill_rect failed.", i_subrect );
1004                                 vlc_mutex_unlock( &p_sys->lock );
1005                                 return false;
1006                             }
1007                         }
1008                         vlc_mutex_unlock( &p_sys->lock );
1009                     }
1010                     break;
1011
1012                 }
1013
1014             }
1015             vlc_mutex_lock( &p_sys->lock );
1016             p_sys->b_need_update = true;
1017             vlc_mutex_unlock( &p_sys->lock );
1018         }
1019         return true;
1020
1021     case rfbSetColourMapEntries:
1022         {
1023             msg->scme.nColours = htons(msg->scme.nColours);
1024             msg->scme.firstColour = htons(msg->scme.firstColour);
1025             int i_datasize;
1026             if ( p_sys->b_alpha_from_vnc == true )
1027             {
1028                 i_datasize = 2 * msg->scme.nColours * 4;
1029             }
1030             else
1031             {
1032                 i_datasize = 2 * msg->scme.nColours * 3;
1033             }
1034             if ( i_datasize > READ_BUFFER_SIZE )
1035             {
1036                 msg_Err( p_filter, "Buffer too small, need %u bytes",
1037                                    i_datasize );
1038                 return false;
1039             }
1040
1041             if ( !read_exact( p_filter, p_sys->i_socket,
1042                               p_sys->read_buffer, i_datasize ) )
1043             {
1044                 msg_Err( p_filter, "Could not read color map data" );
1045                 return false;
1046             }
1047
1048             uint8_t i_red, i_green, i_blue, i_alpha, i_color_index;
1049             uint16_t i_offset = 0;
1050             i_alpha = 255;
1051
1052             for (int i = 0; i < msg->scme.nColours; i++)
1053             {
1054                 i_color_index = i+msg->scme.firstColour;
1055                 if ( p_sys->b_alpha_from_vnc == true )
1056                 {
1057                     i_alpha = p_sys->read_buffer[i_offset];
1058                     i_offset += 2;
1059                 }
1060                 i_red   = p_sys->read_buffer[i_offset];
1061                 i_offset += 2;
1062                 i_green = p_sys->read_buffer[i_offset];
1063                 i_offset += 2;
1064                 i_blue  = p_sys->read_buffer[i_offset];
1065                 i_offset += 2;
1066                 rgb_to_yuv( &p_sys->ar_color_table_yuv[i_color_index][0],
1067                             &p_sys->ar_color_table_yuv[i_color_index][1],
1068                             &p_sys->ar_color_table_yuv[i_color_index][2],
1069                             i_red,
1070                             i_green,
1071                             i_blue );
1072                 p_sys->ar_color_table_yuv[i][3] = i_alpha;
1073             }
1074         }
1075         return true;
1076
1077     case rfbBell:
1078         msg_Err( p_filter, "rfbBell received" );
1079         return true;
1080
1081     case rfbServerCutText:
1082         msg->sct.length = htons(msg->sct.length);
1083         if ( msg->sct.length > READ_BUFFER_SIZE )
1084         {
1085             msg_Err( p_filter, "Buffer too small, need %u bytes", msg->sct.length );
1086             return false;
1087         }
1088         if ( !read_exact(p_filter, p_sys->i_socket,
1089                          p_sys->read_buffer, msg->sct.length ) )
1090         {
1091             msg_Err( p_filter, "Could not read Reading rfbServerCutText data" );
1092             return false;
1093         }
1094         return true;
1095
1096     case rfbReSizeFrameBuffer:
1097         msg_Err( p_filter, "Reading rfbReSizeFrameBuffer not implemented" );
1098         return false;
1099
1100     default:
1101         msg_Err( p_filter, "Invalid message %u received", msg->type );
1102         return false;
1103     }
1104     return false;
1105 }
1106
1107 /****************************************************************************
1108  * Filter: the whole thing
1109  ****************************************************************************
1110  * This function outputs subpictures at regular time intervals.
1111  ****************************************************************************/
1112 static subpicture_t *Filter( filter_t *p_filter, mtime_t date )
1113 {
1114     filter_sys_t *p_sys = p_filter->p_sys;
1115     subpicture_t *p_spu;
1116     subpicture_region_t *p_region;
1117     video_format_t fmt;
1118     picture_t *p_pic;
1119
1120     if( !p_sys->b_need_update )
1121     {
1122         return NULL;
1123     }
1124
1125     vlc_mutex_lock( &p_sys->lock );
1126
1127     p_pic = p_sys->p_pic;
1128
1129     if( !p_pic )
1130     {
1131         vlc_mutex_unlock( &p_sys->lock );
1132         return NULL;
1133     }
1134
1135     /* Allocate the subpicture internal data. */
1136     p_spu = p_filter->pf_sub_buffer_new( p_filter );
1137     if( !p_spu )
1138     {
1139         vlc_mutex_unlock( &p_sys->lock );
1140         return NULL;
1141     }
1142
1143     p_spu->b_absolute = false;
1144     p_spu->i_start = date;
1145     p_spu->i_stop = 0;
1146     p_spu->b_ephemer = true;
1147
1148     /* Create new SPU region */
1149     memset( &fmt, 0, sizeof(video_format_t) );
1150     fmt.i_chroma = VLC_FOURCC('Y','U','V','A');
1151     fmt.i_aspect = VOUT_ASPECT_FACTOR;
1152     fmt.i_sar_num = fmt.i_sar_den = 1;
1153     fmt.i_width = fmt.i_visible_width = p_pic->p[Y_PLANE].i_visible_pitch;
1154     fmt.i_height = fmt.i_visible_height = p_pic->p[Y_PLANE].i_visible_lines;
1155     fmt.i_x_offset = fmt.i_y_offset = 0;
1156     p_region = p_spu->pf_create_region( VLC_OBJECT(p_filter), &fmt );
1157     if( !p_region )
1158     {
1159         msg_Err( p_filter, "cannot allocate SPU region" );
1160         p_filter->pf_sub_buffer_del( p_filter, p_spu );
1161         vlc_mutex_unlock( &p_sys->lock );
1162         return NULL;
1163     }
1164
1165     vout_CopyPicture( p_filter, &p_region->picture, p_pic );
1166
1167     p_sys->b_need_update = false;
1168
1169     vlc_mutex_unlock( &p_sys->lock );
1170
1171     /* set to one of the 9 relative locations */
1172     p_region->i_align = 0; //=CENTER
1173     p_spu->b_absolute = false;
1174
1175
1176     p_spu->i_x = 0;
1177     p_spu->i_y = 0;
1178
1179     p_spu->p_region = p_region;
1180
1181     p_spu->i_alpha = ( p_sys->i_alpha );
1182
1183     return p_spu;
1184 }
1185
1186
1187 static inline void rgb_to_yuv( uint8_t *y, uint8_t *u, uint8_t *v,
1188                                int r, int g, int b )
1189 {
1190     *y = ( ( (  66 * r + 129 * g +  25 * b + 128 ) >> 8 ) + 16 );
1191     *u =   ( ( -38 * r -  74 * g + 112 * b + 128 ) >> 8 ) + 128 ;
1192     *v =   ( ( 112 * r -  94 * g -  18 * b + 128 ) >> 8 ) + 128 ;
1193 }
1194
1195 static inline bool fill_rect( filter_sys_t* p_sys,
1196                               uint16_t i_x, uint16_t i_y,
1197                               uint16_t i_w, uint16_t i_h,
1198                               uint8_t i_color)
1199 {
1200     plane_t *p_outY = p_sys->p_pic->p+Y_PLANE;
1201     plane_t *p_outU = p_sys->p_pic->p+U_PLANE;
1202     plane_t *p_outV = p_sys->p_pic->p+V_PLANE;
1203     plane_t *p_outA = p_sys->p_pic->p+A_PLANE;
1204     int i_pitch = p_outY->i_pitch;
1205     int i_lines = p_outY->i_lines;
1206     if ( i_x + i_w > i_pitch)
1207         return false;
1208     if ( i_y + i_h > i_lines)
1209         return false;
1210     int i_line_offset = i_y * i_pitch;
1211     uint8_t i_yuv_y = p_sys->ar_color_table_yuv[i_color][0];
1212     uint8_t i_yuv_u = p_sys->ar_color_table_yuv[i_color][1];
1213     uint8_t i_yuv_v = p_sys->ar_color_table_yuv[i_color][2];
1214     uint8_t i_alpha = p_sys->ar_color_table_yuv[i_color][3];
1215     for( int i_line = 0; i_line < i_h; i_line++ )
1216     {
1217         for( int i_column = 0; i_column < i_w; i_column++ )
1218         {
1219             int i_total_offset = i_line_offset + i_x + i_column;
1220             p_outY->p_pixels[ i_total_offset ] = i_yuv_y;
1221             p_outU->p_pixels[ i_total_offset ] = i_yuv_u;
1222             p_outV->p_pixels[ i_total_offset ] = i_yuv_v;
1223             p_outA->p_pixels[ i_total_offset ] = i_alpha;
1224         }
1225         i_line_offset += i_pitch;
1226     }
1227     return true;
1228 }
1229
1230 static inline bool raw_line(  filter_sys_t* p_sys,
1231                               uint16_t i_x, uint16_t i_y,
1232                               uint16_t i_w )
1233 {
1234     plane_t *p_outY = p_sys->p_pic->p+Y_PLANE;
1235     plane_t *p_outU = p_sys->p_pic->p+U_PLANE;
1236     plane_t *p_outV = p_sys->p_pic->p+V_PLANE;
1237     plane_t *p_outA = p_sys->p_pic->p+A_PLANE;
1238     int i_pitch = p_outY->i_pitch;
1239     int i_lines = p_outY->i_lines;
1240     if ( i_x + i_w > i_pitch)
1241         return false;
1242     if ( i_y > i_lines)
1243         return false;
1244
1245     int i_line_offset = i_y * i_pitch + i_x;
1246
1247     for( int i_column = 0; i_column < i_w; i_column++ )
1248     {
1249         int i_offset = i_line_offset + i_column;
1250         uint8_t i_color = p_sys->read_buffer[i_column];
1251         p_outY->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][0];
1252         p_outU->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][1];
1253         p_outV->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][2];
1254         p_outA->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][3];
1255     }
1256
1257     return true;
1258 }
1259
1260
1261 /*****************************************************************************
1262  * MouseEvent: callback for mouse events
1263  *****************************************************************************/
1264 static int MouseEvent( vlc_object_t *p_this, char const *psz_var,
1265                        vlc_value_t oldval, vlc_value_t newval, void *p_data )
1266 {
1267     VLC_UNUSED(oldval); VLC_UNUSED(newval); VLC_UNUSED(psz_var);
1268
1269     filter_t *p_filter = (filter_t *)p_data;
1270     filter_sys_t *p_sys = p_filter->p_sys;
1271
1272     if( !p_sys->b_connection_active )
1273         return VLC_SUCCESS;
1274
1275     if( !p_sys->b_vnc_mouse_events )
1276         return VLC_SUCCESS;
1277
1278     vout_thread_t *p_vout = (vout_thread_t*)p_sys->p_vout;
1279     int i_x, i_y;
1280     int i_v;
1281
1282     int v_h = p_vout->output.i_height;
1283     int v_w = p_vout->output.i_width;
1284
1285     i_v = var_GetInteger( p_sys->p_vout, "mouse-button-down" );
1286     i_y = var_GetInteger( p_sys->p_vout, "mouse-y" );
1287     i_x = var_GetInteger( p_sys->p_vout, "mouse-x" );
1288
1289     if( i_y < 0 || i_x < 0 || i_y >= v_h || i_x >= v_w )
1290     {
1291        msg_Dbg( p_this, "invalid mouse event? x=%d y=%d btn=%x", i_x, i_y, i_v );
1292        return VLC_SUCCESS;
1293     }
1294 #ifdef VNC_DEBUG
1295     msg_Dbg( p_this, "mouse event x=%d y=%d btn=%x", i_x, i_y, i_v );
1296 #endif
1297
1298     /* FIXME: calculate x and y coordinates for scaled output */
1299
1300     /* buttonMask bits 0-7 are buttons 1-8, 0=up, 1=down */
1301     rfbPointerEventMsg ev;
1302     ev.type = rfbPointerEvent;
1303     ev.buttonMask = i_v;
1304     ev.x = htons(i_x);
1305     ev.y = htons(i_y);
1306
1307     write_exact( p_filter, p_sys->i_socket,
1308                  (char*)&ev, sz_rfbPointerEventMsg);
1309
1310     return VLC_SUCCESS;
1311 }
1312
1313 /*****************************************************************************
1314  * KeyEvent: callback for keyboard events
1315  *****************************************************************************/
1316 static int KeyEvent( vlc_object_t *p_this, char const *psz_var,
1317                      vlc_value_t oldval, vlc_value_t newval, void *p_data )
1318 {
1319     VLC_UNUSED(psz_var); VLC_UNUSED(oldval);
1320
1321     filter_t *p_filter = (filter_t *)p_data;
1322     filter_sys_t *p_sys = p_filter->p_sys;
1323
1324     if( !p_sys->b_connection_active )
1325         return VLC_SUCCESS;
1326
1327     if( !p_sys->b_vnc_key_events )
1328         return VLC_SUCCESS;
1329
1330     msg_Dbg( p_this, "key pressed (%d) ", newval.i_int );
1331
1332     if ( !newval.i_int )
1333     {
1334         msg_Err( p_this, "Received invalid key event %d", newval.i_int );
1335         return VLC_EGENERIC;
1336     }
1337
1338     uint32_t i_key32 = newval.i_int;
1339     i_key32 = htonl(i_key32);
1340     rfbKeyEventMsg ev;
1341     ev.type = rfbKeyEvent;
1342     ev.down = 1;
1343
1344     /* first key-down for modifier-keys */
1345     if (newval.i_int & KEY_MODIFIER_CTRL)
1346     {
1347         ev.key = 0xffe3;
1348         write_exact( p_filter, p_sys->i_socket,
1349                      (char*)&ev, sz_rfbKeyEventMsg);
1350     }
1351     if (newval.i_int & KEY_MODIFIER_SHIFT)
1352     {
1353         ev.key = 0xffe1;
1354         write_exact( p_filter, p_sys->i_socket,
1355                      (char*)&ev, sz_rfbKeyEventMsg);
1356     }
1357     if (newval.i_int & KEY_MODIFIER_ALT)
1358     {
1359         ev.key = 0xffe9;
1360         write_exact( p_filter, p_sys->i_socket,
1361                      (char*)&ev, sz_rfbKeyEventMsg);
1362     }
1363
1364     /* then key-down for the pressed key */
1365     ev.key = i_key32;
1366     write_exact( p_filter, p_sys->i_socket,
1367                  (char*)&ev, sz_rfbKeyEventMsg);
1368
1369     ev.down = 0;
1370
1371     /* then key-up for the pressed key */
1372     write_exact( p_filter, p_sys->i_socket,
1373                  (char*)&ev, sz_rfbKeyEventMsg);
1374
1375     /* last key-down for modifier-keys */
1376     if (newval.i_int & KEY_MODIFIER_CTRL)
1377     {
1378         ev.key = 0xffe3;
1379         write_exact( p_filter, p_sys->i_socket,
1380                      (char*)&ev, sz_rfbKeyEventMsg);
1381     }
1382     if (newval.i_int & KEY_MODIFIER_SHIFT)
1383     {
1384         ev.key = 0xffe1;
1385         write_exact( p_filter, p_sys->i_socket,
1386                      (char*)&ev, sz_rfbKeyEventMsg);
1387     }
1388     if (newval.i_int & KEY_MODIFIER_ALT)
1389     {
1390        ev.key = 0xffe9;
1391        write_exact( p_filter, p_sys->i_socket,
1392                     (char*)&ev, sz_rfbKeyEventMsg);
1393     }
1394
1395     return VLC_SUCCESS;
1396 }
1397
1398 static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd )
1399 {
1400     unsigned char key[8];
1401     unsigned int i;
1402
1403     for (i = 0; i < 8; i++)
1404         key[i] = i < strlen( passwd ) ? passwd[i] : '\0';
1405
1406     gcry_cipher_hd_t ctx;
1407     gcry_cipher_open( &ctx, GCRY_CIPHER_DES, GCRY_CIPHER_MODE_ECB,0);
1408
1409     /* reverse bits of the key */
1410     for( i = 0 ; i < 8 ; i ++ )
1411         key[i] =
1412             (key[i] >> 7) +
1413             (((key[i] >> 6) & 0x01 ) << 1 ) +
1414             (((key[i] >> 5) & 0x01 ) << 2 ) +
1415             (((key[i] >> 4) & 0x01 ) << 3 ) +
1416             (((key[i] >> 3) & 0x01 ) << 4 ) +
1417             (((key[i] >> 2) & 0x01 ) << 5 ) +
1418             (((key[i] >> 1) & 0x01 ) << 6 ) +
1419             ((key[i] & 0x01) << 7 );
1420
1421     gcry_cipher_setkey( ctx, key, 8 );
1422     gcry_cipher_encrypt( ctx, bytes, CHALLENGESIZE, bytes, CHALLENGESIZE );
1423     gcry_cipher_close( ctx );
1424 }