git.sesse.net Git - vlc/blob - modules/video_filter/remoteosd.c

   1 /*****************************************************************************
   2  * remoteosd.c: remote osd over vnc filter module
   3  *****************************************************************************
   4  * Copyright (C) 2007-2008 Matthias Bauer
   5  * $Id$
   6  *
   7  * Authors: Matthias Bauer <matthias dot bauer #_at_# gmx dot ch>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implid warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  22  *****************************************************************************/
  23
  24 /*****************************************************************************
  25  * RemoteOSD uses the RFB-Protocol of VNC to display an On-Screen-Display
  26  * menu generated by a streaming server as overlay for the streamed video.
  27  *
  28  * The streaming server that implements this is the ffnetdev plugin for VDR.
  29  * VDR (VideoDiskRecorder) is an Linux based OpenSource harddisk recorder
  30  * software.
  31  * The VDR ffnetdev plugin emulates the hardware MPEG decoder and streams the
  32  * video over the network instead of hardware video outputs.
  33  * The OSD menu of VDR is offered with the RFB protocol to a VNC client.
  34  *
  35  * In fact this video-filter is a simple VNC client that could be also used to
  36  * connect to a real VNC host.
  37  * Only 8-bit color is supported at the moment.
  38  * Using password protected VNC hosts is supported but not recommended, because
  39  * you need to insert the used password in the plugin configuration page of
  40  * VLC configuration in plain text and it's saved in plain text.
  41  *****************************************************************************/
  42
  43 //#define VNC_DEBUG
  44
  45 /*****************************************************************************
  46  * Preamble
  47  *****************************************************************************/
  48
  49 #ifdef HAVE_CONFIG_H
  50 # include "config.h"
  51 #endif
  52
  53 #include <vlc_common.h>
  54 #include <vlc_plugin.h>
  55 #include <vlc_vout.h>
  56
  57 #include <vlc_filter.h>
  58 #include "filter_common.h"
  59 #include <vlc_image.h>
  60 #include <vlc_osd.h>
  61 #include <vlc_keys.h>
  62
  63 #include <vlc_network.h>
  64 #include <gcrypt.h>              /* to encrypt password */
  65 #include <vlc_gcrypt.h>
  66
  67 #include "remoteosd_rfbproto.h" /* type definitions of the RFB protocol for VNC */
  68
  69 /*****************************************************************************
  70  * Module descriptor
  71  *****************************************************************************/
  72 #define READ_BUFFER_SIZE 1000000
  73
  74 #define RMTOSD_HOST_TEXT N_("VNC Host")
  75 #define RMTOSD_HOST_LONGTEXT N_( \
  76     "VNC hostname or IP address." )
  77
  78 #define RMTOSD_PORT_TEXT N_("VNC Port")
  79 #define RMTOSD_PORT_LONGTEXT N_( \
  80     "VNC portnumber." )
  81
  82 #define RMTOSD_PASSWORD_TEXT N_("VNC Password")
  83 #define RMTOSD_PASSWORD_LONGTEXT N_( \
  84     "VNC password." )
  85
  86 #define RMTOSD_UPDATE_TEXT N_("VNC poll interval" )
  87 #define RMTOSD_UPDATE_LONGTEXT N_( \
  88     "In this interval an update from VNC is requested, default every 300 ms. ")
  89
  90 #define RMTOSD_POLL_TEXT N_("VNC polling")
  91 #define RMTOSD_POLL_LONGTEXT N_( \
  92     "Activate VNC polling. Do NOT activate for use as VDR ffnetdev client." )
  93
  94 #define RMTOSD_MOUSE_TEXT N_("Mouse events")
  95 #define RMTOSD_MOUSE_LONGTEXT N_( \
  96     "Send mouse events to VNC host. Not needed for use as VDR ffnetdev client." )
  97
  98 #define RMTOSD_KEYS_TEXT N_("Key events")
  99 #define RMTOSD_KEYS_LONGTEXT N_( \
 100     "Send key events to VNC host." )
 101
 102 #define RMTOSD_ALPHA_TEXT N_("Alpha transparency value (default 255)")
 103 #define RMTOSD_ALPHA_LONGTEXT N_( \
 104     "The transparency of the OSD VNC can be changed by giving a value " \
 105     "between 0 and 255. A lower value specifies more transparency a higher " \
 106     "means less transparency. The default is being not transparent " \
 107     "(value 255) the minimum is fully transparent (value 0)." )
 108
 109 #define RMTOSD_CFG "rmtosd-"
 110
 111 #define RMTOSD_UPDATE_MIN     200
 112 #define RMTOSD_UPDATE_DEFAULT 1000
 113 #define RMTOSD_UPDATE_MAX     300
 114
 115 static int  CreateFilter ( vlc_object_t * );
 116 static void DestroyFilter( vlc_object_t * );
 117
 118 vlc_module_begin ()
 119     set_description( N_("Remote-OSD over VNC") )
 120     set_capability( "sub filter", 100 )
 121     set_shortname( N_("Remote-OSD") )
 122     set_category( CAT_VIDEO )
 123     set_subcategory( SUBCAT_VIDEO_SUBPIC )
 124     add_shortcut( "rmtosd" )
 125     set_callbacks( CreateFilter, DestroyFilter )
 126
 127     add_string( RMTOSD_CFG "host", "myvdr", NULL, RMTOSD_HOST_TEXT,
 128         RMTOSD_HOST_LONGTEXT, false )
 129     add_integer_with_range( RMTOSD_CFG "port", 20001, 1, 0xFFFF, NULL,
 130         RMTOSD_PORT_TEXT, RMTOSD_PORT_LONGTEXT, false )
 131     add_password( RMTOSD_CFG "password", "", NULL, RMTOSD_PASSWORD_TEXT,
 132         RMTOSD_PASSWORD_LONGTEXT, false )
 133     add_integer_with_range( RMTOSD_CFG "update", RMTOSD_UPDATE_DEFAULT,
 134         RMTOSD_UPDATE_MIN, RMTOSD_UPDATE_MAX, NULL, RMTOSD_UPDATE_TEXT,
 135         RMTOSD_UPDATE_LONGTEXT, true )
 136     add_bool( RMTOSD_CFG "vnc-polling", false, NULL,
 137               RMTOSD_POLL_TEXT , RMTOSD_POLL_LONGTEXT, false )
 138     add_bool( RMTOSD_CFG "mouse-events", false, NULL,
 139               RMTOSD_MOUSE_TEXT , RMTOSD_MOUSE_LONGTEXT, false )
 140     add_bool( RMTOSD_CFG "key-events", false, NULL,
 141               RMTOSD_KEYS_TEXT , RMTOSD_KEYS_LONGTEXT, false )
 142     add_integer_with_range( RMTOSD_CFG "alpha", 255, 0, 255, NULL,
 143         RMTOSD_ALPHA_TEXT, RMTOSD_ALPHA_LONGTEXT, true )
 144
 145 vlc_module_end ()
 146
 147
 148 /*****************************************************************************
 149  * Local prototypes
 150  *****************************************************************************/
 151 #define CHALLENGESIZE 16
 152 #define MAX_VNC_SERVER_NAME_LENGTH 255
 153
 154 /* subfilter functions */
 155 static subpicture_t *Filter( filter_t *, mtime_t );
 156
 157 static int MouseEvent ( vlc_object_t *p_this, char const *psz_var,
 158                         vlc_value_t oldval, vlc_value_t newval, void *p_data );
 159
 160 static int KeyEvent( vlc_object_t *p_this, char const *psz_var,
 161                      vlc_value_t oldval, vlc_value_t newval, void *p_data );
 162
 163 static void stop_osdvnc ( filter_t *p_filter );
 164
 165 static void* vnc_worker_thread ( vlc_object_t *p_thread_obj );
 166
 167 static void* update_request_thread( vlc_object_t *p_thread_obj );
 168
 169 static bool open_vnc_connection ( filter_t *p_filter );
 170
 171 static bool handshaking ( filter_t *p_filter );
 172
 173 static bool process_server_message ( filter_t *p_filter,
 174                                      rfbServerToClientMsg *msg );
 175
 176 static inline void rgb_to_yuv( uint8_t *y, uint8_t *u, uint8_t *v,
 177                                int r, int g, int b );
 178
 179 static inline bool fill_rect( filter_sys_t* p_sys,
 180                               uint16_t i_x, uint16_t i_y,
 181                               uint16_t i_w, uint16_t i_h,
 182                               uint8_t i_color );
 183 static inline bool copy_rect( filter_sys_t* p_sys,
 184                               uint16_t i_x, uint16_t i_y,
 185                               uint16_t i_w, uint16_t i_h,
 186                               uint16_t i_sx, uint16_t i_sy );
 187
 188
 189 static inline bool raw_line(  filter_sys_t* p_sys,
 190                               uint16_t i_x, uint16_t i_y,
 191                               uint16_t i_w );
 192
 193 static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd );
 194
 195
 196 /*****************************************************************************
 197  * Sub filter code
 198  *****************************************************************************/
 199
 200 /*****************************************************************************
 201  * Local prototypes
 202  *****************************************************************************/
 203 struct filter_sys_t
 204 {
 205     bool          b_need_update;       /* VNC picture is updated, do update the OSD*/
 206     mtime_t       i_vnc_poll_interval; /* Update the OSD menu every n ms */
 207
 208     uint8_t       i_alpha;             /* alpha transparency value */
 209
 210     char          *psz_host;           /* VNC host */
 211     int           i_port;
 212
 213     char          *psz_passwd;         /* VNC password */
 214
 215     bool          b_vnc_poll;          /* Activate VNC polling ? */
 216     bool          b_vnc_mouse_events;  /* Send MouseEvents ? */
 217     bool          b_vnc_key_events;    /* Send KeyEvents ? */
 218
 219     bool          b_connection_active; /* Handshaking finished ? */
 220
 221     vlc_mutex_t   lock;                /* To lock for read/write on picture */
 222
 223     picture_t     *p_pic;              /* The picture with OSD data from VNC */
 224
 225     vout_thread_t *p_vout;             /* Pointer to video-out thread */
 226
 227     int           i_socket;            /* Socket used for VNC */
 228
 229     uint16_t      i_vnc_width;          /* The with of the VNC screen */
 230     uint16_t      i_vnc_height;         /* The height of the VNC screen */
 231         uint32_t      i_vnc_pixels;         /* The pixels of the VNC screen */
 232
 233     bool    b_alpha_from_vnc;    /* Special ffnetdev alpha feature enabled ? */
 234
 235     char          read_buffer[READ_BUFFER_SIZE];
 236
 237     bool          b_continue;
 238
 239     vlc_object_t* p_worker_thread;
 240
 241     uint8_t       ar_color_table_yuv[256][4];
 242 };
 243
 244 /*****************************************************************************
 245  * CreateFilter: Create the filter and open the definition file
 246  *****************************************************************************/
 247 static int CreateFilter ( vlc_object_t *p_this )
 248 {
 249     filter_t *p_filter = (filter_t *)p_this;
 250     filter_sys_t *p_sys = NULL;
 251
 252     msg_Dbg( p_filter, "Creating vnc osd filter..." );
 253
 254     p_filter->p_sys = p_sys = calloc( 1, sizeof(*p_sys) );
 255     if( !p_filter->p_sys )
 256         return VLC_ENOMEM;
 257
 258     /* Populating struct */
 259     vlc_mutex_init( &p_sys->lock );
 260     p_sys->b_continue = true;
 261     p_sys->i_socket = -1;
 262     p_sys->p_pic = NULL;
 263
 264     p_sys->psz_host = var_CreateGetString( p_this, RMTOSD_CFG "host" );
 265     if( EMPTY_STR(p_sys->psz_host) )
 266     {
 267         msg_Err( p_filter, "unable to get vnc host" );
 268         goto error;
 269     }
 270
 271     p_sys->psz_passwd = var_CreateGetString( p_this, RMTOSD_CFG "password" );
 272     if( !p_sys->psz_passwd )
 273     {
 274         msg_Err( p_filter, "unable to get vnc password" );
 275         goto error;
 276     }
 277
 278     p_sys->i_port = var_CreateGetIntegerCommand( p_this, RMTOSD_CFG "port" );
 279
 280     p_sys->i_alpha = var_CreateGetIntegerCommand( p_this, RMTOSD_CFG "alpha" );
 281
 282     /* in milliseconds, 0 disables polling, should not be lower than 100 */
 283     p_sys->i_vnc_poll_interval  = var_CreateGetIntegerCommand( p_this,
 284                                                        RMTOSD_CFG "update" );
 285     if ( p_sys->i_vnc_poll_interval < 100)
 286     {
 287        p_sys->i_vnc_poll_interval = 100;
 288     }
 289
 290     for ( int i = 0; i < 256; i++ )
 291     {
 292         p_sys->ar_color_table_yuv[i][0] = 255;
 293         p_sys->ar_color_table_yuv[i][1] = 255;
 294         p_sys->ar_color_table_yuv[i][2] = 255;
 295         p_sys->ar_color_table_yuv[i][3] = 255;
 296     }
 297
 298     p_sys->b_vnc_poll = var_CreateGetBoolCommand( p_this,
 299                                             RMTOSD_CFG "vnc-polling" );
 300     p_sys->b_vnc_mouse_events = var_CreateGetBoolCommand( p_this,
 301                                             RMTOSD_CFG "mouse-events" );
 302     p_sys->b_vnc_key_events = var_CreateGetBoolCommand( p_this,
 303                                             RMTOSD_CFG "key-events" );
 304
 305     /* Keep track of OSD Events */
 306     p_sys->b_need_update  = false;
 307
 308     /* Attach subpicture filter callback */
 309     p_filter->pf_sub_filter = Filter;
 310
 311     p_sys->p_vout = vlc_object_find( p_this, VLC_OBJECT_VOUT, FIND_PARENT );
 312
 313     if( p_sys->p_vout )
 314     {
 315         var_AddCallback( p_sys->p_vout, "mouse-moved",
 316                          MouseEvent, p_this );
 317         var_AddCallback( p_sys->p_vout, "mouse-button-down",
 318                          MouseEvent, p_this );
 319         var_AddCallback( p_sys->p_vout->p_libvlc, "key-pressed",
 320                          KeyEvent, p_this );
 321     }
 322
 323     es_format_Init( &p_filter->fmt_out, SPU_ES, VLC_CODEC_SPU );
 324     p_filter->fmt_out.i_priority = 0;
 325
 326     vlc_gcrypt_init();
 327
 328     /* create the vnc worker thread */
 329     p_sys->p_worker_thread = vlc_object_create( p_this,
 330                                                 sizeof( vlc_object_t ) );
 331     vlc_object_attach( p_sys->p_worker_thread, p_this );
 332     if( vlc_thread_create( p_sys->p_worker_thread, "vnc worker thread",
 333                            vnc_worker_thread, VLC_THREAD_PRIORITY_LOW ) )
 334     {
 335         vlc_object_detach( p_sys->p_worker_thread );
 336         vlc_object_release( p_sys->p_worker_thread );
 337         msg_Err( p_filter, "cannot spawn vnc message reader thread" );
 338         goto error;
 339     }
 340
 341     msg_Dbg( p_filter, "osdvnc filter started" );
 342
 343     return VLC_SUCCESS;
 344
 345 error:
 346     msg_Err( p_filter, "osdvnc filter discarded" );
 347
 348     stop_osdvnc( p_filter );
 349
 350     vlc_mutex_destroy( &p_sys->lock );
 351     free( p_sys->psz_host );
 352     free( p_sys->psz_passwd );
 353     free( p_sys );
 354
 355     return VLC_EGENERIC;
 356 }
 357
 358 /*****************************************************************************
 359  * DestroyFilter: Make a clean exit of this plugin
 360  *****************************************************************************/
 361 static void DestroyFilter( vlc_object_t *p_this )
 362 {
 363     filter_t     *p_filter = (filter_t*)p_this;
 364     filter_sys_t *p_sys = p_filter->p_sys;
 365
 366     msg_Dbg( p_filter, "DestroyFilter called." );
 367
 368     stop_osdvnc( p_filter );
 369
 370     if( p_sys->p_vout )
 371     {
 372         var_DelCallback( p_sys->p_vout, "mouse-moved",
 373                          MouseEvent, p_this );
 374         var_DelCallback( p_sys->p_vout, "mouse-button-down",
 375                          MouseEvent, p_this );
 376         var_DelCallback( p_sys->p_vout->p_libvlc, "key-pressed",
 377                          KeyEvent, p_this );
 378
 379         vlc_object_release( p_sys->p_vout );
 380     }
 381
 382     var_Destroy( p_this, RMTOSD_CFG "host" );
 383     var_Destroy( p_this, RMTOSD_CFG "port" );
 384     var_Destroy( p_this, RMTOSD_CFG "password" );
 385     var_Destroy( p_this, RMTOSD_CFG "update" );
 386     var_Destroy( p_this, RMTOSD_CFG "vnc-polling" );
 387     var_Destroy( p_this, RMTOSD_CFG "mouse-events" );
 388     var_Destroy( p_this, RMTOSD_CFG "key-events" );
 389     var_Destroy( p_this, RMTOSD_CFG "alpha" );
 390
 391     vlc_mutex_destroy( &p_sys->lock );
 392     free( p_sys->psz_host );
 393     free( p_sys->psz_passwd );
 394     free( p_sys );
 395 }
 396
 397 static void stop_osdvnc ( filter_t *p_filter )
 398 {
 399     filter_sys_t *p_sys = p_filter->p_sys;
 400
 401     /* It will unlock socket reading */
 402     vlc_object_kill( p_filter );
 403
 404     /* */
 405     if( p_sys->p_worker_thread )
 406     {
 407         msg_Dbg( p_filter, "joining worker_thread" );
 408         vlc_object_kill( p_sys->p_worker_thread );
 409         vlc_thread_join( p_sys->p_worker_thread );
 410         vlc_object_detach( p_sys->p_worker_thread );
 411         vlc_object_release( p_sys->p_worker_thread );
 412         msg_Dbg( p_filter, "released worker_thread" );
 413     }
 414
 415     msg_Dbg( p_filter, "osdvnc stopped" );
 416 }
 417
 418 static bool read_exact( filter_t *p_filter,
 419                         int i_socket,
 420                         char* p_readbuf,
 421                         int i_bytes )
 422 {
 423     return i_bytes == net_Read( p_filter, i_socket, NULL,
 424                                   (unsigned char*)p_readbuf,
 425                                   i_bytes, true );
 426 }
 427
 428
 429 static bool write_exact( filter_t *p_filter,
 430                          int i_socket,
 431                          char* p_writebuf,
 432                          int i_bytes )
 433 {
 434     return i_bytes == net_Write( p_filter, i_socket, NULL,
 435                                   (unsigned char*)p_writebuf, i_bytes );
 436 }
 437
 438 static bool open_vnc_connection ( filter_t *p_filter )
 439 {
 440     filter_sys_t *p_sys = p_filter->p_sys;
 441
 442     msg_Dbg( p_filter, "Open socket to vnc server on %s:%u.",
 443               p_sys->psz_host, p_sys->i_port );
 444
 445     p_sys->i_socket = net_ConnectTCP( p_filter, p_sys->psz_host, p_sys->i_port );
 446
 447     if( p_sys->i_socket < 0 )
 448     {
 449         msg_Err( p_filter, "Could not open socket" );
 450         return false;
 451     }
 452
 453     msg_Dbg( p_filter, "socket is open." );
 454
 455     return true;
 456 }
 457
 458 static bool handshaking ( filter_t *p_filter )
 459 {
 460     filter_sys_t *p_sys = p_filter->p_sys;
 461
 462     msg_Dbg( p_filter, "Reading protocol version" );
 463
 464     rfbProtocolVersionMsg pv;
 465     if ( !read_exact( p_filter, p_sys->i_socket, pv,
 466                       sz_rfbProtocolVersionMsg ) )
 467     {
 468         msg_Err( p_filter, "Could not read version message" );
 469         return false;
 470     }
 471     pv[sz_rfbProtocolVersionMsg] = '\0'; /* pv size is sz_rfbProtocolVersionMsg+1 */
 472
 473     msg_Dbg( p_filter, "Server version is %s", pv );
 474
 475     strncpy(pv, "RFB 003.003\n", sz_rfbProtocolVersionMsg);
 476
 477     if( !write_exact(p_filter, p_sys->i_socket, pv,
 478                      sz_rfbProtocolVersionMsg) )
 479     {
 480         msg_Err( p_filter, "Could not write version message" );
 481         return false;
 482     }
 483
 484     msg_Dbg( p_filter, "Reading authentication scheme" );
 485     uint32_t i_authScheme;
 486     if( !read_exact( p_filter, p_sys->i_socket, (char*)&i_authScheme, 4 ) )
 487     {
 488         msg_Err( p_filter, "Could not read authentication scheme" );
 489         return false;
 490     }
 491     i_authScheme = htonl(i_authScheme);
 492
 493     msg_Dbg( p_filter, "Authentication scheme = %x", i_authScheme );
 494     if ( i_authScheme == rfbConnFailed )
 495     {
 496         msg_Err( p_filter, "Connection rejected by server" );
 497         return false;
 498     }
 499     if (i_authScheme == rfbVncAuth)
 500     {
 501         unsigned char challenge[CHALLENGESIZE];
 502         if ( !read_exact( p_filter, p_sys->i_socket,
 503                           (char*)challenge, CHALLENGESIZE ) )
 504         {
 505             msg_Err( p_filter, "Could not read password challenge" );
 506             return false;
 507         }
 508
 509         vnc_encrypt_bytes( challenge, p_sys->psz_passwd );
 510
 511         if( !write_exact(p_filter, p_sys->i_socket,
 512                          (char*)challenge, CHALLENGESIZE ) )
 513         {
 514             msg_Err( p_filter, "Could not write password" );
 515             return false;
 516         }
 517         uint32_t i_authResult;
 518         if( !read_exact( p_filter, p_sys->i_socket, (char*)&i_authResult, 4 ) )
 519         {
 520             msg_Err( p_filter, "Could not read authentication result" );
 521             return false;
 522         }
 523         i_authResult = htonl(i_authResult);
 524         if (i_authResult != rfbVncAuthOK)
 525         {
 526             msg_Err( p_filter, "VNC authentication failed" );
 527             return false;
 528         }
 529     }
 530
 531     msg_Dbg( p_filter, "Writing client init message" );
 532     rfbClientInitMsg ci;
 533     ci.shared = 1;
 534     if( !write_exact( p_filter, p_sys->i_socket,
 535                       (char*)&ci, sz_rfbClientInitMsg ) )
 536     {
 537         msg_Err( p_filter, "Could not write client init message" );
 538         return false;
 539     }
 540
 541     msg_Dbg( p_filter, "Reading server init message" );
 542     rfbServerInitMsg si;
 543     if( !read_exact( p_filter, p_sys->i_socket,
 544                      (char*)&si, sz_rfbServerInitMsg ) )
 545     {
 546         msg_Err( p_filter, "Could not read server init message" );
 547         return false;
 548     }
 549     si.framebufferWidth = htons(si.framebufferWidth);
 550     si.framebufferHeight = htons(si.framebufferHeight);
 551     si.format.redMax = htons(si.format.redMax);
 552     si.format.greenMax = htons(si.format.greenMax);
 553     si.format.blueMax = htons(si.format.blueMax);
 554
 555     p_sys->i_vnc_width = si.framebufferWidth;
 556     p_sys->i_vnc_height = si.framebufferHeight;
 557
 558     msg_Dbg( p_filter, "Servers preferred pixelformat: "
 559                         "%ux%u, R(%u),G(%u),B(%u), %u bit, depht=%u, %s",
 560                         si.framebufferWidth,
 561                         si.framebufferHeight,
 562                         si.format.redMax,
 563                         si.format.greenMax,
 564                         si.format.blueMax,
 565                         si.format.bitsPerPixel,
 566                         si.format.depth,
 567                         si.format.trueColour ? "TrueColor" : "Not-TrueColor");
 568
 569     uint32_t i_nameLength = htonl(si.nameLength);
 570     if( i_nameLength > MAX_VNC_SERVER_NAME_LENGTH )
 571     {
 572         msg_Err( p_filter, "Server name too long" );
 573         return false;
 574     }
 575     char s_ServerName[MAX_VNC_SERVER_NAME_LENGTH+1];
 576
 577     msg_Dbg( p_filter, "Reading server name with size = %u", i_nameLength );
 578     if( !read_exact( p_filter, p_sys->i_socket, s_ServerName, i_nameLength ) )
 579     {
 580         msg_Err( p_filter, "Could not read server name" );
 581         return false;
 582     }
 583     s_ServerName[i_nameLength] = '\0';
 584
 585     if( strcmp( s_ServerName, "VDR-OSD") == 0 )
 586     {
 587         msg_Dbg( p_filter, "Server is a VDR" );
 588         p_sys->b_alpha_from_vnc = true;
 589     }
 590     else
 591     {
 592         msg_Dbg( p_filter, "Server is a normal VNC" );
 593         p_sys->b_alpha_from_vnc = false;
 594     }
 595
 596
 597     msg_Dbg( p_filter, "Server init message read properly" );
 598     msg_Dbg( p_filter, "Server name is %s", s_ServerName );
 599
 600     msg_Dbg( p_filter, "Writing SetPixelFormat message" );
 601
 602     rfbSetPixelFormatMsg sp;
 603     sp.type = rfbSetPixelFormat;
 604     sp.pad1 = sp.pad2 = 0;
 605     sp.format.bitsPerPixel = 8;
 606     sp.format.depth = 8 ;
 607     sp.format.bigEndian = 1;
 608     sp.format.trueColour = 0;
 609     sp.format.redMax = htons(31);
 610     sp.format.greenMax = htons(31);
 611     sp.format.blueMax = htons(31);
 612     sp.format.redShift = 10;
 613     sp.format.greenShift = 5;
 614     sp.format.blueShift = 0;
 615     sp.format.pad1 = sp.format.pad2 = 0;
 616
 617     if( !write_exact( p_filter, p_sys->i_socket,
 618                       (char*)&sp, sz_rfbSetPixelFormatMsg) )
 619     {
 620         msg_Err( p_filter, "Could not write SetPixelFormat message" );
 621         return false;
 622     }
 623
 624     msg_Dbg( p_filter, "Writing SetEncodings message" );
 625
 626     rfbSetEncodingsMsg se;
 627     se.type = rfbSetEncodings;
 628     se.pad = 0;
 629     se.nEncodings = htons( p_sys->b_alpha_from_vnc ? 3 : 2 );
 630
 631     if( !write_exact( p_filter, p_sys->i_socket,
 632                       (char*)&se, sz_rfbSetEncodingsMsg) )
 633     {
 634         msg_Err( p_filter, "Could not write SetEncodings message begin" );
 635         return false;
 636     }
 637
 638     uint32_t i_encoding;
 639
 640     msg_Dbg( p_filter, "Writing SetEncodings rfbEncodingCopyRect" );
 641     i_encoding = htonl(rfbEncodingCopyRect);
 642     if( !write_exact( p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
 643     {
 644         msg_Err( p_filter, "Could not write encoding type rfbEncodingCopyRect." );
 645         return false;
 646     }
 647
 648     msg_Dbg( p_filter, "Writing SetEncodings rfbEncodingRRE" );
 649     i_encoding = htonl(rfbEncodingRRE);
 650     if( !write_exact(p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
 651     {
 652         msg_Err( p_filter, "Could not write encoding type rfbEncodingRRE." );
 653         return false;
 654     }
 655
 656     if( p_sys->b_alpha_from_vnc )
 657     {
 658         msg_Dbg( p_filter, "Writing SetEncodings rfbEncSpecialUseAlpha" );
 659         i_encoding = 0x00F0FFFF; /* rfbEncSpecialUseAlpha is 0xFFFFF000
 660                                   * before we swap it */
 661         if( !write_exact(p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
 662         {
 663             msg_Err( p_filter, "Could not write encoding type rfbEncSpecialUseAlpha." );
 664             return false;
 665         }
 666     }
 667     return true;
 668
 669 }
 670
 671 static void* vnc_worker_thread( vlc_object_t *p_thread_obj )
 672 {
 673     filter_t* p_filter = (filter_t*)(p_thread_obj->p_parent);
 674     filter_sys_t *p_sys = p_filter->p_sys;
 675     vlc_object_t *p_update_request_thread;
 676     int canc = vlc_savecancel ();
 677
 678     msg_Dbg( p_filter, "VNC worker thread started" );
 679
 680     if( !open_vnc_connection ( p_filter ) )
 681     {
 682         msg_Err( p_filter, "Could not connect to vnc host" );
 683         goto exit;
 684     }
 685
 686     if( !handshaking ( p_filter ) )
 687     {
 688         msg_Err( p_filter, "Error occured while handshaking vnc host" );
 689         goto exit;
 690     }
 691
 692     p_sys->b_connection_active = true; /* to enable sending key
 693                                             * and mouse events to host */
 694
 695     /* Create an empty picture for VNC the data */
 696     vlc_mutex_lock( &p_sys->lock );
 697     p_sys->p_pic = picture_New( VLC_CODEC_YUVA,
 698                                 p_sys->i_vnc_width, p_sys->i_vnc_height, 1, 1 );
 699     if( !p_sys->p_pic )
 700     {
 701         vlc_mutex_unlock( &p_sys->lock );
 702         goto exit;
 703     }
 704         p_sys->i_vnc_pixels = p_sys->i_vnc_width * p_sys->i_vnc_height;
 705
 706     vlc_mutex_unlock( &p_sys->lock );
 707
 708     /* create the update request thread */
 709     p_update_request_thread = vlc_object_create( p_filter,
 710                                                  sizeof( vlc_object_t ) );
 711     vlc_object_attach( p_update_request_thread, p_filter );
 712     if( vlc_thread_create( p_update_request_thread,
 713                            "vnc update request thread",
 714                            update_request_thread, VLC_THREAD_PRIORITY_LOW ) )
 715     {
 716         vlc_object_detach( p_update_request_thread );
 717         vlc_object_release( p_update_request_thread );
 718         msg_Err( p_filter, "cannot spawn vnc update request thread" );
 719         goto exit;
 720     }
 721
 722     /* connection is initialized, now read and handle server messages */
 723     while( vlc_object_alive( p_thread_obj ) )
 724     {
 725         rfbServerToClientMsg msg;
 726         int i_msgSize;
 727
 728         memset( &msg, 0, sizeof(msg) );
 729
 730         if( !read_exact(p_filter, p_sys->i_socket, (char*)&msg, 1 ) )
 731         {
 732             msg_Err( p_filter, "Error while waiting for next server message");
 733             break;
 734         }
 735         switch (msg.type)
 736         {
 737         case rfbFramebufferUpdate:
 738             i_msgSize = sz_rfbFramebufferUpdateMsg;
 739             break;
 740         case rfbSetColourMapEntries:
 741             i_msgSize = sz_rfbSetColourMapEntriesMsg;
 742             break;
 743         case rfbBell:
 744             i_msgSize = sz_rfbBellMsg;
 745             break;
 746         case rfbServerCutText:
 747             i_msgSize = sz_rfbServerCutTextMsg;
 748             break;
 749         case rfbReSizeFrameBuffer:
 750             i_msgSize = sz_rfbReSizeFrameBufferMsg;
 751             break;
 752         default:
 753             i_msgSize = 0;
 754             msg_Err( p_filter, "Invalid message %u received", msg.type );
 755             break;
 756         }
 757
 758         if( i_msgSize <= 0 )
 759             break;
 760
 761         if( --i_msgSize > 0 )
 762         {
 763             if ( !read_exact( p_filter, p_sys->i_socket,
 764                               ((char*)&msg)+1, i_msgSize ) )
 765             {
 766                 msg_Err( p_filter, "Error while reading message of type %u",
 767                          msg.type );
 768                 break;
 769             }
 770         }
 771         process_server_message( p_filter, &msg);
 772     }
 773
 774     msg_Dbg( p_filter, "joining update_request_thread" );
 775     vlc_object_kill( p_update_request_thread );
 776     vlc_thread_join( p_update_request_thread );
 777     vlc_object_detach( p_update_request_thread );
 778     vlc_object_release( p_update_request_thread );
 779     msg_Dbg( p_filter, "released update_request_thread" );
 780
 781 exit:
 782
 783     vlc_mutex_lock( &p_sys->lock );
 784     p_sys->b_connection_active = false;
 785
 786     if (p_sys->i_socket >= 0)
 787         net_Close(p_sys->i_socket);
 788
 789     if( p_sys->p_pic )
 790         picture_Release( p_sys->p_pic );
 791
 792     /* It will hide the subtitle */
 793     p_sys->b_continue = false;
 794     p_sys->b_need_update = true;
 795     vlc_mutex_unlock( &p_sys->lock );
 796
 797     msg_Dbg( p_filter, "VNC message reader thread ended" );
 798     vlc_restorecancel (canc);
 799     return NULL;
 800 }
 801
 802 static void* update_request_thread( vlc_object_t *p_thread_obj )
 803 {
 804     filter_t* p_filter = (filter_t*)(p_thread_obj->p_parent);
 805     filter_sys_t *p_sys = p_filter->p_sys;
 806     int canc = vlc_savecancel ();
 807
 808     msg_Dbg( p_filter, "VNC update request thread started" );
 809
 810     rfbFramebufferUpdateRequestMsg udr;
 811     udr.type = rfbFramebufferUpdateRequest;
 812     udr.incremental = 0;
 813     udr.x = 0;
 814     udr.y = 0;
 815     udr.w = htons(p_sys->i_vnc_width);
 816     udr.h = htons(p_sys->i_vnc_height);
 817
 818     if( write_exact(p_filter, p_sys->i_socket, (char*)&udr,
 819            sz_rfbFramebufferUpdateRequestMsg) == false)
 820     {
 821         msg_Err( p_filter, "Could not write rfbFramebufferUpdateRequestMsg." );
 822         p_sys->b_continue = false;
 823         return NULL;
 824     }
 825
 826     udr.incremental = 1;
 827     mtime_t i_poll_interval_microsec = p_sys->i_vnc_poll_interval * 1000;
 828
 829     if( p_sys->b_vnc_poll)
 830     {
 831         while( vlc_object_alive( p_thread_obj ) )
 832         {
 833             msleep( i_poll_interval_microsec );
 834             if( write_exact(p_filter, p_sys->i_socket, (char*)&udr,
 835                    sz_rfbFramebufferUpdateRequestMsg) == false)
 836             {
 837                 msg_Err( p_filter, "Could not write rfbFramebufferUpdateRequestMsg." );
 838                 break;
 839             }
 840         }
 841         p_sys->b_continue = false;
 842     }
 843     else
 844     {
 845         msg_Dbg( p_filter, "VNC polling disabled." );
 846     }
 847
 848     msg_Dbg( p_filter, "VNC update request thread ended" );
 849     vlc_restorecancel (canc);
 850     return NULL;
 851 }
 852
 853 static bool process_server_message ( filter_t *p_filter,
 854                                      rfbServerToClientMsg *msg )
 855 {
 856     filter_sys_t *p_sys = p_filter->p_sys;
 857
 858     switch (msg->type)
 859     {
 860     case rfbFramebufferUpdate:
 861         {
 862             msg->fu.nRects = htons(msg->fu.nRects);
 863             rfbFramebufferUpdateRectHeader hdr;
 864
 865             for (int i_rect = 0; i_rect < msg->fu.nRects; i_rect++)
 866             {
 867                 if (!read_exact(p_filter, p_sys->i_socket, (char*)&hdr,
 868                     sz_rfbFramebufferUpdateRectHeader ) )
 869                 {
 870                     msg_Err( p_filter, "Could not read FrameBufferUpdate header" );
 871                     return false;
 872                 }
 873                 hdr.r.x = htons(hdr.r.x);
 874                 hdr.r.y = htons(hdr.r.y);
 875                 hdr.r.w = htons(hdr.r.w);
 876                 hdr.r.h = htons(hdr.r.h);
 877                 hdr.encoding = htonl(hdr.encoding);
 878
 879                 switch (hdr.encoding)
 880                 {
 881                 case rfbEncodingRaw:
 882                     {
 883                         int i_line;
 884                         for (i_line = 0; i_line < hdr.r.h; i_line++)
 885                         {
 886                             if ( !read_exact( p_filter, p_sys->i_socket,
 887                                     p_sys->read_buffer, hdr.r.w ) )
 888                             {
 889                                 msg_Err( p_filter,
 890                                  "Could not read FrameBufferUpdate line data" );
 891                                return false;
 892                             }
 893                             vlc_mutex_lock( &p_sys->lock );
 894                             if ( !raw_line( p_sys, hdr.r.x,
 895                                             hdr.r.y + i_line,
 896                                             hdr.r.w ) )
 897                             {
 898                                 msg_Err( p_filter, "raw_line failed." );
 899                                 vlc_mutex_unlock( &p_sys->lock );
 900                                 return false;
 901                             }
 902                             vlc_mutex_unlock( &p_sys->lock );
 903                         }
 904                     }
 905                     break;
 906
 907                 case rfbEncodingCopyRect:
 908                     {
 909                         rfbCopyRect rect;
 910
 911                         if ( !read_exact( p_filter, p_sys->i_socket,
 912                                           (char*)&rect,
 913                                           sz_rfbCopyRect ) )
 914                         {
 915                             msg_Err( p_filter, "Could not read rfbCopyRect" );
 916                             return false;
 917                         }
 918                         rect.srcX = htons( rect.srcX );
 919                         rect.srcY = htons( rect.srcY );
 920
 921                         vlc_mutex_lock( &p_sys->lock );
 922                         if ( !copy_rect( p_sys,
 923                                          hdr.r.x,   hdr.r.y,
 924                                          hdr.r.w,   hdr.r.h,
 925                                          rect.srcX, rect.srcY ) )
 926                         {
 927                             msg_Err( p_filter, "copy_rect failed." );
 928                             vlc_mutex_unlock( &p_sys->lock );
 929                             return false;
 930                         }
 931                         vlc_mutex_unlock( &p_sys->lock );
 932                     }
 933                     break;
 934
 935                 case rfbEncodingRRE:
 936                     {
 937                         rfbRREHeader rrehdr;
 938                         if ( !read_exact( p_filter, p_sys->i_socket,
 939                                           (char*)&rrehdr,
 940                                           sz_rfbRREHeader ) )
 941                         {
 942                             msg_Err( p_filter, "Could not read rfbRREHeader" );
 943                             return false;
 944                         }
 945                         uint8_t i_pixcolor;
 946                         if ( !read_exact(p_filter, p_sys->i_socket,
 947                                          (char*)&i_pixcolor, 1 ) )
 948                         {
 949                             msg_Err( p_filter, "Could not read RRE pixcolor" );
 950                             return false;
 951                         }
 952
 953                         vlc_mutex_lock( &p_sys->lock );
 954                         if ( !fill_rect( p_sys,
 955                                         hdr.r.x, hdr.r.y,
 956                                         hdr.r.w, hdr.r.h,
 957                                         i_pixcolor) )
 958                         {
 959                             msg_Err( p_filter, "main fill_rect failed." );
 960                             vlc_mutex_unlock( &p_sys->lock );
 961                             return false;
 962                         }
 963                         vlc_mutex_unlock( &p_sys->lock );
 964
 965                         rrehdr.nSubrects = htonl(rrehdr.nSubrects);
 966
 967                         int i_datasize = rrehdr.nSubrects *
 968                                      ( sizeof(i_pixcolor) + sz_rfbRectangle ) ;
 969                         if ( i_datasize > READ_BUFFER_SIZE )
 970                         {
 971                             msg_Err( p_filter, "Buffer too small, "
 972                                      "need %u bytes", i_datasize );
 973                             return false;
 974                         }
 975                         if ( !read_exact( p_filter, p_sys->i_socket,
 976                                        p_sys->read_buffer, i_datasize ) )
 977                         {
 978                             msg_Err( p_filter,
 979                                      "Could not read RRE subrect data" );
 980                             return false;
 981                         }
 982
 983                         uint32_t i_subrect;
 984                         rfbRectangle* p_subrect;
 985                         int i_offset = 0;
 986                         vlc_mutex_lock( &p_sys->lock );
 987                         for ( i_subrect = 0;
 988                               i_subrect < rrehdr.nSubrects; i_subrect++)
 989                         {
 990                             i_pixcolor = p_sys->read_buffer[i_offset];
 991                             i_offset += sizeof(i_pixcolor);
 992                             p_subrect =
 993                                (rfbRectangle*)(p_sys->read_buffer + i_offset);
 994                             i_offset += sz_rfbRectangle;
 995
 996                             if (!fill_rect( p_sys,
 997                                             htons(p_subrect->x) + hdr.r.x,
 998                                             htons(p_subrect->y) + hdr.r.y,
 999                                             htons(p_subrect->w),
1000                                             htons(p_subrect->h),
1001                                             i_pixcolor) )
1002                             {
1003                                 msg_Err( p_filter,
1004                                     "subrect %u fill_rect failed.", i_subrect );
1005                                 vlc_mutex_unlock( &p_sys->lock );
1006                                 return false;
1007                             }
1008                         }
1009                         vlc_mutex_unlock( &p_sys->lock );
1010                     }
1011                     break;
1012
1013                 }
1014
1015             }
1016             vlc_mutex_lock( &p_sys->lock );
1017             p_sys->b_need_update = true;
1018             vlc_mutex_unlock( &p_sys->lock );
1019         }
1020         return true;
1021
1022     case rfbSetColourMapEntries:
1023         {
1024             msg->scme.nColours = htons(msg->scme.nColours);
1025             msg->scme.firstColour = htons(msg->scme.firstColour);
1026             int i_datasize;
1027             if ( p_sys->b_alpha_from_vnc == true )
1028             {
1029                 i_datasize = 2 * msg->scme.nColours * 4;
1030             }
1031             else
1032             {
1033                 i_datasize = 2 * msg->scme.nColours * 3;
1034             }
1035             if ( i_datasize > READ_BUFFER_SIZE )
1036             {
1037                 msg_Err( p_filter, "Buffer too small, need %u bytes",
1038                                    i_datasize );
1039                 return false;
1040             }
1041
1042             if ( !read_exact( p_filter, p_sys->i_socket,
1043                               p_sys->read_buffer, i_datasize ) )
1044             {
1045                 msg_Err( p_filter, "Could not read color map data" );
1046                 return false;
1047             }
1048
1049             uint8_t i_red, i_green, i_blue, i_alpha, i_color_index;
1050             uint16_t i_offset = 0;
1051             i_alpha = 255;
1052
1053             for (int i = 0; i < msg->scme.nColours; i++)
1054             {
1055                 i_color_index = i+msg->scme.firstColour;
1056                 if ( p_sys->b_alpha_from_vnc == true )
1057                 {
1058                     i_alpha = p_sys->read_buffer[i_offset];
1059                     i_offset += 2;
1060                 }
1061                 i_red   = p_sys->read_buffer[i_offset];
1062                 i_offset += 2;
1063                 i_green = p_sys->read_buffer[i_offset];
1064                 i_offset += 2;
1065                 i_blue  = p_sys->read_buffer[i_offset];
1066                 i_offset += 2;
1067                 rgb_to_yuv( &p_sys->ar_color_table_yuv[i_color_index][0],
1068                             &p_sys->ar_color_table_yuv[i_color_index][1],
1069                             &p_sys->ar_color_table_yuv[i_color_index][2],
1070                             i_red,
1071                             i_green,
1072                             i_blue );
1073                 p_sys->ar_color_table_yuv[i][3] = i_alpha;
1074             }
1075         }
1076         return true;
1077
1078     case rfbBell:
1079         msg_Err( p_filter, "rfbBell received" );
1080         return true;
1081
1082     case rfbServerCutText:
1083         msg->sct.length = htons(msg->sct.length);
1084         if ( msg->sct.length > READ_BUFFER_SIZE )
1085         {
1086             msg_Err( p_filter, "Buffer too small, need %u bytes", msg->sct.length );
1087             return false;
1088         }
1089         if ( !read_exact(p_filter, p_sys->i_socket,
1090                          p_sys->read_buffer, msg->sct.length ) )
1091         {
1092             msg_Err( p_filter, "Could not read Reading rfbServerCutText data" );
1093             return false;
1094         }
1095         return true;
1096
1097     case rfbReSizeFrameBuffer:
1098         msg_Err( p_filter, "Reading rfbReSizeFrameBuffer not implemented" );
1099         return false;
1100
1101     default:
1102         msg_Err( p_filter, "Invalid message %u received", msg->type );
1103         return false;
1104     }
1105     return false;
1106 }
1107
1108 /****************************************************************************
1109  * Filter: the whole thing
1110  ****************************************************************************
1111  * This function outputs subpictures at regular time intervals.
1112  ****************************************************************************/
1113 static subpicture_t *Filter( filter_t *p_filter, mtime_t date )
1114 {
1115     filter_sys_t *p_sys = p_filter->p_sys;
1116     subpicture_t *p_spu;
1117     subpicture_region_t *p_region;
1118     video_format_t fmt;
1119     picture_t *p_pic;
1120
1121     if( !p_sys->b_need_update )
1122     {
1123         return NULL;
1124     }
1125
1126     vlc_mutex_lock( &p_sys->lock );
1127
1128     p_pic = p_sys->p_pic;
1129
1130     if( !p_pic )
1131     {
1132         vlc_mutex_unlock( &p_sys->lock );
1133         return NULL;
1134     }
1135
1136     /* Allocate the subpicture internal data. */
1137     p_spu = filter_NewSubpicture( p_filter );
1138     if( !p_spu )
1139     {
1140         vlc_mutex_unlock( &p_sys->lock );
1141         return NULL;
1142     }
1143
1144     p_spu->b_absolute = false;
1145     p_spu->i_start = date;
1146     p_spu->i_stop = 0;
1147     p_spu->b_ephemer = true;
1148
1149     if( !p_sys->b_continue )
1150         p_spu->i_stop = p_spu->i_start + 1;
1151
1152     /* Create new SPU region */
1153     memset( &fmt, 0, sizeof(video_format_t) );
1154     fmt.i_chroma = VLC_CODEC_YUVA;
1155     fmt.i_aspect = VOUT_ASPECT_FACTOR;
1156     fmt.i_sar_num = fmt.i_sar_den = 1;
1157     fmt.i_width = fmt.i_visible_width = p_pic->p[Y_PLANE].i_visible_pitch;
1158     fmt.i_height = fmt.i_visible_height = p_pic->p[Y_PLANE].i_visible_lines;
1159     fmt.i_x_offset = fmt.i_y_offset = 0;
1160     p_region = subpicture_region_New( &fmt );
1161     if( !p_region )
1162     {
1163         msg_Err( p_filter, "cannot allocate SPU region" );
1164         p_filter->pf_sub_buffer_del( p_filter, p_spu );
1165         vlc_mutex_unlock( &p_sys->lock );
1166         return NULL;
1167     }
1168
1169     /* FIXME the copy is probably not needed anymore */
1170     picture_Copy( p_region->p_picture, p_pic );
1171
1172     p_sys->b_need_update = false;
1173
1174     vlc_mutex_unlock( &p_sys->lock );
1175
1176     /* set to one of the 9 relative locations */
1177     p_region->i_align = 0; /* Center */
1178     p_spu->b_absolute = false;
1179
1180
1181     p_spu->i_original_picture_width = 0; /*Let vout core do the horizontal scaling */
1182     p_spu->i_original_picture_height = fmt.i_height;
1183
1184     p_spu->p_region = p_region;
1185
1186     p_spu->i_alpha = ( p_sys->i_alpha );
1187
1188     return p_spu;
1189 }
1190
1191
1192 static inline void rgb_to_yuv( uint8_t *y, uint8_t *u, uint8_t *v,
1193                                int r, int g, int b )
1194 {
1195     *y = ( ( (  66 * r + 129 * g +  25 * b + 128 ) >> 8 ) + 16 );
1196     *u =   ( ( -38 * r -  74 * g + 112 * b + 128 ) >> 8 ) + 128 ;
1197     *v =   ( ( 112 * r -  94 * g -  18 * b + 128 ) >> 8 ) + 128 ;
1198 }
1199
1200 static inline bool fill_rect( filter_sys_t* p_sys,
1201                               uint16_t i_x, uint16_t i_y,
1202                               uint16_t i_w, uint16_t i_h,
1203                               uint8_t i_color)
1204 {
1205     plane_t *p_outY = p_sys->p_pic->p+Y_PLANE;
1206     plane_t *p_outU = p_sys->p_pic->p+U_PLANE;
1207     plane_t *p_outV = p_sys->p_pic->p+V_PLANE;
1208     plane_t *p_outA = p_sys->p_pic->p+A_PLANE;
1209     int i_pitch = p_outY->i_pitch;
1210     int i_lines = p_outY->i_lines;
1211     if ( i_x + i_w > i_pitch)
1212         return false;
1213     if ( i_y + i_h > i_lines)
1214         return false;
1215     int i_line_offset = i_y * i_pitch;
1216     uint8_t i_yuv_y = p_sys->ar_color_table_yuv[i_color][0];
1217     uint8_t i_yuv_u = p_sys->ar_color_table_yuv[i_color][1];
1218     uint8_t i_yuv_v = p_sys->ar_color_table_yuv[i_color][2];
1219     uint8_t i_alpha = p_sys->ar_color_table_yuv[i_color][3];
1220     for( int i_line = 0; i_line < i_h; i_line++ )
1221     {
1222         for( int i_column = 0; i_column < i_w; i_column++ )
1223         {
1224             int i_total_offset = i_line_offset + i_x + i_column;
1225             p_outY->p_pixels[ i_total_offset ] = i_yuv_y;
1226             p_outU->p_pixels[ i_total_offset ] = i_yuv_u;
1227             p_outV->p_pixels[ i_total_offset ] = i_yuv_v;
1228             p_outA->p_pixels[ i_total_offset ] = i_alpha;
1229         }
1230         i_line_offset += i_pitch;
1231     }
1232     return true;
1233 }
1234
1235 static inline bool copy_rect( filter_sys_t* p_sys,
1236                               uint16_t i_x, uint16_t i_y,
1237                               uint16_t i_w, uint16_t i_h,
1238                               uint16_t i_sx, uint16_t i_sy )
1239 {
1240     plane_t *p_Y = p_sys->p_pic->p+Y_PLANE;
1241     plane_t *p_U = p_sys->p_pic->p+U_PLANE;
1242     plane_t *p_V = p_sys->p_pic->p+V_PLANE;
1243     plane_t *p_A = p_sys->p_pic->p+A_PLANE;
1244
1245     int i_pitch = p_Y->i_pitch;
1246     int i_lines = p_Y->i_lines;
1247
1248     fprintf( stderr, "copy_rect: (%d,%d)+(%d,%d) -> (%d,%d)\n", i_x, i_y, i_w, i_h, i_sx, i_sy );
1249
1250     if( i_x + i_w > i_pitch || i_sx + i_w > i_pitch )
1251         return false;
1252     if( i_y + i_h > i_lines || i_sy + i_h > i_lines)
1253         return false;
1254
1255     if( i_w <= 0 || i_h <= 0 )
1256         return true;
1257
1258     uint8_t *pb_buffer = calloc( i_w * i_h, 4 );
1259     if( !pb_buffer )
1260         return false;
1261
1262     for( int i_line = 0; i_line < i_h; i_line++ )
1263     {
1264         for( int i_column = 0; i_column < i_w; i_column++ )
1265         {
1266             const int i_src_offset = ( i_sy + i_line ) * i_pitch + i_sx + i_column;
1267             const int i_tmp_offset = (    0 + i_line ) *     i_w +    0 + i_column;
1268
1269             pb_buffer[4*i_tmp_offset + 0] = p_Y->p_pixels[i_src_offset];
1270             pb_buffer[4*i_tmp_offset + 1] = p_U->p_pixels[i_src_offset];
1271             pb_buffer[4*i_tmp_offset + 2] = p_V->p_pixels[i_src_offset];
1272             pb_buffer[4*i_tmp_offset + 3] = p_A->p_pixels[i_src_offset];
1273         }
1274     }
1275
1276     for( int i_line = 0; i_line < i_h; i_line++ )
1277     {
1278         for( int i_column = 0; i_column < i_w; i_column++ )
1279         {
1280             const int i_tmp_offset = (   0 + i_line ) *     i_w +   0 + i_column;
1281             const int i_dst_offset = ( i_y + i_line ) * i_pitch + i_x + i_column;
1282
1283             p_Y->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 0];
1284             p_U->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 1];
1285             p_V->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 2];
1286             p_A->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 3];
1287         }
1288     }
1289     free( pb_buffer );
1290     return true;
1291
1292 }
1293
1294 static inline bool raw_line(  filter_sys_t* p_sys,
1295                               uint16_t i_x, uint16_t i_y,
1296                               uint16_t i_w )
1297 {
1298     plane_t *p_outY = p_sys->p_pic->p+Y_PLANE;
1299     plane_t *p_outU = p_sys->p_pic->p+U_PLANE;
1300     plane_t *p_outV = p_sys->p_pic->p+V_PLANE;
1301     plane_t *p_outA = p_sys->p_pic->p+A_PLANE;
1302     int i_pitch = p_outY->i_pitch;
1303     int i_lines = p_outY->i_lines;
1304     if ( i_x + i_w > i_pitch)
1305         return false;
1306     if ( i_y > i_lines)
1307         return false;
1308
1309     int i_line_offset = i_y * i_pitch + i_x;
1310
1311     for( int i_column = 0; i_column < i_w; i_column++ )
1312     {
1313         int i_offset = i_line_offset + i_column;
1314         uint8_t i_color = p_sys->read_buffer[i_column];
1315         p_outY->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][0];
1316         p_outU->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][1];
1317         p_outV->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][2];
1318         p_outA->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][3];
1319     }
1320
1321     return true;
1322 }
1323
1324
1325 /*****************************************************************************
1326  * MouseEvent: callback for mouse events
1327  *****************************************************************************/
1328 static int MouseEvent( vlc_object_t *p_this, char const *psz_var,
1329                        vlc_value_t oldval, vlc_value_t newval, void *p_data )
1330 {
1331     VLC_UNUSED(oldval); VLC_UNUSED(newval); VLC_UNUSED(psz_var);
1332
1333     filter_t *p_filter = (filter_t *)p_data;
1334     filter_sys_t *p_sys = p_filter->p_sys;
1335
1336     if( !p_sys->b_vnc_mouse_events )
1337         return VLC_SUCCESS;
1338
1339     vout_thread_t *p_vout = (vout_thread_t*)p_sys->p_vout;
1340     int i_x, i_y;
1341     int i_v;
1342
1343
1344     i_v = var_GetInteger( p_sys->p_vout, "mouse-button-down" );
1345     i_y = var_GetInteger( p_sys->p_vout, "mouse-y" );
1346     i_x = var_GetInteger( p_sys->p_vout, "mouse-x" );
1347
1348     vlc_mutex_lock( &p_sys->lock );
1349
1350     const int v_h = p_vout->fmt_in.i_visible_height;
1351     const int v_w = p_sys->i_vnc_width * v_h / p_sys->i_vnc_height;
1352     const int v_x = (p_vout->fmt_in.i_visible_width-v_w)/2;
1353
1354     i_x -= v_x;
1355
1356     if( i_y < 0 || i_x < 0 || i_y >= v_h || i_x >= v_w )
1357     {
1358         vlc_mutex_unlock( &p_sys->lock );
1359         msg_Dbg( p_this, "invalid mouse event? x=%d y=%d btn=%x", i_x, i_y, i_v );
1360         return VLC_SUCCESS;
1361     }
1362     if( !p_sys->b_connection_active )
1363     {
1364         vlc_mutex_unlock( &p_sys->lock );
1365         return VLC_SUCCESS;
1366     }
1367
1368 #ifdef VNC_DEBUG
1369     msg_Dbg( p_this, "mouse event x=%d y=%d btn=%x", i_x, i_y, i_v );
1370 #endif
1371
1372     /* */
1373     i_x = i_x * p_sys->i_vnc_width / v_w;
1374     i_y = i_y * p_sys->i_vnc_height / v_h;
1375
1376     /* buttonMask bits 0-7 are buttons 1-8, 0=up, 1=down */
1377     rfbPointerEventMsg ev;
1378     ev.type = rfbPointerEvent;
1379     ev.buttonMask = i_v;
1380     ev.x = htons(i_x);
1381     ev.y = htons(i_y);
1382
1383     write_exact( p_filter, p_sys->i_socket,
1384                  (char*)&ev, sz_rfbPointerEventMsg);
1385
1386     vlc_mutex_unlock( &p_sys->lock );
1387
1388     return VLC_SUCCESS;
1389 }
1390
1391 /*****************************************************************************
1392  * KeyEvent: callback for keyboard events
1393  *****************************************************************************/
1394 static int KeyEvent( vlc_object_t *p_this, char const *psz_var,
1395                      vlc_value_t oldval, vlc_value_t newval, void *p_data )
1396 {
1397     VLC_UNUSED(psz_var); VLC_UNUSED(oldval);
1398
1399     filter_t *p_filter = (filter_t *)p_data;
1400     filter_sys_t *p_sys = p_filter->p_sys;
1401
1402     if( !p_sys->b_vnc_key_events )
1403         return VLC_SUCCESS;
1404
1405     msg_Dbg( p_this, "key pressed (%d) ", newval.i_int );
1406
1407     if ( !newval.i_int )
1408     {
1409         msg_Err( p_this, "Received invalid key event %d", newval.i_int );
1410         return VLC_EGENERIC;
1411     }
1412
1413     vlc_mutex_lock( &p_sys->lock );
1414     if( !p_sys->b_connection_active )
1415     {
1416         vlc_mutex_unlock( &p_sys->lock );
1417         return VLC_SUCCESS;
1418     }
1419
1420     uint32_t i_key32 = newval.i_int;
1421     i_key32 = htonl(i_key32);
1422     rfbKeyEventMsg ev;
1423
1424     ev.type = rfbKeyEvent;
1425     ev.down = 1;
1426     ev.pad = 0;
1427
1428     /* first key-down for modifier-keys */
1429     if (newval.i_int & KEY_MODIFIER_CTRL)
1430     {
1431         ev.key = 0xffe3;
1432         write_exact( p_filter, p_sys->i_socket,
1433                      (char*)&ev, sz_rfbKeyEventMsg);
1434     }
1435     if (newval.i_int & KEY_MODIFIER_SHIFT)
1436     {
1437         ev.key = 0xffe1;
1438         write_exact( p_filter, p_sys->i_socket,
1439                      (char*)&ev, sz_rfbKeyEventMsg);
1440     }
1441     if (newval.i_int & KEY_MODIFIER_ALT)
1442     {
1443         ev.key = 0xffe9;
1444         write_exact( p_filter, p_sys->i_socket,
1445                      (char*)&ev, sz_rfbKeyEventMsg);
1446     }
1447
1448     /* then key-down for the pressed key */
1449     ev.key = i_key32;
1450     write_exact( p_filter, p_sys->i_socket,
1451                  (char*)&ev, sz_rfbKeyEventMsg);
1452
1453     ev.down = 0;
1454
1455     /* then key-up for the pressed key */
1456     write_exact( p_filter, p_sys->i_socket,
1457                  (char*)&ev, sz_rfbKeyEventMsg);
1458
1459     /* last key-down for modifier-keys */
1460     if (newval.i_int & KEY_MODIFIER_CTRL)
1461     {
1462         ev.key = 0xffe3;
1463         write_exact( p_filter, p_sys->i_socket,
1464                      (char*)&ev, sz_rfbKeyEventMsg);
1465     }
1466     if (newval.i_int & KEY_MODIFIER_SHIFT)
1467     {
1468         ev.key = 0xffe1;
1469         write_exact( p_filter, p_sys->i_socket,
1470                      (char*)&ev, sz_rfbKeyEventMsg);
1471     }
1472     if (newval.i_int & KEY_MODIFIER_ALT)
1473     {
1474        ev.key = 0xffe9;
1475        write_exact( p_filter, p_sys->i_socket,
1476                     (char*)&ev, sz_rfbKeyEventMsg);
1477     }
1478     vlc_mutex_unlock( &p_sys->lock );
1479
1480     return VLC_SUCCESS;
1481 }
1482
1483 static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd )
1484 {
1485     unsigned char key[8];
1486     unsigned int i;
1487
1488     for (i = 0; i < 8; i++)
1489         key[i] = i < strlen( passwd ) ? passwd[i] : '\0';
1490
1491     gcry_cipher_hd_t ctx;
1492     gcry_cipher_open( &ctx, GCRY_CIPHER_DES, GCRY_CIPHER_MODE_ECB,0);
1493
1494     /* reverse bits of the key */
1495     for( i = 0 ; i < 8 ; i ++ )
1496         key[i] =
1497             (key[i] >> 7) +
1498             (((key[i] >> 6) & 0x01 ) << 1 ) +
1499             (((key[i] >> 5) & 0x01 ) << 2 ) +
1500             (((key[i] >> 4) & 0x01 ) << 3 ) +
1501             (((key[i] >> 3) & 0x01 ) << 4 ) +
1502             (((key[i] >> 2) & 0x01 ) << 5 ) +
1503             (((key[i] >> 1) & 0x01 ) << 6 ) +
1504             ((key[i] & 0x01) << 7 );
1505
1506     gcry_cipher_setkey( ctx, key, 8 );
1507     gcry_cipher_encrypt( ctx, bytes, CHALLENGESIZE, bytes, CHALLENGESIZE );
1508     gcry_cipher_close( ctx );
1509 }
1510