git.sesse.net Git - vlc/blob - modules/video_filter/remoteosd.c

   1 /*****************************************************************************
   2  * remoteosd.c: remote osd over vnc filter module
   3  *****************************************************************************
   4  * Copyright (C) 2007-2008 Matthias Bauer
   5  * $Id$
   6  *
   7  * Authors: Matthias Bauer <matthias dot bauer #_at_# gmx dot ch>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implid warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  22  *****************************************************************************/
  23
  24 /*****************************************************************************
  25  * RemoteOSD uses the RFB-Protocol of VNC to display an On-Screen-Display
  26  * menu generated by a streaming server as overlay for the streamed video.
  27  *
  28  * The streaming server that implements this is the ffnetdev plugin for VDR.
  29  * VDR (VideoDiskRecorder) is an Linux based OpenSource harddisk recorder
  30  * software.
  31  * The VDR ffnetdev plugin emulates the hardware MPEG decoder and streams the
  32  * video over the network instead of hardware video outputs.
  33  * The OSD menu of VDR is offered with the RFB protocol to a VNC client.
  34  *
  35  * In fact this video-filter is a simple VNC client that could be also used to
  36  * connect to a real VNC host.
  37  * Only 8-bit color is supported at the moment.
  38  * Using password protected VNC hosts is supported but not recommended, because
  39  * you need to insert the used password in the plugin configuration page of
  40  * VLC configuration in plain text and it's saved in plain text.
  41  *****************************************************************************/
  42
  43 //#define VNC_DEBUG
  44
  45 /*****************************************************************************
  46  * Preamble
  47  *****************************************************************************/
  48
  49 #ifdef HAVE_CONFIG_H
  50 # include "config.h"
  51 #endif
  52
  53 #include <vlc_common.h>
  54 #include <vlc_plugin.h>
  55 #include <vlc_vout.h>
  56
  57 #include <vlc_filter.h>
  58 #include <vlc_image.h>
  59 #include <vlc_keys.h>
  60
  61 #include <vlc_network.h>
  62 #include <gcrypt.h>              /* to encrypt password */
  63 #include <vlc_gcrypt.h>
  64
  65 #include "remoteosd_rfbproto.h" /* type definitions of the RFB protocol for VNC */
  66
  67 /*****************************************************************************
  68  * Module descriptor
  69  *****************************************************************************/
  70 #define READ_BUFFER_SIZE 1000000
  71
  72 #define RMTOSD_HOST_TEXT N_("VNC Host")
  73 #define RMTOSD_HOST_LONGTEXT N_( \
  74     "VNC hostname or IP address." )
  75
  76 #define RMTOSD_PORT_TEXT N_("VNC Port")
  77 #define RMTOSD_PORT_LONGTEXT N_( \
  78     "VNC port number." )
  79
  80 #define RMTOSD_PASSWORD_TEXT N_("VNC Password")
  81 #define RMTOSD_PASSWORD_LONGTEXT N_( \
  82     "VNC password." )
  83
  84 #define RMTOSD_UPDATE_TEXT N_("VNC poll interval" )
  85 #define RMTOSD_UPDATE_LONGTEXT N_( \
  86     "In this interval an update from VNC is requested, default every 300 ms. ")
  87
  88 #define RMTOSD_POLL_TEXT N_("VNC polling")
  89 #define RMTOSD_POLL_LONGTEXT N_( \
  90     "Activate VNC polling. Do NOT activate for use as VDR ffnetdev client." )
  91
  92 #define RMTOSD_MOUSE_TEXT N_("Mouse events")
  93 #define RMTOSD_MOUSE_LONGTEXT N_( \
  94     "Send mouse events to VNC host. Not needed for use as VDR ffnetdev client." )
  95
  96 #define RMTOSD_KEYS_TEXT N_("Key events")
  97 #define RMTOSD_KEYS_LONGTEXT N_( \
  98     "Send key events to VNC host." )
  99
 100 #define RMTOSD_ALPHA_TEXT N_("Alpha transparency value (default 255)")
 101 #define RMTOSD_ALPHA_LONGTEXT N_( \
 102     "The transparency of the OSD VNC can be changed by giving a value " \
 103     "between 0 and 255. A lower value specifies more transparency a higher " \
 104     "means less transparency. The default is being not transparent " \
 105     "(value 255) the minimum is fully transparent (value 0)." )
 106
 107 #define RMTOSD_CFG "rmtosd-"
 108
 109 #define RMTOSD_UPDATE_MIN     200
 110 #define RMTOSD_UPDATE_DEFAULT 1000
 111 #define RMTOSD_UPDATE_MAX     300
 112
 113 static int  CreateFilter ( vlc_object_t * );
 114 static void DestroyFilter( vlc_object_t * );
 115
 116 vlc_module_begin ()
 117     set_description( N_("Remote-OSD over VNC") )
 118     set_capability( "sub filter", 100 )
 119     set_shortname( N_("Remote-OSD") )
 120     set_category( CAT_VIDEO )
 121     set_subcategory( SUBCAT_VIDEO_SUBPIC )
 122     add_shortcut( "rmtosd" )
 123     set_callbacks( CreateFilter, DestroyFilter )
 124
 125     add_string( RMTOSD_CFG "host", "myvdr", NULL, RMTOSD_HOST_TEXT,
 126         RMTOSD_HOST_LONGTEXT, false )
 127     add_integer_with_range( RMTOSD_CFG "port", 20001, 1, 0xFFFF, NULL,
 128         RMTOSD_PORT_TEXT, RMTOSD_PORT_LONGTEXT, false )
 129     add_password( RMTOSD_CFG "password", "", NULL, RMTOSD_PASSWORD_TEXT,
 130         RMTOSD_PASSWORD_LONGTEXT, false )
 131     add_integer_with_range( RMTOSD_CFG "update", RMTOSD_UPDATE_DEFAULT,
 132         RMTOSD_UPDATE_MIN, RMTOSD_UPDATE_MAX, NULL, RMTOSD_UPDATE_TEXT,
 133         RMTOSD_UPDATE_LONGTEXT, true )
 134     add_bool( RMTOSD_CFG "vnc-polling", false, NULL,
 135               RMTOSD_POLL_TEXT , RMTOSD_POLL_LONGTEXT, false )
 136     add_bool( RMTOSD_CFG "mouse-events", false, NULL,
 137               RMTOSD_MOUSE_TEXT , RMTOSD_MOUSE_LONGTEXT, false )
 138     add_bool( RMTOSD_CFG "key-events", false, NULL,
 139               RMTOSD_KEYS_TEXT , RMTOSD_KEYS_LONGTEXT, false )
 140     add_integer_with_range( RMTOSD_CFG "alpha", 255, 0, 255, NULL,
 141         RMTOSD_ALPHA_TEXT, RMTOSD_ALPHA_LONGTEXT, true )
 142
 143 vlc_module_end ()
 144
 145
 146 /*****************************************************************************
 147  * Local prototypes
 148  *****************************************************************************/
 149 #define CHALLENGESIZE 16
 150 #define MAX_VNC_SERVER_NAME_LENGTH 255
 151
 152 /* subfilter functions */
 153 static subpicture_t *Filter( filter_t *, mtime_t );
 154
 155 static int MouseEvent( filter_t *,
 156                        const vlc_mouse_t *,
 157                        const vlc_mouse_t *,
 158                        const video_format_t * );
 159
 160 static int KeyEvent( vlc_object_t *p_this, char const *psz_var,
 161                      vlc_value_t oldval, vlc_value_t newval, void *p_data );
 162
 163 static void stop_osdvnc ( filter_t *p_filter );
 164
 165 static void* vnc_worker_thread ( vlc_object_t *p_thread_obj );
 166
 167 static void* update_request_thread( vlc_object_t *p_thread_obj );
 168
 169 static bool open_vnc_connection ( filter_t *p_filter );
 170
 171 static bool handshaking ( filter_t *p_filter );
 172
 173 static bool process_server_message ( filter_t *p_filter,
 174                                      rfbServerToClientMsg *msg );
 175
 176 static inline void rgb_to_yuv( uint8_t *y, uint8_t *u, uint8_t *v,
 177                                int r, int g, int b );
 178
 179 static inline bool fill_rect( filter_sys_t* p_sys,
 180                               uint16_t i_x, uint16_t i_y,
 181                               uint16_t i_w, uint16_t i_h,
 182                               uint8_t i_color );
 183 static inline bool copy_rect( filter_sys_t* p_sys,
 184                               uint16_t i_x, uint16_t i_y,
 185                               uint16_t i_w, uint16_t i_h,
 186                               uint16_t i_sx, uint16_t i_sy );
 187
 188
 189 static inline bool raw_line(  filter_sys_t* p_sys,
 190                               uint16_t i_x, uint16_t i_y,
 191                               uint16_t i_w );
 192
 193 static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd );
 194
 195
 196 /*****************************************************************************
 197  * Sub filter code
 198  *****************************************************************************/
 199
 200 /*****************************************************************************
 201  * Local prototypes
 202  *****************************************************************************/
 203 struct filter_sys_t
 204 {
 205     bool          b_need_update;       /* VNC picture is updated, do update the OSD*/
 206     mtime_t       i_vnc_poll_interval; /* Update the OSD menu every n ms */
 207
 208     uint8_t       i_alpha;             /* alpha transparency value */
 209
 210     char          *psz_host;           /* VNC host */
 211     int           i_port;
 212
 213     char          *psz_passwd;         /* VNC password */
 214
 215     bool          b_vnc_poll;          /* Activate VNC polling ? */
 216     bool          b_vnc_mouse_events;  /* Send MouseEvents ? */
 217     bool          b_vnc_key_events;    /* Send KeyEvents ? */
 218
 219     bool          b_connection_active; /* Handshaking finished ? */
 220
 221     vlc_mutex_t   lock;                /* To lock for read/write on picture */
 222
 223     picture_t     *p_pic;              /* The picture with OSD data from VNC */
 224
 225     vout_thread_t *p_vout;             /* Pointer to video-out thread */
 226
 227     int           i_socket;            /* Socket used for VNC */
 228
 229     uint16_t      i_vnc_width;          /* The with of the VNC screen */
 230     uint16_t      i_vnc_height;         /* The height of the VNC screen */
 231         uint32_t      i_vnc_pixels;         /* The pixels of the VNC screen */
 232
 233     bool    b_alpha_from_vnc;    /* Special ffnetdev alpha feature enabled ? */
 234
 235     char          read_buffer[READ_BUFFER_SIZE];
 236
 237     bool          b_continue;
 238
 239     vlc_object_t* p_worker_thread;
 240
 241     uint8_t       ar_color_table_yuv[256][4];
 242 };
 243
 244 /*****************************************************************************
 245  * CreateFilter: Create the filter and open the definition file
 246  *****************************************************************************/
 247 static int CreateFilter ( vlc_object_t *p_this )
 248 {
 249     filter_t *p_filter = (filter_t *)p_this;
 250     filter_sys_t *p_sys = NULL;
 251
 252     msg_Dbg( p_filter, "Creating vnc osd filter..." );
 253
 254     p_filter->p_sys = p_sys = calloc( 1, sizeof(*p_sys) );
 255     if( !p_filter->p_sys )
 256         return VLC_ENOMEM;
 257
 258     /* Populating struct */
 259     vlc_mutex_init( &p_sys->lock );
 260     p_sys->b_continue = true;
 261     p_sys->i_socket = -1;
 262     p_sys->p_pic = NULL;
 263
 264     p_sys->psz_host = var_CreateGetString( p_this, RMTOSD_CFG "host" );
 265     if( EMPTY_STR(p_sys->psz_host) )
 266     {
 267         msg_Err( p_filter, "unable to get vnc host" );
 268         goto error;
 269     }
 270
 271     p_sys->psz_passwd = var_CreateGetString( p_this, RMTOSD_CFG "password" );
 272     if( !p_sys->psz_passwd )
 273     {
 274         msg_Err( p_filter, "unable to get vnc password" );
 275         goto error;
 276     }
 277
 278     p_sys->i_port = var_CreateGetIntegerCommand( p_this, RMTOSD_CFG "port" );
 279
 280     p_sys->i_alpha = var_CreateGetIntegerCommand( p_this, RMTOSD_CFG "alpha" );
 281
 282     /* in milliseconds, 0 disables polling, should not be lower than 100 */
 283     p_sys->i_vnc_poll_interval  = var_CreateGetIntegerCommand( p_this,
 284                                                        RMTOSD_CFG "update" );
 285     if ( p_sys->i_vnc_poll_interval < 100)
 286     {
 287        p_sys->i_vnc_poll_interval = 100;
 288     }
 289
 290     for ( int i = 0; i < 256; i++ )
 291     {
 292         p_sys->ar_color_table_yuv[i][0] = 255;
 293         p_sys->ar_color_table_yuv[i][1] = 255;
 294         p_sys->ar_color_table_yuv[i][2] = 255;
 295         p_sys->ar_color_table_yuv[i][3] = 255;
 296     }
 297
 298     p_sys->b_vnc_poll = var_CreateGetBoolCommand( p_this,
 299                                             RMTOSD_CFG "vnc-polling" );
 300     p_sys->b_vnc_mouse_events = var_CreateGetBoolCommand( p_this,
 301                                             RMTOSD_CFG "mouse-events" );
 302     p_sys->b_vnc_key_events = var_CreateGetBoolCommand( p_this,
 303                                             RMTOSD_CFG "key-events" );
 304
 305     /* Keep track of OSD Events */
 306     p_sys->b_need_update  = false;
 307
 308     /* Attach subpicture filter callback */
 309     p_filter->pf_sub_filter = Filter;
 310     p_filter->pf_sub_mouse  = MouseEvent;
 311
 312     p_sys->p_vout = vlc_object_find( p_this, VLC_OBJECT_VOUT, FIND_PARENT );
 313
 314     if( p_sys->p_vout )
 315     {
 316         var_AddCallback( p_sys->p_vout->p_libvlc, "key-pressed",
 317                          KeyEvent, p_this );
 318     }
 319
 320     es_format_Init( &p_filter->fmt_out, SPU_ES, VLC_CODEC_SPU );
 321     p_filter->fmt_out.i_priority = 0;
 322
 323     vlc_gcrypt_init();
 324
 325     /* create the vnc worker thread */
 326     p_sys->p_worker_thread = vlc_object_create( p_this,
 327                                                 sizeof( vlc_object_t ) );
 328     vlc_object_attach( p_sys->p_worker_thread, p_this );
 329     if( vlc_thread_create( p_sys->p_worker_thread, "vnc worker thread",
 330                            vnc_worker_thread, VLC_THREAD_PRIORITY_LOW ) )
 331     {
 332         vlc_object_release( p_sys->p_worker_thread );
 333         msg_Err( p_filter, "cannot spawn vnc message reader thread" );
 334         goto error;
 335     }
 336
 337     msg_Dbg( p_filter, "osdvnc filter started" );
 338
 339     return VLC_SUCCESS;
 340
 341 error:
 342     msg_Err( p_filter, "osdvnc filter discarded" );
 343
 344     stop_osdvnc( p_filter );
 345
 346     vlc_mutex_destroy( &p_sys->lock );
 347     free( p_sys->psz_host );
 348     free( p_sys->psz_passwd );
 349     free( p_sys );
 350
 351     return VLC_EGENERIC;
 352 }
 353
 354 /*****************************************************************************
 355  * DestroyFilter: Make a clean exit of this plugin
 356  *****************************************************************************/
 357 static void DestroyFilter( vlc_object_t *p_this )
 358 {
 359     filter_t     *p_filter = (filter_t*)p_this;
 360     filter_sys_t *p_sys = p_filter->p_sys;
 361
 362     msg_Dbg( p_filter, "DestroyFilter called." );
 363
 364     stop_osdvnc( p_filter );
 365
 366     if( p_sys->p_vout )
 367     {
 368         var_DelCallback( p_sys->p_vout->p_libvlc, "key-pressed",
 369                          KeyEvent, p_this );
 370
 371         vlc_object_release( p_sys->p_vout );
 372     }
 373
 374     var_Destroy( p_this, RMTOSD_CFG "host" );
 375     var_Destroy( p_this, RMTOSD_CFG "port" );
 376     var_Destroy( p_this, RMTOSD_CFG "password" );
 377     var_Destroy( p_this, RMTOSD_CFG "update" );
 378     var_Destroy( p_this, RMTOSD_CFG "vnc-polling" );
 379     var_Destroy( p_this, RMTOSD_CFG "mouse-events" );
 380     var_Destroy( p_this, RMTOSD_CFG "key-events" );
 381     var_Destroy( p_this, RMTOSD_CFG "alpha" );
 382
 383     vlc_mutex_destroy( &p_sys->lock );
 384     free( p_sys->psz_host );
 385     free( p_sys->psz_passwd );
 386     free( p_sys );
 387 }
 388
 389 static void stop_osdvnc ( filter_t *p_filter )
 390 {
 391     filter_sys_t *p_sys = p_filter->p_sys;
 392
 393     /* It will unlock socket reading */
 394     vlc_object_kill( p_filter );
 395
 396     /* */
 397     if( p_sys->p_worker_thread )
 398     {
 399         msg_Dbg( p_filter, "joining worker_thread" );
 400         vlc_object_kill( p_sys->p_worker_thread );
 401         vlc_thread_join( p_sys->p_worker_thread );
 402         vlc_object_release( p_sys->p_worker_thread );
 403         msg_Dbg( p_filter, "released worker_thread" );
 404     }
 405
 406     msg_Dbg( p_filter, "osdvnc stopped" );
 407 }
 408
 409 static bool read_exact( filter_t *p_filter,
 410                         int i_socket,
 411                         char* p_readbuf,
 412                         int i_bytes )
 413 {
 414     return i_bytes == net_Read( p_filter, i_socket, NULL,
 415                                   (unsigned char*)p_readbuf,
 416                                   i_bytes, true );
 417 }
 418
 419
 420 static bool write_exact( filter_t *p_filter,
 421                          int i_socket,
 422                          char* p_writebuf,
 423                          int i_bytes )
 424 {
 425     return i_bytes == net_Write( p_filter, i_socket, NULL,
 426                                   (unsigned char*)p_writebuf, i_bytes );
 427 }
 428
 429 static bool open_vnc_connection ( filter_t *p_filter )
 430 {
 431     filter_sys_t *p_sys = p_filter->p_sys;
 432
 433     msg_Dbg( p_filter, "Open socket to vnc server on %s:%u.",
 434               p_sys->psz_host, p_sys->i_port );
 435
 436     p_sys->i_socket = net_ConnectTCP( p_filter, p_sys->psz_host, p_sys->i_port );
 437
 438     if( p_sys->i_socket < 0 )
 439     {
 440         msg_Err( p_filter, "Could not open socket" );
 441         return false;
 442     }
 443
 444     msg_Dbg( p_filter, "socket is open." );
 445
 446     return true;
 447 }
 448
 449 static bool handshaking ( filter_t *p_filter )
 450 {
 451     filter_sys_t *p_sys = p_filter->p_sys;
 452
 453     msg_Dbg( p_filter, "Reading protocol version" );
 454
 455     rfbProtocolVersionMsg pv;
 456     if ( !read_exact( p_filter, p_sys->i_socket, pv,
 457                       sz_rfbProtocolVersionMsg ) )
 458     {
 459         msg_Err( p_filter, "Could not read version message" );
 460         return false;
 461     }
 462     pv[sz_rfbProtocolVersionMsg] = '\0'; /* pv size is sz_rfbProtocolVersionMsg+1 */
 463
 464     msg_Dbg( p_filter, "Server version is %s", pv );
 465
 466     strncpy(pv, "RFB 003.003\n", sz_rfbProtocolVersionMsg);
 467
 468     if( !write_exact(p_filter, p_sys->i_socket, pv,
 469                      sz_rfbProtocolVersionMsg) )
 470     {
 471         msg_Err( p_filter, "Could not write version message" );
 472         return false;
 473     }
 474
 475     msg_Dbg( p_filter, "Reading authentication scheme" );
 476     uint32_t i_authScheme;
 477     if( !read_exact( p_filter, p_sys->i_socket, (char*)&i_authScheme, 4 ) )
 478     {
 479         msg_Err( p_filter, "Could not read authentication scheme" );
 480         return false;
 481     }
 482     i_authScheme = htonl(i_authScheme);
 483
 484     msg_Dbg( p_filter, "Authentication scheme = %x", i_authScheme );
 485     if ( i_authScheme == rfbConnFailed )
 486     {
 487         msg_Err( p_filter, "Connection rejected by server" );
 488         return false;
 489     }
 490     if (i_authScheme == rfbVncAuth)
 491     {
 492         unsigned char challenge[CHALLENGESIZE];
 493         if ( !read_exact( p_filter, p_sys->i_socket,
 494                           (char*)challenge, CHALLENGESIZE ) )
 495         {
 496             msg_Err( p_filter, "Could not read password challenge" );
 497             return false;
 498         }
 499
 500         vnc_encrypt_bytes( challenge, p_sys->psz_passwd );
 501
 502         if( !write_exact(p_filter, p_sys->i_socket,
 503                          (char*)challenge, CHALLENGESIZE ) )
 504         {
 505             msg_Err( p_filter, "Could not write password" );
 506             return false;
 507         }
 508         uint32_t i_authResult;
 509         if( !read_exact( p_filter, p_sys->i_socket, (char*)&i_authResult, 4 ) )
 510         {
 511             msg_Err( p_filter, "Could not read authentication result" );
 512             return false;
 513         }
 514         i_authResult = htonl(i_authResult);
 515         if (i_authResult != rfbVncAuthOK)
 516         {
 517             msg_Err( p_filter, "VNC authentication failed" );
 518             return false;
 519         }
 520     }
 521
 522     msg_Dbg( p_filter, "Writing client init message" );
 523     rfbClientInitMsg ci;
 524     ci.shared = 1;
 525     if( !write_exact( p_filter, p_sys->i_socket,
 526                       (char*)&ci, sz_rfbClientInitMsg ) )
 527     {
 528         msg_Err( p_filter, "Could not write client init message" );
 529         return false;
 530     }
 531
 532     msg_Dbg( p_filter, "Reading server init message" );
 533     rfbServerInitMsg si;
 534     if( !read_exact( p_filter, p_sys->i_socket,
 535                      (char*)&si, sz_rfbServerInitMsg ) )
 536     {
 537         msg_Err( p_filter, "Could not read server init message" );
 538         return false;
 539     }
 540     si.framebufferWidth = htons(si.framebufferWidth);
 541     si.framebufferHeight = htons(si.framebufferHeight);
 542     si.format.redMax = htons(si.format.redMax);
 543     si.format.greenMax = htons(si.format.greenMax);
 544     si.format.blueMax = htons(si.format.blueMax);
 545
 546     p_sys->i_vnc_width = si.framebufferWidth;
 547     p_sys->i_vnc_height = si.framebufferHeight;
 548
 549     msg_Dbg( p_filter, "Servers preferred pixelformat: "
 550                         "%ux%u, R(%u),G(%u),B(%u), %u bit, depht=%u, %s",
 551                         si.framebufferWidth,
 552                         si.framebufferHeight,
 553                         si.format.redMax,
 554                         si.format.greenMax,
 555                         si.format.blueMax,
 556                         si.format.bitsPerPixel,
 557                         si.format.depth,
 558                         si.format.trueColour ? "TrueColor" : "Not-TrueColor");
 559
 560     uint32_t i_nameLength = htonl(si.nameLength);
 561     if( i_nameLength > MAX_VNC_SERVER_NAME_LENGTH )
 562     {
 563         msg_Err( p_filter, "Server name too long" );
 564         return false;
 565     }
 566     char s_ServerName[MAX_VNC_SERVER_NAME_LENGTH+1];
 567
 568     msg_Dbg( p_filter, "Reading server name with size = %u", i_nameLength );
 569     if( !read_exact( p_filter, p_sys->i_socket, s_ServerName, i_nameLength ) )
 570     {
 571         msg_Err( p_filter, "Could not read server name" );
 572         return false;
 573     }
 574     s_ServerName[i_nameLength] = '\0';
 575
 576     if( strcmp( s_ServerName, "VDR-OSD") == 0 )
 577     {
 578         msg_Dbg( p_filter, "Server is a VDR" );
 579         p_sys->b_alpha_from_vnc = true;
 580     }
 581     else
 582     {
 583         msg_Dbg( p_filter, "Server is a normal VNC" );
 584         p_sys->b_alpha_from_vnc = false;
 585     }
 586
 587
 588     msg_Dbg( p_filter, "Server init message read properly" );
 589     msg_Dbg( p_filter, "Server name is %s", s_ServerName );
 590
 591     msg_Dbg( p_filter, "Writing SetPixelFormat message" );
 592
 593     rfbSetPixelFormatMsg sp;
 594     sp.type = rfbSetPixelFormat;
 595     sp.pad1 = sp.pad2 = 0;
 596     sp.format.bitsPerPixel = 8;
 597     sp.format.depth = 8 ;
 598     sp.format.bigEndian = 1;
 599     sp.format.trueColour = 0;
 600     sp.format.redMax = htons(31);
 601     sp.format.greenMax = htons(31);
 602     sp.format.blueMax = htons(31);
 603     sp.format.redShift = 10;
 604     sp.format.greenShift = 5;
 605     sp.format.blueShift = 0;
 606     sp.format.pad1 = sp.format.pad2 = 0;
 607
 608     if( !write_exact( p_filter, p_sys->i_socket,
 609                       (char*)&sp, sz_rfbSetPixelFormatMsg) )
 610     {
 611         msg_Err( p_filter, "Could not write SetPixelFormat message" );
 612         return false;
 613     }
 614
 615     msg_Dbg( p_filter, "Writing SetEncodings message" );
 616
 617     rfbSetEncodingsMsg se;
 618     se.type = rfbSetEncodings;
 619     se.pad = 0;
 620     se.nEncodings = htons( p_sys->b_alpha_from_vnc ? 3 : 2 );
 621
 622     if( !write_exact( p_filter, p_sys->i_socket,
 623                       (char*)&se, sz_rfbSetEncodingsMsg) )
 624     {
 625         msg_Err( p_filter, "Could not write SetEncodings message begin" );
 626         return false;
 627     }
 628
 629     uint32_t i_encoding;
 630
 631     msg_Dbg( p_filter, "Writing SetEncodings rfbEncodingCopyRect" );
 632     i_encoding = htonl(rfbEncodingCopyRect);
 633     if( !write_exact( p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
 634     {
 635         msg_Err( p_filter, "Could not write encoding type rfbEncodingCopyRect." );
 636         return false;
 637     }
 638
 639     msg_Dbg( p_filter, "Writing SetEncodings rfbEncodingRRE" );
 640     i_encoding = htonl(rfbEncodingRRE);
 641     if( !write_exact(p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
 642     {
 643         msg_Err( p_filter, "Could not write encoding type rfbEncodingRRE." );
 644         return false;
 645     }
 646
 647     if( p_sys->b_alpha_from_vnc )
 648     {
 649         msg_Dbg( p_filter, "Writing SetEncodings rfbEncSpecialUseAlpha" );
 650         i_encoding = 0x00F0FFFF; /* rfbEncSpecialUseAlpha is 0xFFFFF000
 651                                   * before we swap it */
 652         if( !write_exact(p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
 653         {
 654             msg_Err( p_filter, "Could not write encoding type rfbEncSpecialUseAlpha." );
 655             return false;
 656         }
 657     }
 658     return true;
 659
 660 }
 661
 662 static void* vnc_worker_thread( vlc_object_t *p_thread_obj )
 663 {
 664     filter_t* p_filter = (filter_t*)(p_thread_obj->p_parent);
 665     filter_sys_t *p_sys = p_filter->p_sys;
 666     vlc_object_t *p_update_request_thread;
 667     int canc = vlc_savecancel ();
 668
 669     msg_Dbg( p_filter, "VNC worker thread started" );
 670
 671     if( !open_vnc_connection ( p_filter ) )
 672     {
 673         msg_Err( p_filter, "Could not connect to vnc host" );
 674         goto exit;
 675     }
 676
 677     if( !handshaking ( p_filter ) )
 678     {
 679         msg_Err( p_filter, "Error occured while handshaking vnc host" );
 680         goto exit;
 681     }
 682
 683     p_sys->b_connection_active = true; /* to enable sending key
 684                                             * and mouse events to host */
 685
 686     /* Create an empty picture for VNC the data */
 687     vlc_mutex_lock( &p_sys->lock );
 688     p_sys->p_pic = picture_New( VLC_CODEC_YUVA,
 689                                 p_sys->i_vnc_width, p_sys->i_vnc_height, 1, 1 );
 690     if( !p_sys->p_pic )
 691     {
 692         vlc_mutex_unlock( &p_sys->lock );
 693         goto exit;
 694     }
 695         p_sys->i_vnc_pixels = p_sys->i_vnc_width * p_sys->i_vnc_height;
 696
 697     vlc_mutex_unlock( &p_sys->lock );
 698
 699     /* create the update request thread */
 700     p_update_request_thread = vlc_object_create( p_filter,
 701                                                  sizeof( vlc_object_t ) );
 702     vlc_object_attach( p_update_request_thread, p_filter );
 703     if( vlc_thread_create( p_update_request_thread,
 704                            "vnc update request thread",
 705                            update_request_thread, VLC_THREAD_PRIORITY_LOW ) )
 706     {
 707         vlc_object_release( p_update_request_thread );
 708         msg_Err( p_filter, "cannot spawn vnc update request thread" );
 709         goto exit;
 710     }
 711
 712     /* connection is initialized, now read and handle server messages */
 713     while( vlc_object_alive( p_thread_obj ) )
 714     {
 715         rfbServerToClientMsg msg;
 716         int i_msgSize;
 717
 718         memset( &msg, 0, sizeof(msg) );
 719
 720         if( !read_exact(p_filter, p_sys->i_socket, (char*)&msg, 1 ) )
 721         {
 722             msg_Err( p_filter, "Error while waiting for next server message");
 723             break;
 724         }
 725         switch (msg.type)
 726         {
 727         case rfbFramebufferUpdate:
 728             i_msgSize = sz_rfbFramebufferUpdateMsg;
 729             break;
 730         case rfbSetColourMapEntries:
 731             i_msgSize = sz_rfbSetColourMapEntriesMsg;
 732             break;
 733         case rfbBell:
 734             i_msgSize = sz_rfbBellMsg;
 735             break;
 736         case rfbServerCutText:
 737             i_msgSize = sz_rfbServerCutTextMsg;
 738             break;
 739         case rfbReSizeFrameBuffer:
 740             i_msgSize = sz_rfbReSizeFrameBufferMsg;
 741             break;
 742         default:
 743             i_msgSize = 0;
 744             msg_Err( p_filter, "Invalid message %u received", msg.type );
 745             break;
 746         }
 747
 748         if( i_msgSize <= 0 )
 749             break;
 750
 751         if( --i_msgSize > 0 )
 752         {
 753             if ( !read_exact( p_filter, p_sys->i_socket,
 754                               ((char*)&msg)+1, i_msgSize ) )
 755             {
 756                 msg_Err( p_filter, "Error while reading message of type %u",
 757                          msg.type );
 758                 break;
 759             }
 760         }
 761         process_server_message( p_filter, &msg);
 762     }
 763
 764     msg_Dbg( p_filter, "joining update_request_thread" );
 765     vlc_object_kill( p_update_request_thread );
 766     vlc_thread_join( p_update_request_thread );
 767     vlc_object_release( p_update_request_thread );
 768     msg_Dbg( p_filter, "released update_request_thread" );
 769
 770 exit:
 771
 772     vlc_mutex_lock( &p_sys->lock );
 773     p_sys->b_connection_active = false;
 774
 775     if (p_sys->i_socket >= 0)
 776         net_Close(p_sys->i_socket);
 777
 778     if( p_sys->p_pic )
 779         picture_Release( p_sys->p_pic );
 780
 781     /* It will hide the subtitle */
 782     p_sys->b_continue = false;
 783     p_sys->b_need_update = true;
 784     vlc_mutex_unlock( &p_sys->lock );
 785
 786     msg_Dbg( p_filter, "VNC message reader thread ended" );
 787     vlc_restorecancel (canc);
 788     return NULL;
 789 }
 790
 791 static void* update_request_thread( vlc_object_t *p_thread_obj )
 792 {
 793     filter_t* p_filter = (filter_t*)(p_thread_obj->p_parent);
 794     filter_sys_t *p_sys = p_filter->p_sys;
 795     int canc = vlc_savecancel ();
 796
 797     msg_Dbg( p_filter, "VNC update request thread started" );
 798
 799     rfbFramebufferUpdateRequestMsg udr;
 800     udr.type = rfbFramebufferUpdateRequest;
 801     udr.incremental = 0;
 802     udr.x = 0;
 803     udr.y = 0;
 804     udr.w = htons(p_sys->i_vnc_width);
 805     udr.h = htons(p_sys->i_vnc_height);
 806
 807     if( write_exact(p_filter, p_sys->i_socket, (char*)&udr,
 808            sz_rfbFramebufferUpdateRequestMsg) == false)
 809     {
 810         msg_Err( p_filter, "Could not write rfbFramebufferUpdateRequestMsg." );
 811         p_sys->b_continue = false;
 812         return NULL;
 813     }
 814
 815     udr.incremental = 1;
 816     mtime_t i_poll_interval_microsec = p_sys->i_vnc_poll_interval * 1000;
 817
 818     if( p_sys->b_vnc_poll)
 819     {
 820         while( vlc_object_alive( p_thread_obj ) )
 821         {
 822             msleep( i_poll_interval_microsec );
 823             if( write_exact(p_filter, p_sys->i_socket, (char*)&udr,
 824                    sz_rfbFramebufferUpdateRequestMsg) == false)
 825             {
 826                 msg_Err( p_filter, "Could not write rfbFramebufferUpdateRequestMsg." );
 827                 break;
 828             }
 829         }
 830         p_sys->b_continue = false;
 831     }
 832     else
 833     {
 834         msg_Dbg( p_filter, "VNC polling disabled." );
 835     }
 836
 837     msg_Dbg( p_filter, "VNC update request thread ended" );
 838     vlc_restorecancel (canc);
 839     return NULL;
 840 }
 841
 842 static bool process_server_message ( filter_t *p_filter,
 843                                      rfbServerToClientMsg *msg )
 844 {
 845     filter_sys_t *p_sys = p_filter->p_sys;
 846
 847     switch (msg->type)
 848     {
 849     case rfbFramebufferUpdate:
 850         {
 851             msg->fu.nRects = htons(msg->fu.nRects);
 852             rfbFramebufferUpdateRectHeader hdr;
 853
 854             for (int i_rect = 0; i_rect < msg->fu.nRects; i_rect++)
 855             {
 856                 if (!read_exact(p_filter, p_sys->i_socket, (char*)&hdr,
 857                     sz_rfbFramebufferUpdateRectHeader ) )
 858                 {
 859                     msg_Err( p_filter, "Could not read FrameBufferUpdate header" );
 860                     return false;
 861                 }
 862                 hdr.r.x = htons(hdr.r.x);
 863                 hdr.r.y = htons(hdr.r.y);
 864                 hdr.r.w = htons(hdr.r.w);
 865                 hdr.r.h = htons(hdr.r.h);
 866                 hdr.encoding = htonl(hdr.encoding);
 867
 868                 switch (hdr.encoding)
 869                 {
 870                 case rfbEncodingRaw:
 871                     {
 872                         int i_line;
 873                         for (i_line = 0; i_line < hdr.r.h; i_line++)
 874                         {
 875                             if ( !read_exact( p_filter, p_sys->i_socket,
 876                                     p_sys->read_buffer, hdr.r.w ) )
 877                             {
 878                                 msg_Err( p_filter,
 879                                  "Could not read FrameBufferUpdate line data" );
 880                                return false;
 881                             }
 882                             vlc_mutex_lock( &p_sys->lock );
 883                             if ( !raw_line( p_sys, hdr.r.x,
 884                                             hdr.r.y + i_line,
 885                                             hdr.r.w ) )
 886                             {
 887                                 msg_Err( p_filter, "raw_line failed." );
 888                                 vlc_mutex_unlock( &p_sys->lock );
 889                                 return false;
 890                             }
 891                             vlc_mutex_unlock( &p_sys->lock );
 892                         }
 893                     }
 894                     break;
 895
 896                 case rfbEncodingCopyRect:
 897                     {
 898                         rfbCopyRect rect;
 899
 900                         if ( !read_exact( p_filter, p_sys->i_socket,
 901                                           (char*)&rect,
 902                                           sz_rfbCopyRect ) )
 903                         {
 904                             msg_Err( p_filter, "Could not read rfbCopyRect" );
 905                             return false;
 906                         }
 907                         rect.srcX = htons( rect.srcX );
 908                         rect.srcY = htons( rect.srcY );
 909
 910                         vlc_mutex_lock( &p_sys->lock );
 911                         if ( !copy_rect( p_sys,
 912                                          hdr.r.x,   hdr.r.y,
 913                                          hdr.r.w,   hdr.r.h,
 914                                          rect.srcX, rect.srcY ) )
 915                         {
 916                             msg_Err( p_filter, "copy_rect failed." );
 917                             vlc_mutex_unlock( &p_sys->lock );
 918                             return false;
 919                         }
 920                         vlc_mutex_unlock( &p_sys->lock );
 921                     }
 922                     break;
 923
 924                 case rfbEncodingRRE:
 925                     {
 926                         rfbRREHeader rrehdr;
 927                         if ( !read_exact( p_filter, p_sys->i_socket,
 928                                           (char*)&rrehdr,
 929                                           sz_rfbRREHeader ) )
 930                         {
 931                             msg_Err( p_filter, "Could not read rfbRREHeader" );
 932                             return false;
 933                         }
 934                         uint8_t i_pixcolor;
 935                         if ( !read_exact(p_filter, p_sys->i_socket,
 936                                          (char*)&i_pixcolor, 1 ) )
 937                         {
 938                             msg_Err( p_filter, "Could not read RRE pixcolor" );
 939                             return false;
 940                         }
 941
 942                         vlc_mutex_lock( &p_sys->lock );
 943                         if ( !fill_rect( p_sys,
 944                                         hdr.r.x, hdr.r.y,
 945                                         hdr.r.w, hdr.r.h,
 946                                         i_pixcolor) )
 947                         {
 948                             msg_Err( p_filter, "main fill_rect failed." );
 949                             vlc_mutex_unlock( &p_sys->lock );
 950                             return false;
 951                         }
 952                         vlc_mutex_unlock( &p_sys->lock );
 953
 954                         rrehdr.nSubrects = htonl(rrehdr.nSubrects);
 955
 956                         int i_datasize = rrehdr.nSubrects *
 957                                      ( sizeof(i_pixcolor) + sz_rfbRectangle ) ;
 958                         if ( i_datasize > READ_BUFFER_SIZE )
 959                         {
 960                             msg_Err( p_filter, "Buffer too small, "
 961                                      "need %u bytes", i_datasize );
 962                             return false;
 963                         }
 964                         if ( !read_exact( p_filter, p_sys->i_socket,
 965                                        p_sys->read_buffer, i_datasize ) )
 966                         {
 967                             msg_Err( p_filter,
 968                                      "Could not read RRE subrect data" );
 969                             return false;
 970                         }
 971
 972                         uint32_t i_subrect;
 973                         rfbRectangle* p_subrect;
 974                         int i_offset = 0;
 975                         vlc_mutex_lock( &p_sys->lock );
 976                         for ( i_subrect = 0;
 977                               i_subrect < rrehdr.nSubrects; i_subrect++)
 978                         {
 979                             i_pixcolor = p_sys->read_buffer[i_offset];
 980                             i_offset += sizeof(i_pixcolor);
 981                             p_subrect =
 982                                (rfbRectangle*)(p_sys->read_buffer + i_offset);
 983                             i_offset += sz_rfbRectangle;
 984
 985                             if (!fill_rect( p_sys,
 986                                             htons(p_subrect->x) + hdr.r.x,
 987                                             htons(p_subrect->y) + hdr.r.y,
 988                                             htons(p_subrect->w),
 989                                             htons(p_subrect->h),
 990                                             i_pixcolor) )
 991                             {
 992                                 msg_Err( p_filter,
 993                                     "subrect %u fill_rect failed.", i_subrect );
 994                                 vlc_mutex_unlock( &p_sys->lock );
 995                                 return false;
 996                             }
 997                         }
 998                         vlc_mutex_unlock( &p_sys->lock );
 999                     }
1000                     break;
1001
1002                 }
1003
1004             }
1005             vlc_mutex_lock( &p_sys->lock );
1006             p_sys->b_need_update = true;
1007             vlc_mutex_unlock( &p_sys->lock );
1008         }
1009         return true;
1010
1011     case rfbSetColourMapEntries:
1012         {
1013             msg->scme.nColours = htons(msg->scme.nColours);
1014             msg->scme.firstColour = htons(msg->scme.firstColour);
1015             int i_datasize;
1016             if ( p_sys->b_alpha_from_vnc == true )
1017             {
1018                 i_datasize = 2 * msg->scme.nColours * 4;
1019             }
1020             else
1021             {
1022                 i_datasize = 2 * msg->scme.nColours * 3;
1023             }
1024             if ( i_datasize > READ_BUFFER_SIZE )
1025             {
1026                 msg_Err( p_filter, "Buffer too small, need %u bytes",
1027                                    i_datasize );
1028                 return false;
1029             }
1030
1031             if ( !read_exact( p_filter, p_sys->i_socket,
1032                               p_sys->read_buffer, i_datasize ) )
1033             {
1034                 msg_Err( p_filter, "Could not read color map data" );
1035                 return false;
1036             }
1037
1038             uint8_t i_red, i_green, i_blue, i_alpha, i_color_index;
1039             uint16_t i_offset = 0;
1040             i_alpha = 255;
1041
1042             for (int i = 0; i < msg->scme.nColours; i++)
1043             {
1044                 i_color_index = i+msg->scme.firstColour;
1045                 if ( p_sys->b_alpha_from_vnc == true )
1046                 {
1047                     i_alpha = p_sys->read_buffer[i_offset];
1048                     i_offset += 2;
1049                 }
1050                 i_red   = p_sys->read_buffer[i_offset];
1051                 i_offset += 2;
1052                 i_green = p_sys->read_buffer[i_offset];
1053                 i_offset += 2;
1054                 i_blue  = p_sys->read_buffer[i_offset];
1055                 i_offset += 2;
1056                 rgb_to_yuv( &p_sys->ar_color_table_yuv[i_color_index][0],
1057                             &p_sys->ar_color_table_yuv[i_color_index][1],
1058                             &p_sys->ar_color_table_yuv[i_color_index][2],
1059                             i_red,
1060                             i_green,
1061                             i_blue );
1062                 p_sys->ar_color_table_yuv[i][3] = i_alpha;
1063             }
1064         }
1065         return true;
1066
1067     case rfbBell:
1068         msg_Err( p_filter, "rfbBell received" );
1069         return true;
1070
1071     case rfbServerCutText:
1072         msg->sct.length = htons(msg->sct.length);
1073         if ( msg->sct.length > READ_BUFFER_SIZE )
1074         {
1075             msg_Err( p_filter, "Buffer too small, need %u bytes", msg->sct.length );
1076             return false;
1077         }
1078         if ( !read_exact(p_filter, p_sys->i_socket,
1079                          p_sys->read_buffer, msg->sct.length ) )
1080         {
1081             msg_Err( p_filter, "Could not read Reading rfbServerCutText data" );
1082             return false;
1083         }
1084         return true;
1085
1086     case rfbReSizeFrameBuffer:
1087         msg_Err( p_filter, "Reading rfbReSizeFrameBuffer not implemented" );
1088         return false;
1089
1090     default:
1091         msg_Err( p_filter, "Invalid message %u received", msg->type );
1092         return false;
1093     }
1094     return false;
1095 }
1096
1097 /****************************************************************************
1098  * Filter: the whole thing
1099  ****************************************************************************
1100  * This function outputs subpictures at regular time intervals.
1101  ****************************************************************************/
1102 static subpicture_t *Filter( filter_t *p_filter, mtime_t date )
1103 {
1104     filter_sys_t *p_sys = p_filter->p_sys;
1105     subpicture_t *p_spu;
1106     subpicture_region_t *p_region;
1107     video_format_t fmt;
1108     picture_t *p_pic;
1109
1110     if( !p_sys->b_need_update )
1111     {
1112         return NULL;
1113     }
1114
1115     vlc_mutex_lock( &p_sys->lock );
1116
1117     p_pic = p_sys->p_pic;
1118
1119     if( !p_pic )
1120     {
1121         vlc_mutex_unlock( &p_sys->lock );
1122         return NULL;
1123     }
1124
1125     /* Allocate the subpicture internal data. */
1126     p_spu = filter_NewSubpicture( p_filter );
1127     if( !p_spu )
1128     {
1129         vlc_mutex_unlock( &p_sys->lock );
1130         return NULL;
1131     }
1132
1133     p_spu->b_absolute = false;
1134     p_spu->i_start = date;
1135     p_spu->i_stop = 0;
1136     p_spu->b_ephemer = true;
1137
1138     if( !p_sys->b_continue )
1139         p_spu->i_stop = p_spu->i_start + 1;
1140
1141     /* Create new SPU region */
1142     memset( &fmt, 0, sizeof(video_format_t) );
1143     fmt.i_chroma = VLC_CODEC_YUVA;
1144     fmt.i_sar_num = fmt.i_sar_den = 1;
1145     fmt.i_width = fmt.i_visible_width = p_pic->p[Y_PLANE].i_visible_pitch;
1146     fmt.i_height = fmt.i_visible_height = p_pic->p[Y_PLANE].i_visible_lines;
1147     fmt.i_x_offset = fmt.i_y_offset = 0;
1148     p_region = subpicture_region_New( &fmt );
1149     if( !p_region )
1150     {
1151         msg_Err( p_filter, "cannot allocate SPU region" );
1152         p_filter->pf_sub_buffer_del( p_filter, p_spu );
1153         vlc_mutex_unlock( &p_sys->lock );
1154         return NULL;
1155     }
1156
1157     /* FIXME the copy is probably not needed anymore */
1158     picture_Copy( p_region->p_picture, p_pic );
1159
1160     p_sys->b_need_update = false;
1161
1162     vlc_mutex_unlock( &p_sys->lock );
1163
1164     /* set to one of the 9 relative locations */
1165     p_region->i_align = 0; /* Center */
1166     p_spu->b_absolute = false;
1167
1168
1169     p_spu->i_original_picture_width = 0; /*Let vout core do the horizontal scaling */
1170     p_spu->i_original_picture_height = fmt.i_height;
1171
1172     p_spu->p_region = p_region;
1173
1174     p_spu->i_alpha = ( p_sys->i_alpha );
1175
1176     return p_spu;
1177 }
1178
1179
1180 static inline void rgb_to_yuv( uint8_t *y, uint8_t *u, uint8_t *v,
1181                                int r, int g, int b )
1182 {
1183     *y = ( ( (  66 * r + 129 * g +  25 * b + 128 ) >> 8 ) + 16 );
1184     *u =   ( ( -38 * r -  74 * g + 112 * b + 128 ) >> 8 ) + 128 ;
1185     *v =   ( ( 112 * r -  94 * g -  18 * b + 128 ) >> 8 ) + 128 ;
1186 }
1187
1188 static inline bool fill_rect( filter_sys_t* p_sys,
1189                               uint16_t i_x, uint16_t i_y,
1190                               uint16_t i_w, uint16_t i_h,
1191                               uint8_t i_color)
1192 {
1193     plane_t *p_outY = p_sys->p_pic->p+Y_PLANE;
1194     plane_t *p_outU = p_sys->p_pic->p+U_PLANE;
1195     plane_t *p_outV = p_sys->p_pic->p+V_PLANE;
1196     plane_t *p_outA = p_sys->p_pic->p+A_PLANE;
1197     int i_pitch = p_outY->i_pitch;
1198     int i_lines = p_outY->i_lines;
1199     if ( i_x + i_w > i_pitch)
1200         return false;
1201     if ( i_y + i_h > i_lines)
1202         return false;
1203     int i_line_offset = i_y * i_pitch;
1204     uint8_t i_yuv_y = p_sys->ar_color_table_yuv[i_color][0];
1205     uint8_t i_yuv_u = p_sys->ar_color_table_yuv[i_color][1];
1206     uint8_t i_yuv_v = p_sys->ar_color_table_yuv[i_color][2];
1207     uint8_t i_alpha = p_sys->ar_color_table_yuv[i_color][3];
1208     for( int i_line = 0; i_line < i_h; i_line++ )
1209     {
1210         for( int i_column = 0; i_column < i_w; i_column++ )
1211         {
1212             int i_total_offset = i_line_offset + i_x + i_column;
1213             p_outY->p_pixels[ i_total_offset ] = i_yuv_y;
1214             p_outU->p_pixels[ i_total_offset ] = i_yuv_u;
1215             p_outV->p_pixels[ i_total_offset ] = i_yuv_v;
1216             p_outA->p_pixels[ i_total_offset ] = i_alpha;
1217         }
1218         i_line_offset += i_pitch;
1219     }
1220     return true;
1221 }
1222
1223 static inline bool copy_rect( filter_sys_t* p_sys,
1224                               uint16_t i_x, uint16_t i_y,
1225                               uint16_t i_w, uint16_t i_h,
1226                               uint16_t i_sx, uint16_t i_sy )
1227 {
1228     plane_t *p_Y = p_sys->p_pic->p+Y_PLANE;
1229     plane_t *p_U = p_sys->p_pic->p+U_PLANE;
1230     plane_t *p_V = p_sys->p_pic->p+V_PLANE;
1231     plane_t *p_A = p_sys->p_pic->p+A_PLANE;
1232
1233     int i_pitch = p_Y->i_pitch;
1234     int i_lines = p_Y->i_lines;
1235
1236     fprintf( stderr, "copy_rect: (%d,%d)+(%d,%d) -> (%d,%d)\n", i_x, i_y, i_w, i_h, i_sx, i_sy );
1237
1238     if( i_x + i_w > i_pitch || i_sx + i_w > i_pitch )
1239         return false;
1240     if( i_y + i_h > i_lines || i_sy + i_h > i_lines)
1241         return false;
1242
1243     if( i_w <= 0 || i_h <= 0 )
1244         return true;
1245
1246     uint8_t *pb_buffer = calloc( i_w * i_h, 4 );
1247     if( !pb_buffer )
1248         return false;
1249
1250     for( int i_line = 0; i_line < i_h; i_line++ )
1251     {
1252         for( int i_column = 0; i_column < i_w; i_column++ )
1253         {
1254             const int i_src_offset = ( i_sy + i_line ) * i_pitch + i_sx + i_column;
1255             const int i_tmp_offset = (    0 + i_line ) *     i_w +    0 + i_column;
1256
1257             pb_buffer[4*i_tmp_offset + 0] = p_Y->p_pixels[i_src_offset];
1258             pb_buffer[4*i_tmp_offset + 1] = p_U->p_pixels[i_src_offset];
1259             pb_buffer[4*i_tmp_offset + 2] = p_V->p_pixels[i_src_offset];
1260             pb_buffer[4*i_tmp_offset + 3] = p_A->p_pixels[i_src_offset];
1261         }
1262     }
1263
1264     for( int i_line = 0; i_line < i_h; i_line++ )
1265     {
1266         for( int i_column = 0; i_column < i_w; i_column++ )
1267         {
1268             const int i_tmp_offset = (   0 + i_line ) *     i_w +   0 + i_column;
1269             const int i_dst_offset = ( i_y + i_line ) * i_pitch + i_x + i_column;
1270
1271             p_Y->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 0];
1272             p_U->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 1];
1273             p_V->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 2];
1274             p_A->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 3];
1275         }
1276     }
1277     free( pb_buffer );
1278     return true;
1279
1280 }
1281
1282 static inline bool raw_line(  filter_sys_t* p_sys,
1283                               uint16_t i_x, uint16_t i_y,
1284                               uint16_t i_w )
1285 {
1286     plane_t *p_outY = p_sys->p_pic->p+Y_PLANE;
1287     plane_t *p_outU = p_sys->p_pic->p+U_PLANE;
1288     plane_t *p_outV = p_sys->p_pic->p+V_PLANE;
1289     plane_t *p_outA = p_sys->p_pic->p+A_PLANE;
1290     int i_pitch = p_outY->i_pitch;
1291     int i_lines = p_outY->i_lines;
1292     if ( i_x + i_w > i_pitch)
1293         return false;
1294     if ( i_y > i_lines)
1295         return false;
1296
1297     int i_line_offset = i_y * i_pitch + i_x;
1298
1299     for( int i_column = 0; i_column < i_w; i_column++ )
1300     {
1301         int i_offset = i_line_offset + i_column;
1302         uint8_t i_color = p_sys->read_buffer[i_column];
1303         p_outY->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][0];
1304         p_outU->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][1];
1305         p_outV->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][2];
1306         p_outA->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][3];
1307     }
1308
1309     return true;
1310 }
1311
1312
1313 /*****************************************************************************
1314  * MouseEvent: callback for mouse events
1315  *****************************************************************************/
1316 static int MouseEvent( filter_t *p_filter,
1317                        const vlc_mouse_t *p_old,
1318                        const vlc_mouse_t *p_new,
1319                        const video_format_t *p_fmt )
1320 {
1321     filter_sys_t *p_sys = p_filter->p_sys;
1322     VLC_UNUSED(p_old);
1323
1324     if( !p_sys->b_vnc_mouse_events )
1325         return VLC_SUCCESS;
1326
1327     int i_v = p_new->i_pressed;
1328     int i_x = p_new->i_x;
1329     int i_y = p_new->i_y;
1330
1331     vlc_mutex_lock( &p_sys->lock );
1332
1333     const int v_h = p_fmt->i_visible_height;
1334     const int v_w = p_sys->i_vnc_width * v_h / p_sys->i_vnc_height;
1335     const int v_x = (p_fmt->i_visible_width-v_w)/2;
1336
1337     i_x -= v_x;
1338
1339     if( i_y < 0 || i_x < 0 || i_y >= v_h || i_x >= v_w )
1340     {
1341         vlc_mutex_unlock( &p_sys->lock );
1342         msg_Dbg( p_filter, "invalid mouse event? x=%d y=%d btn=%x", i_x, i_y, i_v );
1343         return VLC_SUCCESS;
1344     }
1345     if( !p_sys->b_connection_active )
1346     {
1347         vlc_mutex_unlock( &p_sys->lock );
1348         return VLC_SUCCESS;
1349     }
1350
1351 #ifdef VNC_DEBUG
1352     msg_Dbg( p_filter, "mouse event x=%d y=%d btn=%x", i_x, i_y, i_v );
1353 #endif
1354
1355     /* */
1356     i_x = i_x * p_sys->i_vnc_width / v_w;
1357     i_y = i_y * p_sys->i_vnc_height / v_h;
1358
1359     /* buttonMask bits 0-7 are buttons 1-8, 0=up, 1=down */
1360     rfbPointerEventMsg ev;
1361     ev.type = rfbPointerEvent;
1362     ev.buttonMask = i_v;
1363     ev.x = htons(i_x);
1364     ev.y = htons(i_y);
1365
1366     write_exact( p_filter, p_sys->i_socket,
1367                  (char*)&ev, sz_rfbPointerEventMsg);
1368
1369     vlc_mutex_unlock( &p_sys->lock );
1370
1371     return VLC_EGENERIC;
1372 }
1373
1374 /*****************************************************************************
1375  * KeyEvent: callback for keyboard events
1376  *****************************************************************************/
1377 static int KeyEvent( vlc_object_t *p_this, char const *psz_var,
1378                      vlc_value_t oldval, vlc_value_t newval, void *p_data )
1379 {
1380     VLC_UNUSED(psz_var); VLC_UNUSED(oldval);
1381
1382     filter_t *p_filter = (filter_t *)p_data;
1383     filter_sys_t *p_sys = p_filter->p_sys;
1384
1385     if( !p_sys->b_vnc_key_events )
1386         return VLC_SUCCESS;
1387
1388     msg_Dbg( p_this, "key pressed (%"PRId64") ", newval.i_int );
1389
1390     if ( !newval.i_int )
1391     {
1392         msg_Err( p_this, "Received invalid key event 0" );
1393         return VLC_EGENERIC;
1394     }
1395
1396     vlc_mutex_lock( &p_sys->lock );
1397     if( !p_sys->b_connection_active )
1398     {
1399         vlc_mutex_unlock( &p_sys->lock );
1400         return VLC_SUCCESS;
1401     }
1402
1403     uint32_t i_key32 = newval.i_int;
1404     i_key32 = htonl(i_key32);
1405     rfbKeyEventMsg ev;
1406
1407     ev.type = rfbKeyEvent;
1408     ev.down = 1;
1409     ev.pad = 0;
1410
1411     /* first key-down for modifier-keys */
1412     if (newval.i_int & KEY_MODIFIER_CTRL)
1413     {
1414         ev.key = 0xffe3;
1415         write_exact( p_filter, p_sys->i_socket,
1416                      (char*)&ev, sz_rfbKeyEventMsg);
1417     }
1418     if (newval.i_int & KEY_MODIFIER_SHIFT)
1419     {
1420         ev.key = 0xffe1;
1421         write_exact( p_filter, p_sys->i_socket,
1422                      (char*)&ev, sz_rfbKeyEventMsg);
1423     }
1424     if (newval.i_int & KEY_MODIFIER_ALT)
1425     {
1426         ev.key = 0xffe9;
1427         write_exact( p_filter, p_sys->i_socket,
1428                      (char*)&ev, sz_rfbKeyEventMsg);
1429     }
1430
1431     /* then key-down for the pressed key */
1432     ev.key = i_key32;
1433     write_exact( p_filter, p_sys->i_socket,
1434                  (char*)&ev, sz_rfbKeyEventMsg);
1435
1436     ev.down = 0;
1437
1438     /* then key-up for the pressed key */
1439     write_exact( p_filter, p_sys->i_socket,
1440                  (char*)&ev, sz_rfbKeyEventMsg);
1441
1442     /* last key-down for modifier-keys */
1443     if (newval.i_int & KEY_MODIFIER_CTRL)
1444     {
1445         ev.key = 0xffe3;
1446         write_exact( p_filter, p_sys->i_socket,
1447                      (char*)&ev, sz_rfbKeyEventMsg);
1448     }
1449     if (newval.i_int & KEY_MODIFIER_SHIFT)
1450     {
1451         ev.key = 0xffe1;
1452         write_exact( p_filter, p_sys->i_socket,
1453                      (char*)&ev, sz_rfbKeyEventMsg);
1454     }
1455     if (newval.i_int & KEY_MODIFIER_ALT)
1456     {
1457        ev.key = 0xffe9;
1458        write_exact( p_filter, p_sys->i_socket,
1459                     (char*)&ev, sz_rfbKeyEventMsg);
1460     }
1461     vlc_mutex_unlock( &p_sys->lock );
1462
1463     return VLC_SUCCESS;
1464 }
1465
1466 static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd )
1467 {
1468     unsigned char key[8];
1469     unsigned int i;
1470
1471     for (i = 0; i < 8; i++)
1472         key[i] = i < strlen( passwd ) ? passwd[i] : '\0';
1473
1474     gcry_cipher_hd_t ctx;
1475     gcry_cipher_open( &ctx, GCRY_CIPHER_DES, GCRY_CIPHER_MODE_ECB,0);
1476
1477     /* reverse bits of the key */
1478     for( i = 0 ; i < 8 ; i ++ )
1479         key[i] =
1480             (key[i] >> 7) +
1481             (((key[i] >> 6) & 0x01 ) << 1 ) +
1482             (((key[i] >> 5) & 0x01 ) << 2 ) +
1483             (((key[i] >> 4) & 0x01 ) << 3 ) +
1484             (((key[i] >> 3) & 0x01 ) << 4 ) +
1485             (((key[i] >> 2) & 0x01 ) << 5 ) +
1486             (((key[i] >> 1) & 0x01 ) << 6 ) +
1487             ((key[i] & 0x01) << 7 );
1488
1489     gcry_cipher_setkey( ctx, key, 8 );
1490     gcry_cipher_encrypt( ctx, bytes, CHALLENGESIZE, bytes, CHALLENGESIZE );
1491     gcry_cipher_close( ctx );
1492 }
1493