git.sesse.net Git - vlc/blob - modules/video_filter/remoteosd.c

   1 /*****************************************************************************
   2  * remoteosd.c: remote osd over vnc filter module
   3  *****************************************************************************
   4  * Copyright (C) 2007-2008 Matthias Bauer
   5  * $Id$
   6  *
   7  * Authors: Matthias Bauer <matthias dot bauer #_at_# gmx dot ch>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implid warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  22  *****************************************************************************/
  23
  24 /*****************************************************************************
  25  * RemoteOSD uses the RFB-Protocol of VNC to display an On-Screen-Display
  26  * menu generated by a streaming server as overlay for the streamed video.
  27  *
  28  * The streaming server that implements this is the ffnetdev plugin for VDR.
  29  * VDR (VideoDiskRecorder) is an Linux based OpenSource harddisk recorder
  30  * software.
  31  * The VDR ffnetdev plugin emulates the hardware MPEG decoder and streams the
  32  * video over the network instead of hardware video outputs.
  33  * The OSD menu of VDR is offered with the RFB protocol to a VNC client.
  34  *
  35  * In fact this video-filter is a simple VNC client that could be also used to
  36  * connect to a real VNC host.
  37  * Only 8-bit color is supported at the moment.
  38  * Using password protected VNC hosts is supported but not recommended, because
  39  * you need to insert the used password in the plugin configuration page of
  40  * VLC configuration in plain text and it's saved in plain text.
  41  *****************************************************************************/
  42
  43 //#define VNC_DEBUG
  44
  45 /*****************************************************************************
  46  * Preamble
  47  *****************************************************************************/
  48
  49 #ifdef HAVE_CONFIG_H
  50 # include "config.h"
  51 #endif
  52
  53 #include <vlc_common.h>
  54 #include <vlc_plugin.h>
  55 #include <vlc_vout.h>
  56
  57 #include <vlc_filter.h>
  58 #include <vlc_image.h>
  59 #include <vlc_osd.h>
  60 #include <vlc_keys.h>
  61
  62 #include <vlc_network.h>
  63 #include <gcrypt.h>              /* to encrypt password */
  64 #include <vlc_gcrypt.h>
  65
  66 #include "remoteosd_rfbproto.h" /* type definitions of the RFB protocol for VNC */
  67
  68 /*****************************************************************************
  69  * Module descriptor
  70  *****************************************************************************/
  71 #define READ_BUFFER_SIZE 1000000
  72
  73 #define RMTOSD_HOST_TEXT N_("VNC Host")
  74 #define RMTOSD_HOST_LONGTEXT N_( \
  75     "VNC hostname or IP address." )
  76
  77 #define RMTOSD_PORT_TEXT N_("VNC Port")
  78 #define RMTOSD_PORT_LONGTEXT N_( \
  79     "VNC port number." )
  80
  81 #define RMTOSD_PASSWORD_TEXT N_("VNC Password")
  82 #define RMTOSD_PASSWORD_LONGTEXT N_( \
  83     "VNC password." )
  84
  85 #define RMTOSD_UPDATE_TEXT N_("VNC poll interval" )
  86 #define RMTOSD_UPDATE_LONGTEXT N_( \
  87     "In this interval an update from VNC is requested, default every 300 ms. ")
  88
  89 #define RMTOSD_POLL_TEXT N_("VNC polling")
  90 #define RMTOSD_POLL_LONGTEXT N_( \
  91     "Activate VNC polling. Do NOT activate for use as VDR ffnetdev client." )
  92
  93 #define RMTOSD_MOUSE_TEXT N_("Mouse events")
  94 #define RMTOSD_MOUSE_LONGTEXT N_( \
  95     "Send mouse events to VNC host. Not needed for use as VDR ffnetdev client." )
  96
  97 #define RMTOSD_KEYS_TEXT N_("Key events")
  98 #define RMTOSD_KEYS_LONGTEXT N_( \
  99     "Send key events to VNC host." )
 100
 101 #define RMTOSD_ALPHA_TEXT N_("Alpha transparency value (default 255)")
 102 #define RMTOSD_ALPHA_LONGTEXT N_( \
 103     "The transparency of the OSD VNC can be changed by giving a value " \
 104     "between 0 and 255. A lower value specifies more transparency a higher " \
 105     "means less transparency. The default is being not transparent " \
 106     "(value 255) the minimum is fully transparent (value 0)." )
 107
 108 #define RMTOSD_CFG "rmtosd-"
 109
 110 #define RMTOSD_UPDATE_MIN     200
 111 #define RMTOSD_UPDATE_DEFAULT 1000
 112 #define RMTOSD_UPDATE_MAX     300
 113
 114 static int  CreateFilter ( vlc_object_t * );
 115 static void DestroyFilter( vlc_object_t * );
 116
 117 vlc_module_begin ()
 118     set_description( N_("Remote-OSD over VNC") )
 119     set_capability( "sub filter", 100 )
 120     set_shortname( N_("Remote-OSD") )
 121     set_category( CAT_VIDEO )
 122     set_subcategory( SUBCAT_VIDEO_SUBPIC )
 123     add_shortcut( "rmtosd" )
 124     set_callbacks( CreateFilter, DestroyFilter )
 125
 126     add_string( RMTOSD_CFG "host", "myvdr", NULL, RMTOSD_HOST_TEXT,
 127         RMTOSD_HOST_LONGTEXT, false )
 128     add_integer_with_range( RMTOSD_CFG "port", 20001, 1, 0xFFFF, NULL,
 129         RMTOSD_PORT_TEXT, RMTOSD_PORT_LONGTEXT, false )
 130     add_password( RMTOSD_CFG "password", "", NULL, RMTOSD_PASSWORD_TEXT,
 131         RMTOSD_PASSWORD_LONGTEXT, false )
 132     add_integer_with_range( RMTOSD_CFG "update", RMTOSD_UPDATE_DEFAULT,
 133         RMTOSD_UPDATE_MIN, RMTOSD_UPDATE_MAX, NULL, RMTOSD_UPDATE_TEXT,
 134         RMTOSD_UPDATE_LONGTEXT, true )
 135     add_bool( RMTOSD_CFG "vnc-polling", false, NULL,
 136               RMTOSD_POLL_TEXT , RMTOSD_POLL_LONGTEXT, false )
 137     add_bool( RMTOSD_CFG "mouse-events", false, NULL,
 138               RMTOSD_MOUSE_TEXT , RMTOSD_MOUSE_LONGTEXT, false )
 139     add_bool( RMTOSD_CFG "key-events", false, NULL,
 140               RMTOSD_KEYS_TEXT , RMTOSD_KEYS_LONGTEXT, false )
 141     add_integer_with_range( RMTOSD_CFG "alpha", 255, 0, 255, NULL,
 142         RMTOSD_ALPHA_TEXT, RMTOSD_ALPHA_LONGTEXT, true )
 143
 144 vlc_module_end ()
 145
 146
 147 /*****************************************************************************
 148  * Local prototypes
 149  *****************************************************************************/
 150 #define CHALLENGESIZE 16
 151 #define MAX_VNC_SERVER_NAME_LENGTH 255
 152
 153 /* subfilter functions */
 154 static subpicture_t *Filter( filter_t *, mtime_t );
 155
 156 static int MouseEvent( filter_t *,
 157                        const vlc_mouse_t *,
 158                        const vlc_mouse_t *,
 159                        const video_format_t * );
 160
 161 static int KeyEvent( vlc_object_t *p_this, char const *psz_var,
 162                      vlc_value_t oldval, vlc_value_t newval, void *p_data );
 163
 164 static void stop_osdvnc ( filter_t *p_filter );
 165
 166 static void* vnc_worker_thread ( vlc_object_t *p_thread_obj );
 167
 168 static void* update_request_thread( vlc_object_t *p_thread_obj );
 169
 170 static bool open_vnc_connection ( filter_t *p_filter );
 171
 172 static bool handshaking ( filter_t *p_filter );
 173
 174 static bool process_server_message ( filter_t *p_filter,
 175                                      rfbServerToClientMsg *msg );
 176
 177 static inline void rgb_to_yuv( uint8_t *y, uint8_t *u, uint8_t *v,
 178                                int r, int g, int b );
 179
 180 static inline bool fill_rect( filter_sys_t* p_sys,
 181                               uint16_t i_x, uint16_t i_y,
 182                               uint16_t i_w, uint16_t i_h,
 183                               uint8_t i_color );
 184 static inline bool copy_rect( filter_sys_t* p_sys,
 185                               uint16_t i_x, uint16_t i_y,
 186                               uint16_t i_w, uint16_t i_h,
 187                               uint16_t i_sx, uint16_t i_sy );
 188
 189
 190 static inline bool raw_line(  filter_sys_t* p_sys,
 191                               uint16_t i_x, uint16_t i_y,
 192                               uint16_t i_w );
 193
 194 static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd );
 195
 196
 197 /*****************************************************************************
 198  * Sub filter code
 199  *****************************************************************************/
 200
 201 /*****************************************************************************
 202  * Local prototypes
 203  *****************************************************************************/
 204 struct filter_sys_t
 205 {
 206     bool          b_need_update;       /* VNC picture is updated, do update the OSD*/
 207     mtime_t       i_vnc_poll_interval; /* Update the OSD menu every n ms */
 208
 209     uint8_t       i_alpha;             /* alpha transparency value */
 210
 211     char          *psz_host;           /* VNC host */
 212     int           i_port;
 213
 214     char          *psz_passwd;         /* VNC password */
 215
 216     bool          b_vnc_poll;          /* Activate VNC polling ? */
 217     bool          b_vnc_mouse_events;  /* Send MouseEvents ? */
 218     bool          b_vnc_key_events;    /* Send KeyEvents ? */
 219
 220     bool          b_connection_active; /* Handshaking finished ? */
 221
 222     vlc_mutex_t   lock;                /* To lock for read/write on picture */
 223
 224     picture_t     *p_pic;              /* The picture with OSD data from VNC */
 225
 226     vout_thread_t *p_vout;             /* Pointer to video-out thread */
 227
 228     int           i_socket;            /* Socket used for VNC */
 229
 230     uint16_t      i_vnc_width;          /* The with of the VNC screen */
 231     uint16_t      i_vnc_height;         /* The height of the VNC screen */
 232         uint32_t      i_vnc_pixels;         /* The pixels of the VNC screen */
 233
 234     bool    b_alpha_from_vnc;    /* Special ffnetdev alpha feature enabled ? */
 235
 236     char          read_buffer[READ_BUFFER_SIZE];
 237
 238     bool          b_continue;
 239
 240     vlc_object_t* p_worker_thread;
 241
 242     uint8_t       ar_color_table_yuv[256][4];
 243 };
 244
 245 /*****************************************************************************
 246  * CreateFilter: Create the filter and open the definition file
 247  *****************************************************************************/
 248 static int CreateFilter ( vlc_object_t *p_this )
 249 {
 250     filter_t *p_filter = (filter_t *)p_this;
 251     filter_sys_t *p_sys = NULL;
 252
 253     msg_Dbg( p_filter, "Creating vnc osd filter..." );
 254
 255     p_filter->p_sys = p_sys = calloc( 1, sizeof(*p_sys) );
 256     if( !p_filter->p_sys )
 257         return VLC_ENOMEM;
 258
 259     /* Populating struct */
 260     vlc_mutex_init( &p_sys->lock );
 261     p_sys->b_continue = true;
 262     p_sys->i_socket = -1;
 263     p_sys->p_pic = NULL;
 264
 265     p_sys->psz_host = var_CreateGetString( p_this, RMTOSD_CFG "host" );
 266     if( EMPTY_STR(p_sys->psz_host) )
 267     {
 268         msg_Err( p_filter, "unable to get vnc host" );
 269         goto error;
 270     }
 271
 272     p_sys->psz_passwd = var_CreateGetString( p_this, RMTOSD_CFG "password" );
 273     if( !p_sys->psz_passwd )
 274     {
 275         msg_Err( p_filter, "unable to get vnc password" );
 276         goto error;
 277     }
 278
 279     p_sys->i_port = var_CreateGetIntegerCommand( p_this, RMTOSD_CFG "port" );
 280
 281     p_sys->i_alpha = var_CreateGetIntegerCommand( p_this, RMTOSD_CFG "alpha" );
 282
 283     /* in milliseconds, 0 disables polling, should not be lower than 100 */
 284     p_sys->i_vnc_poll_interval  = var_CreateGetIntegerCommand( p_this,
 285                                                        RMTOSD_CFG "update" );
 286     if ( p_sys->i_vnc_poll_interval < 100)
 287     {
 288        p_sys->i_vnc_poll_interval = 100;
 289     }
 290
 291     for ( int i = 0; i < 256; i++ )
 292     {
 293         p_sys->ar_color_table_yuv[i][0] = 255;
 294         p_sys->ar_color_table_yuv[i][1] = 255;
 295         p_sys->ar_color_table_yuv[i][2] = 255;
 296         p_sys->ar_color_table_yuv[i][3] = 255;
 297     }
 298
 299     p_sys->b_vnc_poll = var_CreateGetBoolCommand( p_this,
 300                                             RMTOSD_CFG "vnc-polling" );
 301     p_sys->b_vnc_mouse_events = var_CreateGetBoolCommand( p_this,
 302                                             RMTOSD_CFG "mouse-events" );
 303     p_sys->b_vnc_key_events = var_CreateGetBoolCommand( p_this,
 304                                             RMTOSD_CFG "key-events" );
 305
 306     /* Keep track of OSD Events */
 307     p_sys->b_need_update  = false;
 308
 309     /* Attach subpicture filter callback */
 310     p_filter->pf_sub_filter = Filter;
 311     p_filter->pf_sub_mouse  = MouseEvent;
 312
 313     p_sys->p_vout = vlc_object_find( p_this, VLC_OBJECT_VOUT, FIND_PARENT );
 314
 315     if( p_sys->p_vout )
 316     {
 317         var_AddCallback( p_sys->p_vout->p_libvlc, "key-pressed",
 318                          KeyEvent, p_this );
 319     }
 320
 321     es_format_Init( &p_filter->fmt_out, SPU_ES, VLC_CODEC_SPU );
 322     p_filter->fmt_out.i_priority = 0;
 323
 324     vlc_gcrypt_init();
 325
 326     /* create the vnc worker thread */
 327     p_sys->p_worker_thread = vlc_object_create( p_this,
 328                                                 sizeof( vlc_object_t ) );
 329     vlc_object_attach( p_sys->p_worker_thread, p_this );
 330     if( vlc_thread_create( p_sys->p_worker_thread, "vnc worker thread",
 331                            vnc_worker_thread, VLC_THREAD_PRIORITY_LOW ) )
 332     {
 333         vlc_object_release( p_sys->p_worker_thread );
 334         msg_Err( p_filter, "cannot spawn vnc message reader thread" );
 335         goto error;
 336     }
 337
 338     msg_Dbg( p_filter, "osdvnc filter started" );
 339
 340     return VLC_SUCCESS;
 341
 342 error:
 343     msg_Err( p_filter, "osdvnc filter discarded" );
 344
 345     stop_osdvnc( p_filter );
 346
 347     vlc_mutex_destroy( &p_sys->lock );
 348     free( p_sys->psz_host );
 349     free( p_sys->psz_passwd );
 350     free( p_sys );
 351
 352     return VLC_EGENERIC;
 353 }
 354
 355 /*****************************************************************************
 356  * DestroyFilter: Make a clean exit of this plugin
 357  *****************************************************************************/
 358 static void DestroyFilter( vlc_object_t *p_this )
 359 {
 360     filter_t     *p_filter = (filter_t*)p_this;
 361     filter_sys_t *p_sys = p_filter->p_sys;
 362
 363     msg_Dbg( p_filter, "DestroyFilter called." );
 364
 365     stop_osdvnc( p_filter );
 366
 367     if( p_sys->p_vout )
 368     {
 369         var_DelCallback( p_sys->p_vout->p_libvlc, "key-pressed",
 370                          KeyEvent, p_this );
 371
 372         vlc_object_release( p_sys->p_vout );
 373     }
 374
 375     var_Destroy( p_this, RMTOSD_CFG "host" );
 376     var_Destroy( p_this, RMTOSD_CFG "port" );
 377     var_Destroy( p_this, RMTOSD_CFG "password" );
 378     var_Destroy( p_this, RMTOSD_CFG "update" );
 379     var_Destroy( p_this, RMTOSD_CFG "vnc-polling" );
 380     var_Destroy( p_this, RMTOSD_CFG "mouse-events" );
 381     var_Destroy( p_this, RMTOSD_CFG "key-events" );
 382     var_Destroy( p_this, RMTOSD_CFG "alpha" );
 383
 384     vlc_mutex_destroy( &p_sys->lock );
 385     free( p_sys->psz_host );
 386     free( p_sys->psz_passwd );
 387     free( p_sys );
 388 }
 389
 390 static void stop_osdvnc ( filter_t *p_filter )
 391 {
 392     filter_sys_t *p_sys = p_filter->p_sys;
 393
 394     /* It will unlock socket reading */
 395     vlc_object_kill( p_filter );
 396
 397     /* */
 398     if( p_sys->p_worker_thread )
 399     {
 400         msg_Dbg( p_filter, "joining worker_thread" );
 401         vlc_object_kill( p_sys->p_worker_thread );
 402         vlc_thread_join( p_sys->p_worker_thread );
 403         vlc_object_release( p_sys->p_worker_thread );
 404         msg_Dbg( p_filter, "released worker_thread" );
 405     }
 406
 407     msg_Dbg( p_filter, "osdvnc stopped" );
 408 }
 409
 410 static bool read_exact( filter_t *p_filter,
 411                         int i_socket,
 412                         char* p_readbuf,
 413                         int i_bytes )
 414 {
 415     return i_bytes == net_Read( p_filter, i_socket, NULL,
 416                                   (unsigned char*)p_readbuf,
 417                                   i_bytes, true );
 418 }
 419
 420
 421 static bool write_exact( filter_t *p_filter,
 422                          int i_socket,
 423                          char* p_writebuf,
 424                          int i_bytes )
 425 {
 426     return i_bytes == net_Write( p_filter, i_socket, NULL,
 427                                   (unsigned char*)p_writebuf, i_bytes );
 428 }
 429
 430 static bool open_vnc_connection ( filter_t *p_filter )
 431 {
 432     filter_sys_t *p_sys = p_filter->p_sys;
 433
 434     msg_Dbg( p_filter, "Open socket to vnc server on %s:%u.",
 435               p_sys->psz_host, p_sys->i_port );
 436
 437     p_sys->i_socket = net_ConnectTCP( p_filter, p_sys->psz_host, p_sys->i_port );
 438
 439     if( p_sys->i_socket < 0 )
 440     {
 441         msg_Err( p_filter, "Could not open socket" );
 442         return false;
 443     }
 444
 445     msg_Dbg( p_filter, "socket is open." );
 446
 447     return true;
 448 }
 449
 450 static bool handshaking ( filter_t *p_filter )
 451 {
 452     filter_sys_t *p_sys = p_filter->p_sys;
 453
 454     msg_Dbg( p_filter, "Reading protocol version" );
 455
 456     rfbProtocolVersionMsg pv;
 457     if ( !read_exact( p_filter, p_sys->i_socket, pv,
 458                       sz_rfbProtocolVersionMsg ) )
 459     {
 460         msg_Err( p_filter, "Could not read version message" );
 461         return false;
 462     }
 463     pv[sz_rfbProtocolVersionMsg] = '\0'; /* pv size is sz_rfbProtocolVersionMsg+1 */
 464
 465     msg_Dbg( p_filter, "Server version is %s", pv );
 466
 467     strncpy(pv, "RFB 003.003\n", sz_rfbProtocolVersionMsg);
 468
 469     if( !write_exact(p_filter, p_sys->i_socket, pv,
 470                      sz_rfbProtocolVersionMsg) )
 471     {
 472         msg_Err( p_filter, "Could not write version message" );
 473         return false;
 474     }
 475
 476     msg_Dbg( p_filter, "Reading authentication scheme" );
 477     uint32_t i_authScheme;
 478     if( !read_exact( p_filter, p_sys->i_socket, (char*)&i_authScheme, 4 ) )
 479     {
 480         msg_Err( p_filter, "Could not read authentication scheme" );
 481         return false;
 482     }
 483     i_authScheme = htonl(i_authScheme);
 484
 485     msg_Dbg( p_filter, "Authentication scheme = %x", i_authScheme );
 486     if ( i_authScheme == rfbConnFailed )
 487     {
 488         msg_Err( p_filter, "Connection rejected by server" );
 489         return false;
 490     }
 491     if (i_authScheme == rfbVncAuth)
 492     {
 493         unsigned char challenge[CHALLENGESIZE];
 494         if ( !read_exact( p_filter, p_sys->i_socket,
 495                           (char*)challenge, CHALLENGESIZE ) )
 496         {
 497             msg_Err( p_filter, "Could not read password challenge" );
 498             return false;
 499         }
 500
 501         vnc_encrypt_bytes( challenge, p_sys->psz_passwd );
 502
 503         if( !write_exact(p_filter, p_sys->i_socket,
 504                          (char*)challenge, CHALLENGESIZE ) )
 505         {
 506             msg_Err( p_filter, "Could not write password" );
 507             return false;
 508         }
 509         uint32_t i_authResult;
 510         if( !read_exact( p_filter, p_sys->i_socket, (char*)&i_authResult, 4 ) )
 511         {
 512             msg_Err( p_filter, "Could not read authentication result" );
 513             return false;
 514         }
 515         i_authResult = htonl(i_authResult);
 516         if (i_authResult != rfbVncAuthOK)
 517         {
 518             msg_Err( p_filter, "VNC authentication failed" );
 519             return false;
 520         }
 521     }
 522
 523     msg_Dbg( p_filter, "Writing client init message" );
 524     rfbClientInitMsg ci;
 525     ci.shared = 1;
 526     if( !write_exact( p_filter, p_sys->i_socket,
 527                       (char*)&ci, sz_rfbClientInitMsg ) )
 528     {
 529         msg_Err( p_filter, "Could not write client init message" );
 530         return false;
 531     }
 532
 533     msg_Dbg( p_filter, "Reading server init message" );
 534     rfbServerInitMsg si;
 535     if( !read_exact( p_filter, p_sys->i_socket,
 536                      (char*)&si, sz_rfbServerInitMsg ) )
 537     {
 538         msg_Err( p_filter, "Could not read server init message" );
 539         return false;
 540     }
 541     si.framebufferWidth = htons(si.framebufferWidth);
 542     si.framebufferHeight = htons(si.framebufferHeight);
 543     si.format.redMax = htons(si.format.redMax);
 544     si.format.greenMax = htons(si.format.greenMax);
 545     si.format.blueMax = htons(si.format.blueMax);
 546
 547     p_sys->i_vnc_width = si.framebufferWidth;
 548     p_sys->i_vnc_height = si.framebufferHeight;
 549
 550     msg_Dbg( p_filter, "Servers preferred pixelformat: "
 551                         "%ux%u, R(%u),G(%u),B(%u), %u bit, depht=%u, %s",
 552                         si.framebufferWidth,
 553                         si.framebufferHeight,
 554                         si.format.redMax,
 555                         si.format.greenMax,
 556                         si.format.blueMax,
 557                         si.format.bitsPerPixel,
 558                         si.format.depth,
 559                         si.format.trueColour ? "TrueColor" : "Not-TrueColor");
 560
 561     uint32_t i_nameLength = htonl(si.nameLength);
 562     if( i_nameLength > MAX_VNC_SERVER_NAME_LENGTH )
 563     {
 564         msg_Err( p_filter, "Server name too long" );
 565         return false;
 566     }
 567     char s_ServerName[MAX_VNC_SERVER_NAME_LENGTH+1];
 568
 569     msg_Dbg( p_filter, "Reading server name with size = %u", i_nameLength );
 570     if( !read_exact( p_filter, p_sys->i_socket, s_ServerName, i_nameLength ) )
 571     {
 572         msg_Err( p_filter, "Could not read server name" );
 573         return false;
 574     }
 575     s_ServerName[i_nameLength] = '\0';
 576
 577     if( strcmp( s_ServerName, "VDR-OSD") == 0 )
 578     {
 579         msg_Dbg( p_filter, "Server is a VDR" );
 580         p_sys->b_alpha_from_vnc = true;
 581     }
 582     else
 583     {
 584         msg_Dbg( p_filter, "Server is a normal VNC" );
 585         p_sys->b_alpha_from_vnc = false;
 586     }
 587
 588
 589     msg_Dbg( p_filter, "Server init message read properly" );
 590     msg_Dbg( p_filter, "Server name is %s", s_ServerName );
 591
 592     msg_Dbg( p_filter, "Writing SetPixelFormat message" );
 593
 594     rfbSetPixelFormatMsg sp;
 595     sp.type = rfbSetPixelFormat;
 596     sp.pad1 = sp.pad2 = 0;
 597     sp.format.bitsPerPixel = 8;
 598     sp.format.depth = 8 ;
 599     sp.format.bigEndian = 1;
 600     sp.format.trueColour = 0;
 601     sp.format.redMax = htons(31);
 602     sp.format.greenMax = htons(31);
 603     sp.format.blueMax = htons(31);
 604     sp.format.redShift = 10;
 605     sp.format.greenShift = 5;
 606     sp.format.blueShift = 0;
 607     sp.format.pad1 = sp.format.pad2 = 0;
 608
 609     if( !write_exact( p_filter, p_sys->i_socket,
 610                       (char*)&sp, sz_rfbSetPixelFormatMsg) )
 611     {
 612         msg_Err( p_filter, "Could not write SetPixelFormat message" );
 613         return false;
 614     }
 615
 616     msg_Dbg( p_filter, "Writing SetEncodings message" );
 617
 618     rfbSetEncodingsMsg se;
 619     se.type = rfbSetEncodings;
 620     se.pad = 0;
 621     se.nEncodings = htons( p_sys->b_alpha_from_vnc ? 3 : 2 );
 622
 623     if( !write_exact( p_filter, p_sys->i_socket,
 624                       (char*)&se, sz_rfbSetEncodingsMsg) )
 625     {
 626         msg_Err( p_filter, "Could not write SetEncodings message begin" );
 627         return false;
 628     }
 629
 630     uint32_t i_encoding;
 631
 632     msg_Dbg( p_filter, "Writing SetEncodings rfbEncodingCopyRect" );
 633     i_encoding = htonl(rfbEncodingCopyRect);
 634     if( !write_exact( p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
 635     {
 636         msg_Err( p_filter, "Could not write encoding type rfbEncodingCopyRect." );
 637         return false;
 638     }
 639
 640     msg_Dbg( p_filter, "Writing SetEncodings rfbEncodingRRE" );
 641     i_encoding = htonl(rfbEncodingRRE);
 642     if( !write_exact(p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
 643     {
 644         msg_Err( p_filter, "Could not write encoding type rfbEncodingRRE." );
 645         return false;
 646     }
 647
 648     if( p_sys->b_alpha_from_vnc )
 649     {
 650         msg_Dbg( p_filter, "Writing SetEncodings rfbEncSpecialUseAlpha" );
 651         i_encoding = 0x00F0FFFF; /* rfbEncSpecialUseAlpha is 0xFFFFF000
 652                                   * before we swap it */
 653         if( !write_exact(p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
 654         {
 655             msg_Err( p_filter, "Could not write encoding type rfbEncSpecialUseAlpha." );
 656             return false;
 657         }
 658     }
 659     return true;
 660
 661 }
 662
 663 static void* vnc_worker_thread( vlc_object_t *p_thread_obj )
 664 {
 665     filter_t* p_filter = (filter_t*)(p_thread_obj->p_parent);
 666     filter_sys_t *p_sys = p_filter->p_sys;
 667     vlc_object_t *p_update_request_thread;
 668     int canc = vlc_savecancel ();
 669
 670     msg_Dbg( p_filter, "VNC worker thread started" );
 671
 672     if( !open_vnc_connection ( p_filter ) )
 673     {
 674         msg_Err( p_filter, "Could not connect to vnc host" );
 675         goto exit;
 676     }
 677
 678     if( !handshaking ( p_filter ) )
 679     {
 680         msg_Err( p_filter, "Error occured while handshaking vnc host" );
 681         goto exit;
 682     }
 683
 684     p_sys->b_connection_active = true; /* to enable sending key
 685                                             * and mouse events to host */
 686
 687     /* Create an empty picture for VNC the data */
 688     vlc_mutex_lock( &p_sys->lock );
 689     p_sys->p_pic = picture_New( VLC_CODEC_YUVA,
 690                                 p_sys->i_vnc_width, p_sys->i_vnc_height, 1, 1 );
 691     if( !p_sys->p_pic )
 692     {
 693         vlc_mutex_unlock( &p_sys->lock );
 694         goto exit;
 695     }
 696         p_sys->i_vnc_pixels = p_sys->i_vnc_width * p_sys->i_vnc_height;
 697
 698     vlc_mutex_unlock( &p_sys->lock );
 699
 700     /* create the update request thread */
 701     p_update_request_thread = vlc_object_create( p_filter,
 702                                                  sizeof( vlc_object_t ) );
 703     vlc_object_attach( p_update_request_thread, p_filter );
 704     if( vlc_thread_create( p_update_request_thread,
 705                            "vnc update request thread",
 706                            update_request_thread, VLC_THREAD_PRIORITY_LOW ) )
 707     {
 708         vlc_object_release( p_update_request_thread );
 709         msg_Err( p_filter, "cannot spawn vnc update request thread" );
 710         goto exit;
 711     }
 712
 713     /* connection is initialized, now read and handle server messages */
 714     while( vlc_object_alive( p_thread_obj ) )
 715     {
 716         rfbServerToClientMsg msg;
 717         int i_msgSize;
 718
 719         memset( &msg, 0, sizeof(msg) );
 720
 721         if( !read_exact(p_filter, p_sys->i_socket, (char*)&msg, 1 ) )
 722         {
 723             msg_Err( p_filter, "Error while waiting for next server message");
 724             break;
 725         }
 726         switch (msg.type)
 727         {
 728         case rfbFramebufferUpdate:
 729             i_msgSize = sz_rfbFramebufferUpdateMsg;
 730             break;
 731         case rfbSetColourMapEntries:
 732             i_msgSize = sz_rfbSetColourMapEntriesMsg;
 733             break;
 734         case rfbBell:
 735             i_msgSize = sz_rfbBellMsg;
 736             break;
 737         case rfbServerCutText:
 738             i_msgSize = sz_rfbServerCutTextMsg;
 739             break;
 740         case rfbReSizeFrameBuffer:
 741             i_msgSize = sz_rfbReSizeFrameBufferMsg;
 742             break;
 743         default:
 744             i_msgSize = 0;
 745             msg_Err( p_filter, "Invalid message %u received", msg.type );
 746             break;
 747         }
 748
 749         if( i_msgSize <= 0 )
 750             break;
 751
 752         if( --i_msgSize > 0 )
 753         {
 754             if ( !read_exact( p_filter, p_sys->i_socket,
 755                               ((char*)&msg)+1, i_msgSize ) )
 756             {
 757                 msg_Err( p_filter, "Error while reading message of type %u",
 758                          msg.type );
 759                 break;
 760             }
 761         }
 762         process_server_message( p_filter, &msg);
 763     }
 764
 765     msg_Dbg( p_filter, "joining update_request_thread" );
 766     vlc_object_kill( p_update_request_thread );
 767     vlc_thread_join( p_update_request_thread );
 768     vlc_object_release( p_update_request_thread );
 769     msg_Dbg( p_filter, "released update_request_thread" );
 770
 771 exit:
 772
 773     vlc_mutex_lock( &p_sys->lock );
 774     p_sys->b_connection_active = false;
 775
 776     if (p_sys->i_socket >= 0)
 777         net_Close(p_sys->i_socket);
 778
 779     if( p_sys->p_pic )
 780         picture_Release( p_sys->p_pic );
 781
 782     /* It will hide the subtitle */
 783     p_sys->b_continue = false;
 784     p_sys->b_need_update = true;
 785     vlc_mutex_unlock( &p_sys->lock );
 786
 787     msg_Dbg( p_filter, "VNC message reader thread ended" );
 788     vlc_restorecancel (canc);
 789     return NULL;
 790 }
 791
 792 static void* update_request_thread( vlc_object_t *p_thread_obj )
 793 {
 794     filter_t* p_filter = (filter_t*)(p_thread_obj->p_parent);
 795     filter_sys_t *p_sys = p_filter->p_sys;
 796     int canc = vlc_savecancel ();
 797
 798     msg_Dbg( p_filter, "VNC update request thread started" );
 799
 800     rfbFramebufferUpdateRequestMsg udr;
 801     udr.type = rfbFramebufferUpdateRequest;
 802     udr.incremental = 0;
 803     udr.x = 0;
 804     udr.y = 0;
 805     udr.w = htons(p_sys->i_vnc_width);
 806     udr.h = htons(p_sys->i_vnc_height);
 807
 808     if( write_exact(p_filter, p_sys->i_socket, (char*)&udr,
 809            sz_rfbFramebufferUpdateRequestMsg) == false)
 810     {
 811         msg_Err( p_filter, "Could not write rfbFramebufferUpdateRequestMsg." );
 812         p_sys->b_continue = false;
 813         return NULL;
 814     }
 815
 816     udr.incremental = 1;
 817     mtime_t i_poll_interval_microsec = p_sys->i_vnc_poll_interval * 1000;
 818
 819     if( p_sys->b_vnc_poll)
 820     {
 821         while( vlc_object_alive( p_thread_obj ) )
 822         {
 823             msleep( i_poll_interval_microsec );
 824             if( write_exact(p_filter, p_sys->i_socket, (char*)&udr,
 825                    sz_rfbFramebufferUpdateRequestMsg) == false)
 826             {
 827                 msg_Err( p_filter, "Could not write rfbFramebufferUpdateRequestMsg." );
 828                 break;
 829             }
 830         }
 831         p_sys->b_continue = false;
 832     }
 833     else
 834     {
 835         msg_Dbg( p_filter, "VNC polling disabled." );
 836     }
 837
 838     msg_Dbg( p_filter, "VNC update request thread ended" );
 839     vlc_restorecancel (canc);
 840     return NULL;
 841 }
 842
 843 static bool process_server_message ( filter_t *p_filter,
 844                                      rfbServerToClientMsg *msg )
 845 {
 846     filter_sys_t *p_sys = p_filter->p_sys;
 847
 848     switch (msg->type)
 849     {
 850     case rfbFramebufferUpdate:
 851         {
 852             msg->fu.nRects = htons(msg->fu.nRects);
 853             rfbFramebufferUpdateRectHeader hdr;
 854
 855             for (int i_rect = 0; i_rect < msg->fu.nRects; i_rect++)
 856             {
 857                 if (!read_exact(p_filter, p_sys->i_socket, (char*)&hdr,
 858                     sz_rfbFramebufferUpdateRectHeader ) )
 859                 {
 860                     msg_Err( p_filter, "Could not read FrameBufferUpdate header" );
 861                     return false;
 862                 }
 863                 hdr.r.x = htons(hdr.r.x);
 864                 hdr.r.y = htons(hdr.r.y);
 865                 hdr.r.w = htons(hdr.r.w);
 866                 hdr.r.h = htons(hdr.r.h);
 867                 hdr.encoding = htonl(hdr.encoding);
 868
 869                 switch (hdr.encoding)
 870                 {
 871                 case rfbEncodingRaw:
 872                     {
 873                         int i_line;
 874                         for (i_line = 0; i_line < hdr.r.h; i_line++)
 875                         {
 876                             if ( !read_exact( p_filter, p_sys->i_socket,
 877                                     p_sys->read_buffer, hdr.r.w ) )
 878                             {
 879                                 msg_Err( p_filter,
 880                                  "Could not read FrameBufferUpdate line data" );
 881                                return false;
 882                             }
 883                             vlc_mutex_lock( &p_sys->lock );
 884                             if ( !raw_line( p_sys, hdr.r.x,
 885                                             hdr.r.y + i_line,
 886                                             hdr.r.w ) )
 887                             {
 888                                 msg_Err( p_filter, "raw_line failed." );
 889                                 vlc_mutex_unlock( &p_sys->lock );
 890                                 return false;
 891                             }
 892                             vlc_mutex_unlock( &p_sys->lock );
 893                         }
 894                     }
 895                     break;
 896
 897                 case rfbEncodingCopyRect:
 898                     {
 899                         rfbCopyRect rect;
 900
 901                         if ( !read_exact( p_filter, p_sys->i_socket,
 902                                           (char*)&rect,
 903                                           sz_rfbCopyRect ) )
 904                         {
 905                             msg_Err( p_filter, "Could not read rfbCopyRect" );
 906                             return false;
 907                         }
 908                         rect.srcX = htons( rect.srcX );
 909                         rect.srcY = htons( rect.srcY );
 910
 911                         vlc_mutex_lock( &p_sys->lock );
 912                         if ( !copy_rect( p_sys,
 913                                          hdr.r.x,   hdr.r.y,
 914                                          hdr.r.w,   hdr.r.h,
 915                                          rect.srcX, rect.srcY ) )
 916                         {
 917                             msg_Err( p_filter, "copy_rect failed." );
 918                             vlc_mutex_unlock( &p_sys->lock );
 919                             return false;
 920                         }
 921                         vlc_mutex_unlock( &p_sys->lock );
 922                     }
 923                     break;
 924
 925                 case rfbEncodingRRE:
 926                     {
 927                         rfbRREHeader rrehdr;
 928                         if ( !read_exact( p_filter, p_sys->i_socket,
 929                                           (char*)&rrehdr,
 930                                           sz_rfbRREHeader ) )
 931                         {
 932                             msg_Err( p_filter, "Could not read rfbRREHeader" );
 933                             return false;
 934                         }
 935                         uint8_t i_pixcolor;
 936                         if ( !read_exact(p_filter, p_sys->i_socket,
 937                                          (char*)&i_pixcolor, 1 ) )
 938                         {
 939                             msg_Err( p_filter, "Could not read RRE pixcolor" );
 940                             return false;
 941                         }
 942
 943                         vlc_mutex_lock( &p_sys->lock );
 944                         if ( !fill_rect( p_sys,
 945                                         hdr.r.x, hdr.r.y,
 946                                         hdr.r.w, hdr.r.h,
 947                                         i_pixcolor) )
 948                         {
 949                             msg_Err( p_filter, "main fill_rect failed." );
 950                             vlc_mutex_unlock( &p_sys->lock );
 951                             return false;
 952                         }
 953                         vlc_mutex_unlock( &p_sys->lock );
 954
 955                         rrehdr.nSubrects = htonl(rrehdr.nSubrects);
 956
 957                         int i_datasize = rrehdr.nSubrects *
 958                                      ( sizeof(i_pixcolor) + sz_rfbRectangle ) ;
 959                         if ( i_datasize > READ_BUFFER_SIZE )
 960                         {
 961                             msg_Err( p_filter, "Buffer too small, "
 962                                      "need %u bytes", i_datasize );
 963                             return false;
 964                         }
 965                         if ( !read_exact( p_filter, p_sys->i_socket,
 966                                        p_sys->read_buffer, i_datasize ) )
 967                         {
 968                             msg_Err( p_filter,
 969                                      "Could not read RRE subrect data" );
 970                             return false;
 971                         }
 972
 973                         uint32_t i_subrect;
 974                         rfbRectangle* p_subrect;
 975                         int i_offset = 0;
 976                         vlc_mutex_lock( &p_sys->lock );
 977                         for ( i_subrect = 0;
 978                               i_subrect < rrehdr.nSubrects; i_subrect++)
 979                         {
 980                             i_pixcolor = p_sys->read_buffer[i_offset];
 981                             i_offset += sizeof(i_pixcolor);
 982                             p_subrect =
 983                                (rfbRectangle*)(p_sys->read_buffer + i_offset);
 984                             i_offset += sz_rfbRectangle;
 985
 986                             if (!fill_rect( p_sys,
 987                                             htons(p_subrect->x) + hdr.r.x,
 988                                             htons(p_subrect->y) + hdr.r.y,
 989                                             htons(p_subrect->w),
 990                                             htons(p_subrect->h),
 991                                             i_pixcolor) )
 992                             {
 993                                 msg_Err( p_filter,
 994                                     "subrect %u fill_rect failed.", i_subrect );
 995                                 vlc_mutex_unlock( &p_sys->lock );
 996                                 return false;
 997                             }
 998                         }
 999                         vlc_mutex_unlock( &p_sys->lock );
1000                     }
1001                     break;
1002
1003                 }
1004
1005             }
1006             vlc_mutex_lock( &p_sys->lock );
1007             p_sys->b_need_update = true;
1008             vlc_mutex_unlock( &p_sys->lock );
1009         }
1010         return true;
1011
1012     case rfbSetColourMapEntries:
1013         {
1014             msg->scme.nColours = htons(msg->scme.nColours);
1015             msg->scme.firstColour = htons(msg->scme.firstColour);
1016             int i_datasize;
1017             if ( p_sys->b_alpha_from_vnc == true )
1018             {
1019                 i_datasize = 2 * msg->scme.nColours * 4;
1020             }
1021             else
1022             {
1023                 i_datasize = 2 * msg->scme.nColours * 3;
1024             }
1025             if ( i_datasize > READ_BUFFER_SIZE )
1026             {
1027                 msg_Err( p_filter, "Buffer too small, need %u bytes",
1028                                    i_datasize );
1029                 return false;
1030             }
1031
1032             if ( !read_exact( p_filter, p_sys->i_socket,
1033                               p_sys->read_buffer, i_datasize ) )
1034             {
1035                 msg_Err( p_filter, "Could not read color map data" );
1036                 return false;
1037             }
1038
1039             uint8_t i_red, i_green, i_blue, i_alpha, i_color_index;
1040             uint16_t i_offset = 0;
1041             i_alpha = 255;
1042
1043             for (int i = 0; i < msg->scme.nColours; i++)
1044             {
1045                 i_color_index = i+msg->scme.firstColour;
1046                 if ( p_sys->b_alpha_from_vnc == true )
1047                 {
1048                     i_alpha = p_sys->read_buffer[i_offset];
1049                     i_offset += 2;
1050                 }
1051                 i_red   = p_sys->read_buffer[i_offset];
1052                 i_offset += 2;
1053                 i_green = p_sys->read_buffer[i_offset];
1054                 i_offset += 2;
1055                 i_blue  = p_sys->read_buffer[i_offset];
1056                 i_offset += 2;
1057                 rgb_to_yuv( &p_sys->ar_color_table_yuv[i_color_index][0],
1058                             &p_sys->ar_color_table_yuv[i_color_index][1],
1059                             &p_sys->ar_color_table_yuv[i_color_index][2],
1060                             i_red,
1061                             i_green,
1062                             i_blue );
1063                 p_sys->ar_color_table_yuv[i][3] = i_alpha;
1064             }
1065         }
1066         return true;
1067
1068     case rfbBell:
1069         msg_Err( p_filter, "rfbBell received" );
1070         return true;
1071
1072     case rfbServerCutText:
1073         msg->sct.length = htons(msg->sct.length);
1074         if ( msg->sct.length > READ_BUFFER_SIZE )
1075         {
1076             msg_Err( p_filter, "Buffer too small, need %u bytes", msg->sct.length );
1077             return false;
1078         }
1079         if ( !read_exact(p_filter, p_sys->i_socket,
1080                          p_sys->read_buffer, msg->sct.length ) )
1081         {
1082             msg_Err( p_filter, "Could not read Reading rfbServerCutText data" );
1083             return false;
1084         }
1085         return true;
1086
1087     case rfbReSizeFrameBuffer:
1088         msg_Err( p_filter, "Reading rfbReSizeFrameBuffer not implemented" );
1089         return false;
1090
1091     default:
1092         msg_Err( p_filter, "Invalid message %u received", msg->type );
1093         return false;
1094     }
1095     return false;
1096 }
1097
1098 /****************************************************************************
1099  * Filter: the whole thing
1100  ****************************************************************************
1101  * This function outputs subpictures at regular time intervals.
1102  ****************************************************************************/
1103 static subpicture_t *Filter( filter_t *p_filter, mtime_t date )
1104 {
1105     filter_sys_t *p_sys = p_filter->p_sys;
1106     subpicture_t *p_spu;
1107     subpicture_region_t *p_region;
1108     video_format_t fmt;
1109     picture_t *p_pic;
1110
1111     if( !p_sys->b_need_update )
1112     {
1113         return NULL;
1114     }
1115
1116     vlc_mutex_lock( &p_sys->lock );
1117
1118     p_pic = p_sys->p_pic;
1119
1120     if( !p_pic )
1121     {
1122         vlc_mutex_unlock( &p_sys->lock );
1123         return NULL;
1124     }
1125
1126     /* Allocate the subpicture internal data. */
1127     p_spu = filter_NewSubpicture( p_filter );
1128     if( !p_spu )
1129     {
1130         vlc_mutex_unlock( &p_sys->lock );
1131         return NULL;
1132     }
1133
1134     p_spu->b_absolute = false;
1135     p_spu->i_start = date;
1136     p_spu->i_stop = 0;
1137     p_spu->b_ephemer = true;
1138
1139     if( !p_sys->b_continue )
1140         p_spu->i_stop = p_spu->i_start + 1;
1141
1142     /* Create new SPU region */
1143     memset( &fmt, 0, sizeof(video_format_t) );
1144     fmt.i_chroma = VLC_CODEC_YUVA;
1145     fmt.i_sar_num = fmt.i_sar_den = 1;
1146     fmt.i_width = fmt.i_visible_width = p_pic->p[Y_PLANE].i_visible_pitch;
1147     fmt.i_height = fmt.i_visible_height = p_pic->p[Y_PLANE].i_visible_lines;
1148     fmt.i_x_offset = fmt.i_y_offset = 0;
1149     p_region = subpicture_region_New( &fmt );
1150     if( !p_region )
1151     {
1152         msg_Err( p_filter, "cannot allocate SPU region" );
1153         p_filter->pf_sub_buffer_del( p_filter, p_spu );
1154         vlc_mutex_unlock( &p_sys->lock );
1155         return NULL;
1156     }
1157
1158     /* FIXME the copy is probably not needed anymore */
1159     picture_Copy( p_region->p_picture, p_pic );
1160
1161     p_sys->b_need_update = false;
1162
1163     vlc_mutex_unlock( &p_sys->lock );
1164
1165     /* set to one of the 9 relative locations */
1166     p_region->i_align = 0; /* Center */
1167     p_spu->b_absolute = false;
1168
1169
1170     p_spu->i_original_picture_width = 0; /*Let vout core do the horizontal scaling */
1171     p_spu->i_original_picture_height = fmt.i_height;
1172
1173     p_spu->p_region = p_region;
1174
1175     p_spu->i_alpha = ( p_sys->i_alpha );
1176
1177     return p_spu;
1178 }
1179
1180
1181 static inline void rgb_to_yuv( uint8_t *y, uint8_t *u, uint8_t *v,
1182                                int r, int g, int b )
1183 {
1184     *y = ( ( (  66 * r + 129 * g +  25 * b + 128 ) >> 8 ) + 16 );
1185     *u =   ( ( -38 * r -  74 * g + 112 * b + 128 ) >> 8 ) + 128 ;
1186     *v =   ( ( 112 * r -  94 * g -  18 * b + 128 ) >> 8 ) + 128 ;
1187 }
1188
1189 static inline bool fill_rect( filter_sys_t* p_sys,
1190                               uint16_t i_x, uint16_t i_y,
1191                               uint16_t i_w, uint16_t i_h,
1192                               uint8_t i_color)
1193 {
1194     plane_t *p_outY = p_sys->p_pic->p+Y_PLANE;
1195     plane_t *p_outU = p_sys->p_pic->p+U_PLANE;
1196     plane_t *p_outV = p_sys->p_pic->p+V_PLANE;
1197     plane_t *p_outA = p_sys->p_pic->p+A_PLANE;
1198     int i_pitch = p_outY->i_pitch;
1199     int i_lines = p_outY->i_lines;
1200     if ( i_x + i_w > i_pitch)
1201         return false;
1202     if ( i_y + i_h > i_lines)
1203         return false;
1204     int i_line_offset = i_y * i_pitch;
1205     uint8_t i_yuv_y = p_sys->ar_color_table_yuv[i_color][0];
1206     uint8_t i_yuv_u = p_sys->ar_color_table_yuv[i_color][1];
1207     uint8_t i_yuv_v = p_sys->ar_color_table_yuv[i_color][2];
1208     uint8_t i_alpha = p_sys->ar_color_table_yuv[i_color][3];
1209     for( int i_line = 0; i_line < i_h; i_line++ )
1210     {
1211         for( int i_column = 0; i_column < i_w; i_column++ )
1212         {
1213             int i_total_offset = i_line_offset + i_x + i_column;
1214             p_outY->p_pixels[ i_total_offset ] = i_yuv_y;
1215             p_outU->p_pixels[ i_total_offset ] = i_yuv_u;
1216             p_outV->p_pixels[ i_total_offset ] = i_yuv_v;
1217             p_outA->p_pixels[ i_total_offset ] = i_alpha;
1218         }
1219         i_line_offset += i_pitch;
1220     }
1221     return true;
1222 }
1223
1224 static inline bool copy_rect( filter_sys_t* p_sys,
1225                               uint16_t i_x, uint16_t i_y,
1226                               uint16_t i_w, uint16_t i_h,
1227                               uint16_t i_sx, uint16_t i_sy )
1228 {
1229     plane_t *p_Y = p_sys->p_pic->p+Y_PLANE;
1230     plane_t *p_U = p_sys->p_pic->p+U_PLANE;
1231     plane_t *p_V = p_sys->p_pic->p+V_PLANE;
1232     plane_t *p_A = p_sys->p_pic->p+A_PLANE;
1233
1234     int i_pitch = p_Y->i_pitch;
1235     int i_lines = p_Y->i_lines;
1236
1237     fprintf( stderr, "copy_rect: (%d,%d)+(%d,%d) -> (%d,%d)\n", i_x, i_y, i_w, i_h, i_sx, i_sy );
1238
1239     if( i_x + i_w > i_pitch || i_sx + i_w > i_pitch )
1240         return false;
1241     if( i_y + i_h > i_lines || i_sy + i_h > i_lines)
1242         return false;
1243
1244     if( i_w <= 0 || i_h <= 0 )
1245         return true;
1246
1247     uint8_t *pb_buffer = calloc( i_w * i_h, 4 );
1248     if( !pb_buffer )
1249         return false;
1250
1251     for( int i_line = 0; i_line < i_h; i_line++ )
1252     {
1253         for( int i_column = 0; i_column < i_w; i_column++ )
1254         {
1255             const int i_src_offset = ( i_sy + i_line ) * i_pitch + i_sx + i_column;
1256             const int i_tmp_offset = (    0 + i_line ) *     i_w +    0 + i_column;
1257
1258             pb_buffer[4*i_tmp_offset + 0] = p_Y->p_pixels[i_src_offset];
1259             pb_buffer[4*i_tmp_offset + 1] = p_U->p_pixels[i_src_offset];
1260             pb_buffer[4*i_tmp_offset + 2] = p_V->p_pixels[i_src_offset];
1261             pb_buffer[4*i_tmp_offset + 3] = p_A->p_pixels[i_src_offset];
1262         }
1263     }
1264
1265     for( int i_line = 0; i_line < i_h; i_line++ )
1266     {
1267         for( int i_column = 0; i_column < i_w; i_column++ )
1268         {
1269             const int i_tmp_offset = (   0 + i_line ) *     i_w +   0 + i_column;
1270             const int i_dst_offset = ( i_y + i_line ) * i_pitch + i_x + i_column;
1271
1272             p_Y->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 0];
1273             p_U->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 1];
1274             p_V->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 2];
1275             p_A->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 3];
1276         }
1277     }
1278     free( pb_buffer );
1279     return true;
1280
1281 }
1282
1283 static inline bool raw_line(  filter_sys_t* p_sys,
1284                               uint16_t i_x, uint16_t i_y,
1285                               uint16_t i_w )
1286 {
1287     plane_t *p_outY = p_sys->p_pic->p+Y_PLANE;
1288     plane_t *p_outU = p_sys->p_pic->p+U_PLANE;
1289     plane_t *p_outV = p_sys->p_pic->p+V_PLANE;
1290     plane_t *p_outA = p_sys->p_pic->p+A_PLANE;
1291     int i_pitch = p_outY->i_pitch;
1292     int i_lines = p_outY->i_lines;
1293     if ( i_x + i_w > i_pitch)
1294         return false;
1295     if ( i_y > i_lines)
1296         return false;
1297
1298     int i_line_offset = i_y * i_pitch + i_x;
1299
1300     for( int i_column = 0; i_column < i_w; i_column++ )
1301     {
1302         int i_offset = i_line_offset + i_column;
1303         uint8_t i_color = p_sys->read_buffer[i_column];
1304         p_outY->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][0];
1305         p_outU->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][1];
1306         p_outV->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][2];
1307         p_outA->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][3];
1308     }
1309
1310     return true;
1311 }
1312
1313
1314 /*****************************************************************************
1315  * MouseEvent: callback for mouse events
1316  *****************************************************************************/
1317 static int MouseEvent( filter_t *p_filter,
1318                        const vlc_mouse_t *p_old,
1319                        const vlc_mouse_t *p_new,
1320                        const video_format_t *p_fmt )
1321 {
1322     filter_sys_t *p_sys = p_filter->p_sys;
1323     VLC_UNUSED(p_old);
1324
1325     if( !p_sys->b_vnc_mouse_events )
1326         return VLC_SUCCESS;
1327
1328     int i_v = p_new->i_pressed;
1329     int i_x = p_new->i_x;
1330     int i_y = p_new->i_y;
1331
1332     vlc_mutex_lock( &p_sys->lock );
1333
1334     const int v_h = p_fmt->i_visible_height;
1335     const int v_w = p_sys->i_vnc_width * v_h / p_sys->i_vnc_height;
1336     const int v_x = (p_fmt->i_visible_width-v_w)/2;
1337
1338     i_x -= v_x;
1339
1340     if( i_y < 0 || i_x < 0 || i_y >= v_h || i_x >= v_w )
1341     {
1342         vlc_mutex_unlock( &p_sys->lock );
1343         msg_Dbg( p_filter, "invalid mouse event? x=%d y=%d btn=%x", i_x, i_y, i_v );
1344         return VLC_SUCCESS;
1345     }
1346     if( !p_sys->b_connection_active )
1347     {
1348         vlc_mutex_unlock( &p_sys->lock );
1349         return VLC_SUCCESS;
1350     }
1351
1352 #ifdef VNC_DEBUG
1353     msg_Dbg( p_filter, "mouse event x=%d y=%d btn=%x", i_x, i_y, i_v );
1354 #endif
1355
1356     /* */
1357     i_x = i_x * p_sys->i_vnc_width / v_w;
1358     i_y = i_y * p_sys->i_vnc_height / v_h;
1359
1360     /* buttonMask bits 0-7 are buttons 1-8, 0=up, 1=down */
1361     rfbPointerEventMsg ev;
1362     ev.type = rfbPointerEvent;
1363     ev.buttonMask = i_v;
1364     ev.x = htons(i_x);
1365     ev.y = htons(i_y);
1366
1367     write_exact( p_filter, p_sys->i_socket,
1368                  (char*)&ev, sz_rfbPointerEventMsg);
1369
1370     vlc_mutex_unlock( &p_sys->lock );
1371
1372     return VLC_EGENERIC;
1373 }
1374
1375 /*****************************************************************************
1376  * KeyEvent: callback for keyboard events
1377  *****************************************************************************/
1378 static int KeyEvent( vlc_object_t *p_this, char const *psz_var,
1379                      vlc_value_t oldval, vlc_value_t newval, void *p_data )
1380 {
1381     VLC_UNUSED(psz_var); VLC_UNUSED(oldval);
1382
1383     filter_t *p_filter = (filter_t *)p_data;
1384     filter_sys_t *p_sys = p_filter->p_sys;
1385
1386     if( !p_sys->b_vnc_key_events )
1387         return VLC_SUCCESS;
1388
1389     msg_Dbg( p_this, "key pressed (%d) ", newval.i_int );
1390
1391     if ( !newval.i_int )
1392     {
1393         msg_Err( p_this, "Received invalid key event %d", newval.i_int );
1394         return VLC_EGENERIC;
1395     }
1396
1397     vlc_mutex_lock( &p_sys->lock );
1398     if( !p_sys->b_connection_active )
1399     {
1400         vlc_mutex_unlock( &p_sys->lock );
1401         return VLC_SUCCESS;
1402     }
1403
1404     uint32_t i_key32 = newval.i_int;
1405     i_key32 = htonl(i_key32);
1406     rfbKeyEventMsg ev;
1407
1408     ev.type = rfbKeyEvent;
1409     ev.down = 1;
1410     ev.pad = 0;
1411
1412     /* first key-down for modifier-keys */
1413     if (newval.i_int & KEY_MODIFIER_CTRL)
1414     {
1415         ev.key = 0xffe3;
1416         write_exact( p_filter, p_sys->i_socket,
1417                      (char*)&ev, sz_rfbKeyEventMsg);
1418     }
1419     if (newval.i_int & KEY_MODIFIER_SHIFT)
1420     {
1421         ev.key = 0xffe1;
1422         write_exact( p_filter, p_sys->i_socket,
1423                      (char*)&ev, sz_rfbKeyEventMsg);
1424     }
1425     if (newval.i_int & KEY_MODIFIER_ALT)
1426     {
1427         ev.key = 0xffe9;
1428         write_exact( p_filter, p_sys->i_socket,
1429                      (char*)&ev, sz_rfbKeyEventMsg);
1430     }
1431
1432     /* then key-down for the pressed key */
1433     ev.key = i_key32;
1434     write_exact( p_filter, p_sys->i_socket,
1435                  (char*)&ev, sz_rfbKeyEventMsg);
1436
1437     ev.down = 0;
1438
1439     /* then key-up for the pressed key */
1440     write_exact( p_filter, p_sys->i_socket,
1441                  (char*)&ev, sz_rfbKeyEventMsg);
1442
1443     /* last key-down for modifier-keys */
1444     if (newval.i_int & KEY_MODIFIER_CTRL)
1445     {
1446         ev.key = 0xffe3;
1447         write_exact( p_filter, p_sys->i_socket,
1448                      (char*)&ev, sz_rfbKeyEventMsg);
1449     }
1450     if (newval.i_int & KEY_MODIFIER_SHIFT)
1451     {
1452         ev.key = 0xffe1;
1453         write_exact( p_filter, p_sys->i_socket,
1454                      (char*)&ev, sz_rfbKeyEventMsg);
1455     }
1456     if (newval.i_int & KEY_MODIFIER_ALT)
1457     {
1458        ev.key = 0xffe9;
1459        write_exact( p_filter, p_sys->i_socket,
1460                     (char*)&ev, sz_rfbKeyEventMsg);
1461     }
1462     vlc_mutex_unlock( &p_sys->lock );
1463
1464     return VLC_SUCCESS;
1465 }
1466
1467 static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd )
1468 {
1469     unsigned char key[8];
1470     unsigned int i;
1471
1472     for (i = 0; i < 8; i++)
1473         key[i] = i < strlen( passwd ) ? passwd[i] : '\0';
1474
1475     gcry_cipher_hd_t ctx;
1476     gcry_cipher_open( &ctx, GCRY_CIPHER_DES, GCRY_CIPHER_MODE_ECB,0);
1477
1478     /* reverse bits of the key */
1479     for( i = 0 ; i < 8 ; i ++ )
1480         key[i] =
1481             (key[i] >> 7) +
1482             (((key[i] >> 6) & 0x01 ) << 1 ) +
1483             (((key[i] >> 5) & 0x01 ) << 2 ) +
1484             (((key[i] >> 4) & 0x01 ) << 3 ) +
1485             (((key[i] >> 3) & 0x01 ) << 4 ) +
1486             (((key[i] >> 2) & 0x01 ) << 5 ) +
1487             (((key[i] >> 1) & 0x01 ) << 6 ) +
1488             ((key[i] & 0x01) << 7 );
1489
1490     gcry_cipher_setkey( ctx, key, 8 );
1491     gcry_cipher_encrypt( ctx, bytes, CHALLENGESIZE, bytes, CHALLENGESIZE );
1492     gcry_cipher_close( ctx );
1493 }
1494