git.sesse.net Git - rdpsrv/blob - Xserver/programs/Xserver/hw/vnc/hextile.c

   1 /*
   2  * hextile.c
   3  *
   4  * Routines to implement Hextile Encoding
   5  */
   6
   7 /*
   8  *  Copyright (C) 2002 RealVNC Ltd.
   9  *  Copyright (C) 1999 AT&T Laboratories Cambridge.  All Rights Reserved.
  10  *
  11  *  This is free software; you can redistribute it and/or modify
  12  *  it under the terms of the GNU General Public License as published by
  13  *  the Free Software Foundation; either version 2 of the License, or
  14  *  (at your option) any later version.
  15  *
  16  *  This software is distributed in the hope that it will be useful,
  17  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  *  GNU General Public License for more details.
  20  *
  21  *  You should have received a copy of the GNU General Public License
  22  *  along with this software; if not, write to the Free Software
  23  *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
  24  *  USA.
  25  */
  26
  27 #include <stdio.h>
  28 #include "rfb.h"
  29
  30 static Bool sendHextiles8(rfbClientPtr cl, int x, int y, int w, int h);
  31 static Bool sendHextiles16(rfbClientPtr cl, int x, int y, int w, int h);
  32 static Bool sendHextiles32(rfbClientPtr cl, int x, int y, int w, int h);
  33
  34
  35 /*
  36  * rfbSendRectEncodingHextile - send a rectangle using hextile encoding.
  37  */
  38
  39 Bool
  40 rfbSendRectEncodingHextile(cl, x, y, w, h)
  41     rfbClientPtr cl;
  42     int x, y, w, h;
  43 {
  44     rfbFramebufferUpdateRectHeader rect;
  45
  46     if (ublen + sz_rfbFramebufferUpdateRectHeader > UPDATE_BUF_SIZE) {
  47         if (!rfbSendUpdateBuf(cl))
  48             return FALSE;
  49     }
  50
  51     rect.r.x = Swap16IfLE(x);
  52     rect.r.y = Swap16IfLE(y);
  53     rect.r.w = Swap16IfLE(w);
  54     rect.r.h = Swap16IfLE(h);
  55     rect.encoding = Swap32IfLE(rfbEncodingHextile);
  56
  57     memcpy(&updateBuf[ublen], (char *)&rect,
  58            sz_rfbFramebufferUpdateRectHeader);
  59     ublen += sz_rfbFramebufferUpdateRectHeader;
  60
  61     cl->rfbRectanglesSent[rfbEncodingHextile]++;
  62     cl->rfbBytesSent[rfbEncodingHextile] += sz_rfbFramebufferUpdateRectHeader;
  63
  64     switch (cl->format.bitsPerPixel) {
  65     case 8:
  66         return sendHextiles8(cl, x, y, w, h);
  67     case 16:
  68         return sendHextiles16(cl, x, y, w, h);
  69     case 32:
  70         return sendHextiles32(cl, x, y, w, h);
  71     }
  72
  73     rfbLog("rfbSendRectEncodingHextile: bpp %d?\n", cl->format.bitsPerPixel);
  74     return FALSE;
  75 }
  76
  77
  78 #define PUT_PIXEL8(pix) (updateBuf[ublen++] = (pix))
  79
  80 #define PUT_PIXEL16(pix) (updateBuf[ublen++] = ((char*)&(pix))[0], \
  81                           updateBuf[ublen++] = ((char*)&(pix))[1])
  82
  83 #define PUT_PIXEL32(pix) (updateBuf[ublen++] = ((char*)&(pix))[0], \
  84                           updateBuf[ublen++] = ((char*)&(pix))[1], \
  85                           updateBuf[ublen++] = ((char*)&(pix))[2], \
  86                           updateBuf[ublen++] = ((char*)&(pix))[3])
  87
  88
  89 #define DEFINE_SEND_HEXTILES(bpp)                                             \
  90                                                                               \
  91                                                                               \
  92 static Bool subrectEncode##bpp(CARD##bpp *data, int w, int h, CARD##bpp bg,   \
  93                                CARD##bpp fg, Bool mono);                      \
  94 static void testColours##bpp(CARD##bpp *data, int size, Bool *mono,           \
  95                              Bool *solid, CARD##bpp *bg, CARD##bpp *fg);      \
  96                                                                               \
  97                                                                               \
  98 /*                                                                            \
  99  * rfbSendHextiles                                                            \
 100  */                                                                           \
 101                                                                               \
 102 static Bool                                                                   \
 103 sendHextiles##bpp(cl, rx, ry, rw, rh)                                         \
 104     rfbClientPtr cl;                                                          \
 105     int rx, ry, rw, rh;                                                       \
 106 {                                                                             \
 107     int x, y, w, h;                                                           \
 108     int startUblen;                                                           \
 109     char *fbptr;                                                              \
 110     CARD##bpp bg=0, fg=0, newBg, newFg;                                       \
 111     Bool mono, solid;                                                         \
 112     Bool validBg = FALSE;                                                     \
 113     Bool validFg = FALSE;                                                     \
 114     CARD##bpp clientPixelData[16*16*(bpp/8)];                                 \
 115                                                                               \
 116     for (y = ry; y < ry+rh; y += 16) {                                        \
 117         for (x = rx; x < rx+rw; x += 16) {                                    \
 118             w = h = 16;                                                       \
 119             if (rx+rw - x < 16)                                               \
 120                 w = rx+rw - x;                                                \
 121             if (ry+rh - y < 16)                                               \
 122                 h = ry+rh - y;                                                \
 123                                                                               \
 124             if ((ublen + 1 + (2 + 16 * 16) * (bpp/8)) > UPDATE_BUF_SIZE) {    \
 125                 if (!rfbSendUpdateBuf(cl))                                    \
 126                     return FALSE;                                             \
 127             }                                                                 \
 128                                                                               \
 129             fbptr = (rfbScreen.pfbMemory + (rfbScreen.paddedWidthInBytes * y) \
 130                      + (x * (rfbScreen.bitsPerPixel / 8)));                   \
 131                                                                               \
 132             (*cl->translateFn)(cl->translateLookupTable, &rfbServerFormat,    \
 133                                &cl->format, fbptr, (char *)clientPixelData,   \
 134                                rfbScreen.paddedWidthInBytes, w, h);           \
 135                                                                               \
 136             startUblen = ublen;                                               \
 137             updateBuf[startUblen] = 0;                                        \
 138             ublen++;                                                          \
 139                                                                               \
 140             testColours##bpp(clientPixelData, w * h,                          \
 141                              &mono, &solid, &newBg, &newFg);                  \
 142                                                                               \
 143             if (!validBg || (newBg != bg)) {                                  \
 144                 validBg = TRUE;                                               \
 145                 bg = newBg;                                                   \
 146                 updateBuf[startUblen] |= rfbHextileBackgroundSpecified;       \
 147                 PUT_PIXEL##bpp(bg);                                           \
 148             }                                                                 \
 149                                                                               \
 150             if (solid) {                                                      \
 151                 cl->rfbBytesSent[rfbEncodingHextile] += ublen - startUblen;   \
 152                 continue;                                                     \
 153             }                                                                 \
 154                                                                               \
 155             updateBuf[startUblen] |= rfbHextileAnySubrects;                   \
 156                                                                               \
 157             if (mono) {                                                       \
 158                 if (!validFg || (newFg != fg)) {                              \
 159                     validFg = TRUE;                                           \
 160                     fg = newFg;                                               \
 161                     updateBuf[startUblen] |= rfbHextileForegroundSpecified;   \
 162                     PUT_PIXEL##bpp(fg);                                       \
 163                 }                                                             \
 164             } else {                                                          \
 165                 validFg = FALSE;                                              \
 166                 updateBuf[startUblen] |= rfbHextileSubrectsColoured;          \
 167             }                                                                 \
 168                                                                               \
 169             if (!subrectEncode##bpp(clientPixelData, w, h, bg, fg, mono)) {   \
 170                 /* encoding was too large, use raw */                         \
 171                 validBg = FALSE;                                              \
 172                 validFg = FALSE;                                              \
 173                 ublen = startUblen;                                           \
 174                 updateBuf[ublen++] = rfbHextileRaw;                           \
 175                 (*cl->translateFn)(cl->translateLookupTable,                  \
 176                                    &rfbServerFormat, &cl->format, fbptr,      \
 177                                    (char *)clientPixelData,                   \
 178                                    rfbScreen.paddedWidthInBytes, w, h);       \
 179                                                                               \
 180                 memcpy(&updateBuf[ublen], (char *)clientPixelData,            \
 181                        w * h * (bpp/8));                                      \
 182                                                                               \
 183                 ublen += w * h * (bpp/8);                                     \
 184             }                                                                 \
 185                                                                               \
 186             cl->rfbBytesSent[rfbEncodingHextile] += ublen - startUblen;       \
 187         }                                                                     \
 188     }                                                                         \
 189                                                                               \
 190     return TRUE;                                                              \
 191 }                                                                             \
 192                                                                               \
 193                                                                               \
 194 static Bool                                                                   \
 195 subrectEncode##bpp(CARD##bpp *data, int w, int h, CARD##bpp bg,               \
 196                    CARD##bpp fg, Bool mono)                                   \
 197 {                                                                             \
 198     CARD##bpp cl;                                                             \
 199     int x,y;                                                                  \
 200     int i,j;                                                                  \
 201     int hx=0,hy,vx=0,vy;                                                      \
 202     int hyflag;                                                               \
 203     CARD##bpp *seg;                                                           \
 204     CARD##bpp *line;                                                          \
 205     int hw,hh,vw,vh;                                                          \
 206     int thex,they,thew,theh;                                                  \
 207     int numsubs = 0;                                                          \
 208     int newLen;                                                               \
 209     int nSubrectsUblen;                                                       \
 210                                                                               \
 211     nSubrectsUblen = ublen;                                                   \
 212     ublen++;                                                                  \
 213                                                                               \
 214     for (y=0; y<h; y++) {                                                     \
 215         line = data+(y*w);                                                    \
 216         for (x=0; x<w; x++) {                                                 \
 217             if (line[x] != bg) {                                              \
 218                 cl = line[x];                                                 \
 219                 hy = y-1;                                                     \
 220                 hyflag = 1;                                                   \
 221                 for (j=y; j<h; j++) {                                         \
 222                     seg = data+(j*w);                                         \
 223                     if (seg[x] != cl) {break;}                                \
 224                     i = x;                                                    \
 225                     while ((seg[i] == cl) && (i < w)) i += 1;                 \
 226                     i -= 1;                                                   \
 227                     if (j == y) vx = hx = i;                                  \
 228                     if (i < vx) vx = i;                                       \
 229                     if ((hyflag > 0) && (i >= hx)) {                          \
 230                         hy += 1;                                              \
 231                     } else {                                                  \
 232                         hyflag = 0;                                           \
 233                     }                                                         \
 234                 }                                                             \
 235                 vy = j-1;                                                     \
 236                                                                               \
 237                 /* We now have two possible subrects: (x,y,hx,hy) and         \
 238                  * (x,y,vx,vy).  We'll choose the bigger of the two.          \
 239                  */                                                           \
 240                 hw = hx-x+1;                                                  \
 241                 hh = hy-y+1;                                                  \
 242                 vw = vx-x+1;                                                  \
 243                 vh = vy-y+1;                                                  \
 244                                                                               \
 245                 thex = x;                                                     \
 246                 they = y;                                                     \
 247                                                                               \
 248                 if ((hw*hh) > (vw*vh)) {                                      \
 249                     thew = hw;                                                \
 250                     theh = hh;                                                \
 251                 } else {                                                      \
 252                     thew = vw;                                                \
 253                     theh = vh;                                                \
 254                 }                                                             \
 255                                                                               \
 256                 if (mono) {                                                   \
 257                     newLen = ublen - nSubrectsUblen + 2;                      \
 258                 } else {                                                      \
 259                     newLen = ublen - nSubrectsUblen + bpp/8 + 2;              \
 260                 }                                                             \
 261                                                                               \
 262                 if (newLen > (w * h * (bpp/8)))                               \
 263                     return FALSE;                                             \
 264                                                                               \
 265                 numsubs += 1;                                                 \
 266                                                                               \
 267                 if (!mono) PUT_PIXEL##bpp(cl);                                \
 268                                                                               \
 269                 updateBuf[ublen++] = rfbHextilePackXY(thex,they);             \
 270                 updateBuf[ublen++] = rfbHextilePackWH(thew,theh);             \
 271                                                                               \
 272                 /*                                                            \
 273                  * Now mark the subrect as done.                              \
 274                  */                                                           \
 275                 for (j=they; j < (they+theh); j++) {                          \
 276                     for (i=thex; i < (thex+thew); i++) {                      \
 277                         data[j*w+i] = bg;                                     \
 278                     }                                                         \
 279                 }                                                             \
 280             }                                                                 \
 281         }                                                                     \
 282     }                                                                         \
 283                                                                               \
 284     updateBuf[nSubrectsUblen] = numsubs;                                      \
 285                                                                               \
 286     return TRUE;                                                              \
 287 }                                                                             \
 288                                                                               \
 289                                                                               \
 290 /*                                                                            \
 291  * testColours() tests if there are one (solid), two (mono) or more           \
 292  * colours in a tile and gets a reasonable guess at the best background       \
 293  * pixel, and the foreground pixel for mono.                                  \
 294  */                                                                           \
 295                                                                               \
 296 static void                                                                   \
 297 testColours##bpp(data,size,mono,solid,bg,fg)                                  \
 298     CARD##bpp *data;                                                          \
 299     int size;                                                                 \
 300     Bool *mono;                                                               \
 301     Bool *solid;                                                              \
 302     CARD##bpp *bg;                                                            \
 303     CARD##bpp *fg;                                                            \
 304 {                                                                             \
 305     CARD##bpp colour1=0, colour2=0;                                           \
 306     int n1 = 0, n2 = 0;                                                       \
 307     *mono = TRUE;                                                             \
 308     *solid = TRUE;                                                            \
 309                                                                               \
 310     for (; size > 0; size--, data++) {                                        \
 311                                                                               \
 312         if (n1 == 0)                                                          \
 313             colour1 = *data;                                                  \
 314                                                                               \
 315         if (*data == colour1) {                                               \
 316             n1++;                                                             \
 317             continue;                                                         \
 318         }                                                                     \
 319                                                                               \
 320         if (n2 == 0) {                                                        \
 321             *solid = FALSE;                                                   \
 322             colour2 = *data;                                                  \
 323         }                                                                     \
 324                                                                               \
 325         if (*data == colour2) {                                               \
 326             n2++;                                                             \
 327             continue;                                                         \
 328         }                                                                     \
 329                                                                               \
 330         *mono = FALSE;                                                        \
 331         break;                                                                \
 332     }                                                                         \
 333                                                                               \
 334     if (n1 > n2) {                                                            \
 335         *bg = colour1;                                                        \
 336         *fg = colour2;                                                        \
 337     } else {                                                                  \
 338         *bg = colour2;                                                        \
 339         *fg = colour1;                                                        \
 340     }                                                                         \
 341 }
 342
 343 DEFINE_SEND_HEXTILES(8)
 344 DEFINE_SEND_HEXTILES(16)
 345 DEFINE_SEND_HEXTILES(32)