2 * VMware Screen Codec (VMnc) decoder
3 * Copyright (c) 2006 Konstantin Shishkov
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 * VMware Screen Codec (VMnc) decoder
24 * As Alex Beregszaszi discovered, this is effectively RFB data dump
34 MAGIC_WMVd = 0x574D5664,
44 HT_RAW = 1, // tile is raw
45 HT_BKG = 2, // background color is present
46 HT_FG = 4, // foreground color is present
47 HT_SUB = 8, // subrects are present
48 HT_CLR = 16 // each subrect has own color
54 typedef struct VmncContext {
55 AVCodecContext *avctx;
68 uint8_t* curbits, *curmask;
72 /* read pixel value from stream */
73 static always_inline int vmnc_get_pixel(uint8_t* buf, int bpp, int be) {
74 switch(bpp * 2 + be) {
77 case 4: return LE_16(buf);
78 case 5: return BE_16(buf);
79 case 8: return LE_32(buf);
80 case 9: return BE_32(buf);
85 static void load_cursor(VmncContext *c, uint8_t *src)
88 const int bpp = c->bpp2;
89 uint8_t *dst8 = c->curbits;
90 uint16_t *dst16 = (uint16_t*)c->curbits;
91 uint32_t *dst32 = (uint32_t*)c->curbits;
93 for(j = 0; j < c->cur_h; j++) {
94 for(i = 0; i < c->cur_w; i++) {
95 p = vmnc_get_pixel(src, bpp, c->bigendian);
97 if(bpp == 1) *dst8++ = p;
98 if(bpp == 2) *dst16++ = p;
99 if(bpp == 4) *dst32++ = p;
103 dst16 = (uint16_t*)c->curmask;
104 dst32 = (uint32_t*)c->curmask;
105 for(j = 0; j < c->cur_h; j++) {
106 for(i = 0; i < c->cur_w; i++) {
107 p = vmnc_get_pixel(src, bpp, c->bigendian);
109 if(bpp == 1) *dst8++ = p;
110 if(bpp == 2) *dst16++ = p;
111 if(bpp == 4) *dst32++ = p;
116 static void put_cursor(uint8_t *dst, int stride, VmncContext *c, int dx, int dy)
121 if(c->width < c->cur_x + c->cur_w) w = c->width - c->cur_x;
123 if(c->height < c->cur_y + c->cur_h) h = c->height - c->cur_y;
135 if((w < 1) || (h < 1)) return;
136 dst += x * c->bpp2 + y * stride;
139 uint8_t* cd = c->curbits, *msk = c->curmask;
140 for(j = 0; j < h; j++) {
141 for(i = 0; i < w; i++)
142 dst[i] = (dst[i] & cd[i]) ^ msk[i];
147 } else if(c->bpp2 == 2) {
148 uint16_t* cd = (uint16_t*)c->curbits, *msk = (uint16_t*)c->curmask;
150 for(j = 0; j < h; j++) {
151 dst2 = (uint16_t*)dst;
152 for(i = 0; i < w; i++)
153 dst2[i] = (dst2[i] & cd[i]) ^ msk[i];
158 } else if(c->bpp2 == 4) {
159 uint32_t* cd = (uint32_t*)c->curbits, *msk = (uint32_t*)c->curmask;
161 for(j = 0; j < h; j++) {
162 dst2 = (uint32_t*)dst;
163 for(i = 0; i < w; i++)
164 dst2[i] = (dst2[i] & cd[i]) ^ msk[i];
172 /* fill rectangle with given colour */
173 static always_inline void paint_rect(uint8_t *dst, int dx, int dy, int w, int h, int color, int bpp, int stride)
176 dst += dx * bpp + dy * stride;
178 for(j = 0; j < h; j++) {
179 memset(dst, color, w);
184 for(j = 0; j < h; j++) {
185 dst2 = (uint16_t*)dst;
186 for(i = 0; i < w; i++) {
193 for(j = 0; j < h; j++) {
194 dst2 = (uint32_t*)dst;
195 for(i = 0; i < w; i++) {
203 static always_inline void paint_raw(uint8_t *dst, int w, int h, uint8_t* src, int bpp, int be, int stride)
206 for(j = 0; j < h; j++) {
207 for(i = 0; i < w; i++) {
208 p = vmnc_get_pixel(src, bpp, be);
215 ((uint16_t*)dst)[i] = p;
218 ((uint32_t*)dst)[i] = p;
226 static int decode_hextile(VmncContext *c, uint8_t* dst, uint8_t* src, int ssize, int w, int h, int stride)
229 int bg = 0, fg = 0, rects, color, flags, xy, wh;
230 const int bpp = c->bpp2;
232 int bw = 16, bh = 16;
235 for(j = 0; j < h; j += 16) {
238 if(j + 16 > h) bh = h - j;
239 for(i = 0; i < w; i += 16, dst2 += 16 * bpp) {
240 if(src - ssrc >= ssize) {
241 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");
244 if(i + 16 > w) bw = w - i;
247 if(src - ssrc > ssize - bw * bh * bpp) {
248 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");
251 paint_raw(dst2, bw, bh, src, bpp, c->bigendian, stride);
252 src += bw * bh * bpp;
255 bg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp;
258 fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp;
263 color = !!(flags & HT_CLR);
265 paint_rect(dst2, 0, 0, bw, bh, bg, bpp, stride);
267 if(src - ssrc > ssize - rects * (color * bpp + 2)) {
268 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");
271 for(k = 0; k < rects; k++) {
273 fg = vmnc_get_pixel(src, bpp, c->bigendian); src += bpp;
277 paint_rect(dst2, xy >> 4, xy & 0xF, (wh>>4)+1, (wh & 0xF)+1, fg, bpp, stride);
286 static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, uint8_t *buf, int buf_size)
288 VmncContext * const c = (VmncContext *)avctx->priv_data;
291 int dx, dy, w, h, depth, enc, chunks, res, size_left;
293 c->pic.reference = 1;
294 c->pic.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE;
295 if(avctx->reget_buffer(avctx, &c->pic) < 0){
296 av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
300 c->pic.key_frame = 0;
301 c->pic.pict_type = FF_P_TYPE;
303 //restore screen after cursor
307 if(c->width < c->cur_x + w) w = c->width - c->cur_x;
309 if(c->height < c->cur_y + h) h = c->height - c->cur_y;
320 if((w > 0) && (h > 0)) {
321 outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0];
322 for(i = 0; i < h; i++) {
323 memcpy(outptr, c->screendta + i * c->cur_w * c->bpp2, w * c->bpp2);
324 outptr += c->pic.linesize[0];
329 chunks = BE_16(src); src += 2;
331 dx = BE_16(src); src += 2;
332 dy = BE_16(src); src += 2;
333 w = BE_16(src); src += 2;
334 h = BE_16(src); src += 2;
335 enc = BE_32(src); src += 4;
336 outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0];
337 size_left = buf_size - (src - buf);
339 case MAGIC_WMVd: // cursor
340 if(size_left < 2 + w * h * c->bpp2 * 2) {
341 av_log(avctx, AV_LOG_ERROR, "Premature end of data! (need %i got %i)\n", 2 + w * h * c->bpp2 * 2, size_left);
349 if((c->cur_hx > c->cur_w) || (c->cur_hy > c->cur_h)) {
350 av_log(avctx, AV_LOG_ERROR, "Cursor hot spot is not in image: %ix%i of %ix%i cursor size\n", c->cur_hx, c->cur_hy, c->cur_w, c->cur_h);
351 c->cur_hx = c->cur_hy = 0;
353 c->curbits = av_realloc(c->curbits, c->cur_w * c->cur_h * c->bpp2);
354 c->curmask = av_realloc(c->curmask, c->cur_w * c->cur_h * c->bpp2);
355 c->screendta = av_realloc(c->screendta, c->cur_w * c->cur_h * c->bpp2);
357 src += w * h * c->bpp2 * 2;
359 case MAGIC_WMVe: // unknown
362 case MAGIC_WMVf: // update cursor position
363 c->cur_x = dx - c->cur_hx;
364 c->cur_y = dy - c->cur_hy;
366 case MAGIC_WMVg: // unknown
369 case MAGIC_WMVh: // unknown
372 case MAGIC_WMVi: // ServerInitialization struct
373 c->pic.key_frame = 1;
374 c->pic.pict_type = FF_I_TYPE;
376 if(depth != c->bpp) {
377 av_log(avctx, AV_LOG_INFO, "Depth mismatch. Container %i bpp, Frame data: %i bpp\n", c->bpp, depth);
380 c->bigendian = *src++;
381 if(c->bigendian & (~1)) {
382 av_log(avctx, AV_LOG_INFO, "Invalid header: bigendian flag = %i\n", c->bigendian);
385 //skip the rest of pixel format data
388 case MAGIC_WMVj: // unknown
391 case 0x00000000: // raw rectangle data
392 if((dx + w > c->width) || (dy + h > c->height)) {
393 av_log(avctx, AV_LOG_ERROR, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w, h, dx, dy, c->width, c->height);
396 if(size_left < w * h * c->bpp2) {
397 av_log(avctx, AV_LOG_ERROR, "Premature end of data! (need %i got %i)\n", w * h * c->bpp2, size_left);
400 paint_raw(outptr, w, h, src, c->bpp2, c->bigendian, c->pic.linesize[0]);
401 src += w * h * c->bpp2;
403 case 0x00000005: // HexTile encoded rectangle
404 if((dx + w > c->width) || (dy + h > c->height)) {
405 av_log(avctx, AV_LOG_ERROR, "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", w, h, dx, dy, c->width, c->height);
408 res = decode_hextile(c, outptr, src, size_left, w, h, c->pic.linesize[0]);
414 av_log(avctx, AV_LOG_ERROR, "Unsupported block type 0x%08X\n", enc);
415 chunks = 0; // leave chunks decoding loop
420 //save screen data before painting cursor
422 if(c->width < c->cur_x + w) w = c->width - c->cur_x;
424 if(c->height < c->cur_y + h) h = c->height - c->cur_y;
435 if((w > 0) && (h > 0)) {
436 outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0];
437 for(i = 0; i < h; i++) {
438 memcpy(c->screendta + i * c->cur_w * c->bpp2, outptr, w * c->bpp2);
439 outptr += c->pic.linesize[0];
441 outptr = c->pic.data[0];
442 put_cursor(outptr, c->pic.linesize[0], c, c->cur_x, c->cur_y);
445 *data_size = sizeof(AVFrame);
446 *(AVFrame*)data = c->pic;
448 /* always report that the buffer was completely consumed */
459 static int decode_init(AVCodecContext *avctx)
461 VmncContext * const c = (VmncContext *)avctx->priv_data;
464 avctx->has_b_frames = 0;
466 c->pic.data[0] = NULL;
467 c->width = avctx->width;
468 c->height = avctx->height;
470 if (avcodec_check_dimensions(avctx, avctx->width, avctx->height) < 0) {
473 c->bpp = avctx->bits_per_sample;
478 avctx->pix_fmt = PIX_FMT_PAL8;
481 avctx->pix_fmt = PIX_FMT_RGB555;
484 avctx->pix_fmt = PIX_FMT_RGB32;
487 av_log(avctx, AV_LOG_ERROR, "Unsupported bitdepth %i\n", c->bpp);
497 * Uninit VMnc decoder
500 static int decode_end(AVCodecContext *avctx)
502 VmncContext * const c = (VmncContext *)avctx->priv_data;
505 avctx->release_buffer(avctx, &c->pic);
509 av_free(c->screendta);
513 AVCodec vmnc_decoder = {