qsv: Load the hw hevc plugin by default on Linux

[ffmpeg] / libavcodec / vmnc.c

diff --git a/libavcodec/vmnc.c b/libavcodec/vmnc.c
index 1073cb202f557b22705d96211104563d1861b87f..7a01f1e2e63b88a48b3c636367c23b4286a6f0bb 100644 (file)
--- a/libavcodec/vmnc.c
+++ b/libavcodec/vmnc.c
@@ -57,7 +57,7 @@ enum HexTile_Flags {
  */
 typedef struct VmncContext {
     AVCodecContext *avctx;
-    AVFrame pic;
+    AVFrame *pic;
 
     int bpp;
     int bpp2;
@@ -287,12 +287,24 @@ static int decode_hextile(VmncContext *c, uint8_t* dst, GetByteContext *gb,
                     return AVERROR_INVALIDDATA;
                 }
                 for (k = 0; k < rects; k++) {
+                    int rect_x, rect_y, rect_w, rect_h;
                     if (color)
                         fg = vmnc_get_pixel(gb, bpp, c->bigendian);
                     xy = bytestream2_get_byte(gb);
                     wh = bytestream2_get_byte(gb);
-                    paint_rect(dst2, xy >> 4, xy & 0xF,
-                               (wh>>4)+1, (wh & 0xF)+1, fg, bpp, stride);
+
+                    rect_x = xy >> 4;
+                    rect_y = xy & 0xF;
+                    rect_w = (wh >> 4) + 1;
+                    rect_h = (wh & 0xF) + 1;
+
+                    if (rect_x + rect_w > bw || rect_y + rect_h > bh) {
+                        av_log(c->avctx, AV_LOG_ERROR, "Invalid subrect\n");
+                        return AVERROR_INVALIDDATA;
+                    }
+
+                    paint_rect(dst2, rect_x, rect_y,
+                               rect_w, rect_h, fg, bpp, stride);
                 }
             }
         }
@@ -319,15 +331,15 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
     uint8_t *outptr;
     int dx, dy, w, h, depth, enc, chunks, res, size_left, ret;
 
-    if ((ret = ff_reget_buffer(avctx, &c->pic)) < 0) {
+    if ((ret = ff_reget_buffer(avctx, c->pic)) < 0) {
         av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
         return ret;
     }
 
     bytestream2_init(gb, buf, buf_size);
 
-    c->pic.key_frame = 0;
-    c->pic.pict_type = AV_PICTURE_TYPE_P;
+    c->pic->key_frame = 0;
+    c->pic->pict_type = AV_PICTURE_TYPE_P;
 
     // restore screen after cursor
     if (c->screendta) {
@@ -349,11 +361,11 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
             dy = 0;
         }
         if ((w > 0) && (h > 0)) {
-            outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0];
+            outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0];
             for (i = 0; i < h; i++) {
                 memcpy(outptr, c->screendta + i * c->cur_w * c->bpp2,
                        w * c->bpp2);
-                outptr += c->pic.linesize[0];
+                outptr += c->pic->linesize[0];
             }
         }
     }
@@ -365,7 +377,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
         w   = bytestream2_get_be16(gb);
         h   = bytestream2_get_be16(gb);
         enc = bytestream2_get_be32(gb);
-        outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0];
+        outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0];
         size_left = bytestream2_get_bytes_left(gb);
         switch (enc) {
         case MAGIC_WMVd: // cursor
@@ -415,14 +427,22 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
             bytestream2_skip(gb, 4);
             break;
         case MAGIC_WMVi: // ServerInitialization struct
-            c->pic.key_frame = 1;
-            c->pic.pict_type = AV_PICTURE_TYPE_I;
+            c->pic->key_frame = 1;
+            c->pic->pict_type = AV_PICTURE_TYPE_I;
             depth = bytestream2_get_byte(gb);
             if (depth != c->bpp) {
-                av_log(avctx, AV_LOG_INFO,
-                       "Depth mismatch. Container %i bpp, "
-                       "Frame data: %i bpp\n",
-                       c->bpp, depth);
+                av_log(avctx, AV_LOG_WARNING, "Depth mismatch. "
+                       "Container %i bpp / Codec %i bpp\n", c->bpp, depth);
+
+                if (depth != 8 && depth != 16 && depth != 32) {
+                    av_log(avctx, AV_LOG_ERROR,
+                           "Unsupported codec bitdepth %i\n", depth);
+                    return AVERROR_INVALIDDATA;
+                }
+
+                /* reset values */
+                c->bpp  = depth;
+                c->bpp2 = c->bpp / 8;
             }
             bytestream2_skip(gb, 1);
             c->bigendian = bytestream2_get_byte(gb);
@@ -451,7 +471,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
                 return AVERROR_INVALIDDATA;
             }
             paint_raw(outptr, w, h, gb, c->bpp2, c->bigendian,
-                      c->pic.linesize[0]);
+                      c->pic->linesize[0]);
             break;
         case 0x00000005: // HexTile encoded rectangle
             if ((dx + w > c->width) || (dy + h > c->height)) {
@@ -460,7 +480,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
                        w, h, dx, dy, c->width, c->height);
                 return AVERROR_INVALIDDATA;
             }
-            res = decode_hextile(c, outptr, gb, w, h, c->pic.linesize[0]);
+            res = decode_hextile(c, outptr, gb, w, h, c->pic->linesize[0]);
             if (res < 0)
                 return res;
             break;
@@ -489,18 +509,18 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
             dy = 0;
         }
         if ((w > 0) && (h > 0)) {
-            outptr = c->pic.data[0] + dx * c->bpp2 + dy * c->pic.linesize[0];
+            outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0];
             for (i = 0; i < h; i++) {
                 memcpy(c->screendta + i * c->cur_w * c->bpp2, outptr,
                        w * c->bpp2);
-                outptr += c->pic.linesize[0];
+                outptr += c->pic->linesize[0];
             }
-            outptr = c->pic.data[0];
-            put_cursor(outptr, c->pic.linesize[0], c, c->cur_x, c->cur_y);
+            outptr = c->pic->data[0];
+            put_cursor(outptr, c->pic->linesize[0], c, c->cur_x, c->cur_y);
         }
     }
     *got_frame = 1;
-    if ((ret = av_frame_ref(data, &c->pic)) < 0)
+    if ((ret = av_frame_ref(data, c->pic)) < 0)
         return ret;
 
     /* always report that the buffer was completely consumed */
@@ -524,6 +544,9 @@ static av_cold int decode_init(AVCodecContext *avctx)
     case 16:
         avctx->pix_fmt = AV_PIX_FMT_RGB555;
         break;
+    case 24:
+        /* 24 bits is not technically supported, but some clients might
+         * mistakenly set it -- delay the actual check until decode_frame() */
     case 32:
         avctx->pix_fmt = AV_PIX_FMT_RGB32;
         break;
@@ -532,7 +555,9 @@ static av_cold int decode_init(AVCodecContext *avctx)
         return AVERROR_INVALIDDATA;
     }
 
-    avcodec_get_frame_defaults(&c->pic);
+    c->pic = av_frame_alloc();
+    if (!c->pic)
+        return AVERROR(ENOMEM);
 
     return 0;
 }
@@ -541,7 +566,7 @@ static av_cold int decode_end(AVCodecContext *avctx)
 {
     VmncContext * const c = avctx->priv_data;
 
-    av_frame_unref(&c->pic);
+    av_frame_free(&c->pic);
 
     av_free(c->curbits);
     av_free(c->curmask);
@@ -558,5 +583,5 @@ AVCodec ff_vmnc_decoder = {
     .init           = decode_init,
     .close          = decode_end,
     .decode         = decode_frame,
-    .capabilities   = CODEC_CAP_DR1,
+    .capabilities   = AV_CODEC_CAP_DR1,
 };