avfilter/vf_uspp: fix integer overflow in intermediate

[ffmpeg] / libavfilter / vf_uspp.c

diff --git a/libavfilter/vf_uspp.c b/libavfilter/vf_uspp.c
index fd07eb9a563b9374f700a2024beb6a07ecb62299..d0ce63e02f78ddae05c3ab0d9deb1d18b7fc9284 100644 (file)
--- a/libavfilter/vf_uspp.c
+++ b/libavfilter/vf_uspp.c
@@ -45,7 +45,7 @@ typedef struct {
     int qscale_type;
     int temp_stride[3];
     uint8_t *src[3];
-    int16_t *temp[3];
+    uint16_t *temp[3];
     int outbuf_size;
     uint8_t *outbuf;
     AVCodecContext *avctx_enc[BLOCK*BLOCK];
@@ -152,7 +152,7 @@ static const uint8_t offset[511][2] = {
     { 3, 4}, {11, 4}, { 3,12}, {11,12}, { 7, 0}, {15, 0}, { 7, 8}, {15, 8},
 };
 
-static void store_slice_c(uint8_t *dst, const int16_t *src,
+static void store_slice_c(uint8_t *dst, const uint16_t *src,
                           int dst_stride, int src_stride,
                           int width, int height, int log2_scale)
 {
@@ -227,8 +227,16 @@ static void filter(USPPContext *p, uint8_t *dst[3], uint8_t *src[3],
 
     if (p->qp)
         p->frame->quality = p->qp * FF_QP2LAMBDA;
-    else
-        p->frame->quality = norm_qscale(qp_store[0], p->qscale_type) * FF_QP2LAMBDA;
+    else {
+        int qpsum=0;
+        int qpcount = (height>>4) * (height>>4);
+
+        for (y = 0; y < (height>>4); y++) {
+            for (x = 0; x < (width>>4); x++)
+                qpsum += qp_store[x + y * qp_stride];
+        }
+        p->frame->quality = norm_qscale((qpsum + qpcount/2) / qpcount, p->qscale_type) * FF_QP2LAMBDA;
+    }
 //    init per MB qscale stuff FIXME
     p->frame->height = height;
     p->frame->width  = width;
@@ -280,11 +288,11 @@ static void filter(USPPContext *p, uint8_t *dst[3], uint8_t *src[3],
 static int query_formats(AVFilterContext *ctx)
 {
     static const enum PixelFormat pix_fmts[] = {
-        AV_PIX_FMT_YUV444P,  AV_PIX_FMT_YUV422P,
-        AV_PIX_FMT_YUV420P,  AV_PIX_FMT_YUV411P,
-        AV_PIX_FMT_YUV410P,  AV_PIX_FMT_YUV440P,
-        AV_PIX_FMT_YUVJ444P, AV_PIX_FMT_YUVJ422P,
-        AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ440P,
+        AV_PIX_FMT_YUV444P,
+        AV_PIX_FMT_YUV420P,
+        AV_PIX_FMT_YUV410P,
+        AV_PIX_FMT_YUVJ444P,
+        AV_PIX_FMT_YUVJ420P,
         AV_PIX_FMT_NONE
     };
     ff_set_common_formats(ctx, ff_make_format_list(pix_fmts));
@@ -379,7 +387,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
                 w = FF_CEIL_RSHIFT(inlink->w, 4);
                 h = 1;
             } else {
-                w = FF_CEIL_RSHIFT(qp_stride, 4);
+                w = qp_stride;
                 h = FF_CEIL_RSHIFT(inlink->h, 4);
             }