h264: fix invalid pointer arithmetic

[ffmpeg] / libavcodec / snow.c

diff --git a/libavcodec/snow.c b/libavcodec/snow.c
index 27c7e2518987e5cafe7722f411c194e7a4428aa9..d69f452e5d4c7c12150667dc3ac76db4c5cbb132 100644 (file)
--- a/libavcodec/snow.c
+++ b/libavcodec/snow.c
@@ -99,7 +99,7 @@ static void init_qexp(void){
     double v=128;
 
     for(i=0; i<QROOT; i++){
-        qexp[i]= lrintf(v);
+        ff_qexp[i]= lrintf(v);
         v *= pow(2, 1.0 / QROOT);
     }
 }
@@ -143,7 +143,7 @@ static void mc_block(Plane *p, uint8_t *dst, const uint8_t *src, int stride, int
 
     int x, y, b, r, l;
     int16_t tmpIt   [64*(32+HTAPS_MAX)];
-    uint8_t tmp2t[3][stride*(32+HTAPS_MAX)];
+    uint8_t tmp2t[3][64*(32+HTAPS_MAX)];
     int16_t *tmpI= tmpIt;
     uint8_t *tmp2= tmp2t[0];
     const uint8_t *hpel[11];
@@ -181,7 +181,7 @@ static void mc_block(Plane *p, uint8_t *dst, const uint8_t *src, int stride, int
                 tmp2[x]= am;
             }
             tmpI+= 64;
-            tmp2+= stride;
+            tmp2+= 64;
             src += stride;
         }
         src -= stride*y;
@@ -210,7 +210,7 @@ static void mc_block(Plane *p, uint8_t *dst, const uint8_t *src, int stride, int
                 tmp2[x]= am;
             }
             src += stride;
-            tmp2+= stride;
+            tmp2+= 64;
         }
         src -= stride*y;
     }
@@ -237,12 +237,12 @@ static void mc_block(Plane *p, uint8_t *dst, const uint8_t *src, int stride, int
                 tmp2[x]= am;
             }
             tmpI+= 64;
-            tmp2+= stride;
+            tmp2+= 64;
         }
     }
 
     hpel[ 0]= src;
-    hpel[ 1]= tmp2t[0] + stride*(HTAPS_MAX/2-1);
+    hpel[ 1]= tmp2t[0] + 64*(HTAPS_MAX/2-1);
     hpel[ 2]= src + 1;
 
     hpel[ 4]= tmp2t[1];
@@ -250,14 +250,21 @@ static void mc_block(Plane *p, uint8_t *dst, const uint8_t *src, int stride, int
     hpel[ 6]= tmp2t[1] + 1;
 
     hpel[ 8]= src + stride;
-    hpel[ 9]= hpel[1] + stride;
+    hpel[ 9]= hpel[1] + 64;
     hpel[10]= hpel[8] + 1;
 
+#define MC_STRIDE(x) (needs[x] ? 64 : stride)
+
     if(b==15){
-        const uint8_t *src1= hpel[dx/8 + dy/8*4  ];
-        const uint8_t *src2= hpel[dx/8 + dy/8*4+1];
-        const uint8_t *src3= hpel[dx/8 + dy/8*4+4];
-        const uint8_t *src4= hpel[dx/8 + dy/8*4+5];
+        int dxy = dx / 8 + dy / 8 * 4;
+        const uint8_t *src1 = hpel[dxy    ];
+        const uint8_t *src2 = hpel[dxy + 1];
+        const uint8_t *src3 = hpel[dxy + 4];
+        const uint8_t *src4 = hpel[dxy + 5];
+        int stride1 = MC_STRIDE(dxy);
+        int stride2 = MC_STRIDE(dxy + 1);
+        int stride3 = MC_STRIDE(dxy + 4);
+        int stride4 = MC_STRIDE(dxy + 5);
         dx&=7;
         dy&=7;
         for(y=0; y < b_h; y++){
@@ -265,23 +272,25 @@ static void mc_block(Plane *p, uint8_t *dst, const uint8_t *src, int stride, int
                 dst[x]= ((8-dx)*(8-dy)*src1[x] + dx*(8-dy)*src2[x]+
                          (8-dx)*   dy *src3[x] + dx*   dy *src4[x]+32)>>6;
             }
-            src1+=stride;
-            src2+=stride;
-            src3+=stride;
-            src4+=stride;
+            src1+=stride1;
+            src2+=stride2;
+            src3+=stride3;
+            src4+=stride4;
             dst +=stride;
         }
     }else{
         const uint8_t *src1= hpel[l];
         const uint8_t *src2= hpel[r];
+        int stride1 = MC_STRIDE(l);
+        int stride2 = MC_STRIDE(r);
         int a= weight[((dx&7) + (8*(dy&7)))];
         int b= 8-a;
         for(y=0; y < b_h; y++){
             for(x=0; x < b_w; x++){
                 dst[x]= (a*src1[x] + b*src2[x] + 4)>>3;
             }
-            src1+=stride;
-            src2+=stride;
+            src1+=stride1;
+            src2+=stride2;
             dst +=stride;
         }
     }
@@ -385,7 +394,8 @@ mca( 8, 8,8)
 av_cold int ff_snow_common_init(AVCodecContext *avctx){
     SnowContext *s = avctx->priv_data;
     int width, height;
-    int i, j;
+    int i, j, ret;
+    int emu_buf_size;
 
     s->avctx= avctx;
     s->max_ref_frames=1; //just make sure its not an invalid value in case of no initial keyframe
@@ -438,17 +448,27 @@ av_cold int ff_snow_common_init(AVCodecContext *avctx){
     width= s->avctx->width;
     height= s->avctx->height;
 
-    s->spatial_idwt_buffer= av_mallocz(width*height*sizeof(IDWTELEM));
-    s->spatial_dwt_buffer= av_mallocz(width*height*sizeof(DWTELEM)); //FIXME this does not belong here
+    FF_ALLOCZ_OR_GOTO(avctx, s->spatial_idwt_buffer, width * height * sizeof(IDWTELEM), fail);
+    FF_ALLOCZ_OR_GOTO(avctx, s->spatial_dwt_buffer,  width * height * sizeof(DWTELEM),  fail); //FIXME this does not belong here
+    FF_ALLOCZ_OR_GOTO(avctx, s->temp_dwt_buffer,     width * sizeof(DWTELEM),  fail);
+    FF_ALLOCZ_OR_GOTO(avctx, s->temp_idwt_buffer,    width * sizeof(IDWTELEM), fail);
+    FF_ALLOC_OR_GOTO(avctx,  s->run_buffer,          ((width + 1) >> 1) * ((height + 1) >> 1) * sizeof(*s->run_buffer), fail);
 
     for(i=0; i<MAX_REF_FRAMES; i++)
         for(j=0; j<MAX_REF_FRAMES; j++)
-            scale_mv_ref[i][j] = 256*(i+1)/(j+1);
+            ff_scale_mv_ref[i][j] = 256*(i+1)/(j+1);
 
-    s->avctx->get_buffer(s->avctx, &s->mconly_picture);
-    s->scratchbuf = av_malloc(s->mconly_picture.linesize[0]*7*MB_SIZE);
+    if ((ret = s->avctx->get_buffer(s->avctx, &s->mconly_picture)) < 0) {
+        av_log(s->avctx, AV_LOG_ERROR, "get_buffer() failed\n");
+        return ret;
+    }
+    FF_ALLOC_OR_GOTO(avctx, s->scratchbuf, s->mconly_picture.linesize[0]*7*MB_SIZE, fail);
+    emu_buf_size = s->mconly_picture.linesize[0] * (2 * MB_SIZE + HTAPS_MAX - 1);
+    FF_ALLOC_OR_GOTO(avctx, s->emu_edge_buffer, emu_buf_size, fail);
 
     return 0;
+fail:
+    return AVERROR(ENOMEM);
 }
 
 int ff_snow_common_init_after_header(AVCodecContext *avctx) {
@@ -618,7 +638,10 @@ av_cold void ff_snow_common_end(SnowContext *s)
     int plane_index, level, orientation, i;
 
     av_freep(&s->spatial_dwt_buffer);
+    av_freep(&s->temp_dwt_buffer);
     av_freep(&s->spatial_idwt_buffer);
+    av_freep(&s->temp_idwt_buffer);
+    av_freep(&s->run_buffer);
 
     s->m.me.temp= NULL;
     av_freep(&s->m.me.scratchpad);
@@ -628,6 +651,7 @@ av_cold void ff_snow_common_end(SnowContext *s)
 
     av_freep(&s->block);
     av_freep(&s->scratchbuf);
+    av_freep(&s->emu_edge_buffer);
 
     for(i=0; i<MAX_REF_FRAMES; i++){
         av_freep(&s->ref_mvs[i]);