rdo: Fix potential CAVLC overflow issues

[x264] / encoder / me.c

diff --git a/encoder/me.c b/encoder/me.c
index 8238b96bb09145eb4f5ed85981c1a3bacb1a4967..d34644d6c9c31cf73c38e84cf4d5a5ce7d28eca4 100644 (file)
--- a/encoder/me.c
+++ b/encoder/me.c
@@ -1,7 +1,7 @@
 /*****************************************************************************
  * me.c: motion estimation
  *****************************************************************************
- * Copyright (C) 2003-2013 x264 project
+ * Copyright (C) 2003-2015 x264 project
  *
  * Authors: Loren Merritt <lorenm@u.washington.edu>
  *          Laurent Aimar <fenrir@via.ecp.fr>
@@ -191,7 +191,7 @@ void x264_me_search_ref( x264_t *h, x264_me_t *m, int16_t (*mvc)[2], int i_mvc,
     int omx, omy, pmx, pmy;
     pixel *p_fenc = m->p_fenc[0];
     pixel *p_fref_w = m->p_fref_w;
-    ALIGNED_ARRAY_16( pixel, pix,[16*16] );
+    ALIGNED_ARRAY_N( pixel, pix,[16*16] );
     ALIGNED_ARRAY_8( int16_t, mvc_temp,[16],[2] );
 
     ALIGNED_ARRAY_16( int, costs,[16] );
@@ -365,14 +365,14 @@ void x264_me_search_ref( x264_t *h, x264_me_t *m, int16_t (*mvc)[2], int i_mvc,
 
             /* hexagon */
             COST_MV_X3_DIR( -2,0, -1, 2,  1, 2, costs   );
-            COST_MV_X3_DIR(  2,0,  1,-2, -1,-2, costs+3 );
+            COST_MV_X3_DIR(  2,0,  1,-2, -1,-2, costs+4 ); /* +4 for 16-byte alignment */
             bcost <<= 3;
             COPY1_IF_LT( bcost, (costs[0]<<3)+2 );
             COPY1_IF_LT( bcost, (costs[1]<<3)+3 );
             COPY1_IF_LT( bcost, (costs[2]<<3)+4 );
-            COPY1_IF_LT( bcost, (costs[3]<<3)+5 );
-            COPY1_IF_LT( bcost, (costs[4]<<3)+6 );
-            COPY1_IF_LT( bcost, (costs[5]<<3)+7 );
+            COPY1_IF_LT( bcost, (costs[4]<<3)+5 );
+            COPY1_IF_LT( bcost, (costs[5]<<3)+6 );
+            COPY1_IF_LT( bcost, (costs[6]<<3)+7 );
 
             if( bcost&7 )
             {
@@ -671,7 +671,7 @@ void x264_me_search_ref( x264_t *h, x264_me_t *m, int16_t (*mvc)[2], int i_mvc,
                     for( i = 0; i < xn-2; i += 3 )
                     {
                         pixel *ref = p_fref_w+min_x+my*stride;
-                        int sads[3];
+                        ALIGNED_ARRAY_16( int, sads,[4] ); /* padded to [4] for asm */
                         h->pixf.sad_x3[i_pixel]( p_fenc, ref+xs[i], ref+xs[i+1], ref+xs[i+2], stride, sads );
                         for( int j = 0; j < 3; j++ )
                         {
@@ -1033,9 +1033,9 @@ static void ALWAYS_INLINE x264_me_refine_bidir( x264_t *h, x264_me_t *m0, x264_m
     const int i_pixel = m0->i_pixel;
     const int bw = x264_pixel_size[i_pixel].w;
     const int bh = x264_pixel_size[i_pixel].h;
-    ALIGNED_ARRAY_16( pixel, pixy_buf,[2],[9][16*16] );
-    ALIGNED_ARRAY_16( pixel, pixu_buf,[2],[9][16*16] );
-    ALIGNED_ARRAY_16( pixel, pixv_buf,[2],[9][16*16] );
+    ALIGNED_ARRAY_N( pixel, pixy_buf,[2],[9][16*16] );
+    ALIGNED_ARRAY_N( pixel, pixu_buf,[2],[9][16*16] );
+    ALIGNED_ARRAY_N( pixel, pixv_buf,[2],[9][16*16] );
     pixel *src[3][2][9];
     int chromapix = h->luma2chroma_pixel[i_pixel];
     int chroma_v_shift = CHROMA_V_SHIFT;