fix a nondeterminism involving 8x8dct, rdo, and threads.

git-svn-id: svn://svn.videolan.org/x264/trunk@713 df754926-b1dd-0310-bc7b-ec298dee348c

diff --git a/common/common.h b/common/common.h
index 10164e06d40a14274842a664ba00ba4f1cfd9a5e..88606519c88d8a4d320ea504fa41d24fac0d7fa8 100644 (file)
--- a/common/common.h
+++ b/common/common.h
@@ -471,7 +471,6 @@ struct x264_t
 
             /* number of neighbors (top and left) that used 8x8 dct */
             int     i_neighbour_transform_size;
-            int     b_transform_8x8_allowed;
             int     i_neighbour_interlaced;
         } cache;
 

diff --git a/common/macroblock.c b/common/macroblock.c
index a8056de5ea3a96902cb3adc1202183595ed84dea..e0307e13df6724f2d98ff253357c148b366f21b1 100644 (file)
--- a/common/macroblock.c
+++ b/common/macroblock.c
@@ -52,22 +52,40 @@ int x264_mb_predict_non_zero_code( x264_t *h, int idx )
 
 int x264_mb_transform_8x8_allowed( x264_t *h )
 {
-    if( IS_SKIP( h->mb.i_type ) )
+    // intra and skip are disallowed
+    // large partitions are allowed
+    // direct and 8x8 are conditional
+    static const uint8_t partition_tab[X264_MBTYPE_MAX] = {
+        0,0,0,0,1,2,0,2,1,1,1,1,1,1,1,1,1,2,0,
+    };
+    int p, i;
+
+    if( !h->pps->b_transform_8x8_mode )
         return 0;
-    if( h->mb.i_type == P_8x8 || h->mb.i_type == B_8x8 )
+    p = partition_tab[h->mb.i_type];
+    if( p < 2 )
+        return p;
+    else if( h->mb.i_type == B_DIRECT )
+        return h->sps->b_direct8x8_inference;
+    else if( h->mb.i_type == P_8x8 )
+    {
+        if( !(h->param.analyse.inter & X264_ANALYSE_PSUB8x8) )
+            return 1;
+        for( i=0; i<4; i++ )
+            if( h->mb.i_sub_partition[i] != D_L0_8x8 )
+                return 0;
+        return 1;
+    }
+    else // B_8x8
     {
-        int i;
-        for( i = 0; i < 4; i++ )
-            if( !IS_SUB8x8(h->mb.i_sub_partition[i])
-                || ( h->mb.i_sub_partition[i] == D_DIRECT_8x8 && !h->sps->b_direct8x8_inference ) )
-            {
+        // x264 currently doesn't use sub-8x8 B partitions, so don't check for them
+        if( h->sps->b_direct8x8_inference )
+            return 1;
+        for( i=0; i<4; i++ )
+            if( h->mb.i_sub_partition[i] == D_DIRECT_8x8 )
                 return 0;
-            }
+        return 1;
     }
-    if( h->mb.i_type == B_DIRECT && !h->sps->b_direct8x8_inference )
-        return 0;
-
-    return 1;
 }
 
 void x264_mb_predict_mv( x264_t *h, int i_list, int idx, int i_width, int mvp[2] )

diff --git a/common/macroblock.h b/common/macroblock.h
index 1b4a48696413429959c589d8be014df8b841d679..570bb80dd75dbfbb5bb4bece4c6bf5975dd74b9b 100644 (file)
--- a/common/macroblock.h
+++ b/common/macroblock.h
@@ -287,7 +287,7 @@ int  x264_mb_predict_non_zero_code( x264_t *h, int idx );
 /* x264_mb_transform_8x8_allowed:
  *      check whether any partition is smaller than 8x8 (or at least
  *      might be, according to just partition type.)
- *      doesn't check for intra or cbp */
+ *      doesn't check for cbp */
 int  x264_mb_transform_8x8_allowed( x264_t *h );
 
 void x264_mb_mc( x264_t *h );

diff --git a/encoder/analyse.c b/encoder/analyse.c
index 3d7ba90ff0e176a5cda4f0eb686256e5d633f5e9..4532bf224ebc159f91daa58b6c7ef8017cb53478 100644 (file)
--- a/encoder/analyse.c
+++ b/encoder/analyse.c
@@ -1984,11 +1984,7 @@ static void refine_bidir( x264_t *h, x264_mb_analysis_t *a )
 
 static inline void x264_mb_analyse_transform( x264_t *h )
 {
-    h->mb.cache.b_transform_8x8_allowed =
-        h->pps->b_transform_8x8_mode
-        && !IS_INTRA( h->mb.i_type ) && x264_mb_transform_8x8_allowed( h );
-
-    if( h->mb.cache.b_transform_8x8_allowed && h->param.analyse.b_transform_8x8 )
+    if( x264_mb_transform_8x8_allowed( h ) && h->param.analyse.b_transform_8x8 )
     {
         int i_cost4, i_cost8;
         /* FIXME only luma mc is needed */
@@ -2005,10 +2001,7 @@ static inline void x264_mb_analyse_transform( x264_t *h )
 
 static inline void x264_mb_analyse_transform_rd( x264_t *h, x264_mb_analysis_t *a, int *i_satd, int *i_rd )
 {
-    h->mb.cache.b_transform_8x8_allowed =
-        h->pps->b_transform_8x8_mode && x264_mb_transform_8x8_allowed( h );
-
-    if( h->mb.cache.b_transform_8x8_allowed && h->param.analyse.b_transform_8x8 )
+    if( x264_mb_transform_8x8_allowed( h ) && h->param.analyse.b_transform_8x8 )
     {
         int i_rd8;
         x264_analyse_update_cache( h, a );

diff --git a/encoder/cabac.c b/encoder/cabac.c
index f5ece2bc5e6bbcbec3f87e8502da3aadebc6d56c..6b3a0180fbc3ed29be63731ac7ed14310fe2a8f2 100644 (file)
--- a/encoder/cabac.c
+++ b/encoder/cabac.c
@@ -1009,7 +1009,7 @@ void x264_macroblock_write_cabac( x264_t *h, x264_cabac_t *cb )
         x264_cabac_mb_cbp_chroma( h, cb );
     }
 
-    if( h->mb.cache.b_transform_8x8_allowed && h->mb.i_cbp_luma && !IS_INTRA(i_mb_type) )
+    if( x264_mb_transform_8x8_allowed( h ) && h->mb.i_cbp_luma )
     {
         x264_cabac_mb_transform_size( h, cb );
     }

diff --git a/encoder/cavlc.c b/encoder/cavlc.c
index 6ac15b191b401b6165f597905b9ae8533c67cce2..a20c0b0f4d9e9211c3ca3c80239398dc17bfa516 100644 (file)
--- a/encoder/cavlc.c
+++ b/encoder/cavlc.c
@@ -650,7 +650,7 @@ void x264_macroblock_write_cavlc( x264_t *h, bs_t *s )
     }
 
     /* transform size 8x8 flag */
-    if( h->mb.cache.b_transform_8x8_allowed && h->mb.i_cbp_luma && !IS_INTRA(i_mb_type) )
+    if( x264_mb_transform_8x8_allowed( h ) && h->mb.i_cbp_luma )
     {
         bs_write1( s, h->mb.b_transform_8x8 );
     }