Fix satd predictors with high bit depth

Resulted in odd CRF-mode results with --no-mbtree, as well as suboptimal VBV handling.

diff --git a/encoder/slicetype.c b/encoder/slicetype.c
index fcf30a5ce89c0faa6a5c141cf0a23540f7689373..5c52aaf99a6f668b3b94b57396f5f8fa33653009 100644 (file)
--- a/encoder/slicetype.c
+++ b/encoder/slicetype.c
@@ -1647,5 +1647,9 @@ int x264_rc_analyse_slice( x264_t *h )
         }
     }
 
-    return cost;
+    if( BIT_DEPTH > 8 )
+        for( int y = 0; y < h->mb.i_mb_height; y++ )
+            h->fdec->i_row_satd[y] >>= (BIT_DEPTH - 8);
+
+    return cost >> (BIT_DEPTH - 8);
 }