Safety check against malicious high bit-depth input which could cause crash

diff --git a/encoder/rdo.c b/encoder/rdo.c
index 93fdc3a6d238e5d8304f16e4333634980645500a..ceb2e2cf61d8ceca3a00bd9f6dc888eb5bab590b 100644 (file)
--- a/encoder/rdo.c
+++ b/encoder/rdo.c
@@ -186,7 +186,7 @@ static int x264_rd_cost_mb( x264_t *h, int i_lambda2 )
     h->mb.b_transform_8x8 = b_transform_bak;
     h->mb.i_type = type_bak;
 
-    return i_ssd + i_bits;
+    return X264_MIN( i_ssd + i_bits, COST_MAX );
 }
 
 /* partition RD functions use 8 bits more precision to avoid large rounding errors at low QPs */