From 24e4fed388fcb34c33df7c87e7d6758b9ebed40c Mon Sep 17 00:00:00 2001 From: Anton Mitrofanov Date: Sun, 12 Oct 2014 21:01:53 +0400 Subject: [PATCH] Safety check against malicious high bit-depth input which could cause crash --- encoder/rdo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/encoder/rdo.c b/encoder/rdo.c index 93fdc3a6..ceb2e2cf 100644 --- a/encoder/rdo.c +++ b/encoder/rdo.c @@ -186,7 +186,7 @@ static int x264_rd_cost_mb( x264_t *h, int i_lambda2 ) h->mb.b_transform_8x8 = b_transform_bak; h->mb.i_type = type_bak; - return i_ssd + i_bits; + return X264_MIN( i_ssd + i_bits, COST_MAX ); } /* partition RD functions use 8 bits more precision to avoid large rounding errors at low QPs */ -- 2.39.2