From 12681eea55239d21eb8ce0ca058c5d859afedc9f Mon Sep 17 00:00:00 2001 From: Loren Merritt Date: Wed, 4 Apr 2007 18:59:20 +0000 Subject: [PATCH] fix an arithmetic overflow in trellis with QP >= 42 git-svn-id: svn://svn.videolan.org/x264/trunk@643 df754926-b1dd-0310-bc7b-ec298dee348c --- common/set.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/set.c b/common/set.c index 9ae8aa5d..8a6886f7 100644 --- a/common/set.c +++ b/common/set.c @@ -148,7 +148,7 @@ int x264_cqm_init( x264_t *h ) for( i_list = 0; i_list < 4; i_list++ ) for( i = 0; i < 16; i++ ) { - h->unquant4_mf[i_list][q][i] = (1 << (q/6 + 15 + 8)) / quant4_mf[i_list][q%6][0][i]; + h->unquant4_mf[i_list][q][i] = (1ULL << (q/6 + 15 + 8)) / quant4_mf[i_list][q%6][0][i]; h-> quant4_mf[i_list][q][i] = j = SHIFT(quant4_mf[i_list][q%6][0][i], q/6 - 1); // round to nearest, unless that would cause the deadzone to be negative h->quant4_bias[i_list][q][i] = X264_MIN( DIV(deadzone[i_list]<<10, j), (1<<15)/j ); @@ -159,7 +159,7 @@ int x264_cqm_init( x264_t *h ) for( i_list = 0; i_list < 2; i_list++ ) for( i = 0; i < 64; i++ ) { - h->unquant8_mf[i_list][q][i] = (1 << (q/6 + 16 + 8)) / quant8_mf[i_list][q%6][0][i]; + h->unquant8_mf[i_list][q][i] = (1ULL << (q/6 + 16 + 8)) / quant8_mf[i_list][q%6][0][i]; h-> quant8_mf[i_list][q][i] = j = SHIFT(quant8_mf[i_list][q%6][0][i], q/6); h->quant8_bias[i_list][q][i] = X264_MIN( DIV(deadzone[i_list]<<10, j), (1<<15)/j ); if( j > 0xffff && q > max_qp_err ) -- 2.39.2