From aa61f21988e3733c2a3eb36f9b57993f38a5635d Mon Sep 17 00:00:00 2001 From: Laurent Aimar Date: Wed, 28 Jul 2010 23:50:22 +0200 Subject: [PATCH] Fixed invalid assumption about picture pitches in rotate filter. It may segfault or create a broken image. --- modules/video_filter/rotate.c | 59 ++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/modules/video_filter/rotate.c b/modules/video_filter/rotate.c index eabdfcb9e2..52bc735ad0 100644 --- a/modules/video_filter/rotate.c +++ b/modules/video_filter/rotate.c @@ -187,10 +187,11 @@ static picture_t *Filter( filter_t *p_filter, picture_t *p_pic ) for( int i_plane = 0 ; i_plane < p_pic->i_planes ; i_plane++ ) { - const int i_visible_lines = p_pic->p[i_plane].i_visible_lines; - const int i_visible_pitch = p_pic->p[i_plane].i_visible_pitch; - const int i_pitch = p_pic->p[i_plane].i_pitch; - const int i_hidden_pitch = i_pitch - i_visible_pitch; + plane_t *p_srcp = &p_pic->p[i_plane]; + plane_t *p_dstp = &p_outpic->p[i_plane]; + + const int i_visible_lines = p_srcp->i_visible_lines; + const int i_visible_pitch = p_srcp->i_visible_pitch; const int i_aspect = __MAX( 1, ( i_visible_lines * p_pic->p[Y_PLANE].i_visible_pitch ) / ( p_pic->p[Y_PLANE].i_visible_lines * i_visible_pitch )); /* = 2 for U and V planes in YUV 4:2:2, = 1 otherwise */ @@ -198,11 +199,6 @@ static picture_t *Filter( filter_t *p_filter, picture_t *p_pic ) const int i_line_center = i_visible_lines>>1; const int i_col_center = i_visible_pitch>>1; - const uint8_t *p_in = p_pic->p[i_plane].p_pixels; - uint8_t *p_out = p_outpic->p[i_plane].p_pixels; - uint8_t *p_outendline = p_out + i_visible_pitch; - const uint8_t *p_outend = p_out + i_visible_lines * i_pitch; - const uint8_t black_pixel = ( i_plane == Y_PLANE ) ? 0x00 : 0x80; const int i_line_next = i_cos / i_aspect -i_sin*i_visible_pitch; @@ -211,17 +207,15 @@ static picture_t *Filter( filter_t *p_filter, picture_t *p_pic ) - i_sin * i_col_center + (1<<11) ); int i_col_orig0 = i_sin * i_line_center / i_aspect - i_cos * i_col_center + (1<<11); - for( ; p_outendline < p_outend; - p_out += i_hidden_pitch, p_outendline += i_pitch, - i_line_orig0 += i_line_next, i_col_orig0 += i_col_next ) + for( int y = 0; y < i_visible_lines; y++) { - for( ; p_out < p_outendline; - p_out++, i_line_orig0 += i_sin, i_col_orig0 += i_cos ) + uint8_t *p_out = &p_dstp->p_pixels[y * p_dstp->i_pitch]; + + for( int x = 0; x < i_visible_pitch; x++, p_out++ ) { const int i_line_orig = (i_line_orig0>>12)*i_aspect + i_line_center; const int i_col_orig = (i_col_orig0>>12) + i_col_center; - const uint8_t* p_orig_offset = p_in + i_line_orig * i_pitch - + i_col_orig; + const uint8_t *p_orig_offset = &p_srcp->p_pixels[i_line_orig * p_srcp->i_pitch + i_col_orig]; const uint8_t i_line_percent = (i_line_orig0>>4) & 255; const uint8_t i_col_percent = (i_col_orig0 >>4) & 255; @@ -246,7 +240,7 @@ static picture_t *Filter( filter_t *p_filter, picture_t *p_pic ) && ( i_line_orig >= 0 ) ) i_colpix = *p_orig_offset; - p_orig_offset+=i_pitch; + p_orig_offset += p_srcp->i_pitch; if( ( i_line_orig < i_visible_lines - 1) && ( i_col_orig < i_visible_pitch - 1) ) i_nexpix = *p_orig_offset; @@ -280,7 +274,13 @@ static picture_t *Filter( filter_t *p_filter, picture_t *p_pic ) { *p_out = black_pixel; } + + i_line_orig0 += i_sin; + i_col_orig0 += i_cos; } + + i_line_orig0 += i_line_next; + i_col_orig0 += i_col_next; } } @@ -315,17 +315,18 @@ static picture_t *FilterPacked( filter_t *p_filter, picture_t *p_pic ) return NULL; } + const int i_visible_pitch = p_pic->p->i_visible_pitch>>1; /* In fact it's i_visible_pixels */ + const int i_visible_lines = p_pic->p->i_visible_lines; + const uint8_t *p_in = p_pic->p->p_pixels+i_y_offset; const uint8_t *p_in_u = p_pic->p->p_pixels+i_u_offset; const uint8_t *p_in_v = p_pic->p->p_pixels+i_v_offset; - - const int i_pitch = p_pic->p->i_pitch; - const int i_visible_pitch = p_pic->p->i_visible_pitch>>1; /* In fact it's i_visible_pixels */ - const int i_visible_lines = p_pic->p->i_visible_lines; + const int i_in_pitch = p_pic->p->i_pitch; uint8_t *p_out = p_outpic->p->p_pixels+i_y_offset; uint8_t *p_out_u = p_outpic->p->p_pixels+i_u_offset; uint8_t *p_out_v = p_outpic->p->p_pixels+i_v_offset; + const int i_out_pitch = p_outpic->p->i_pitch; const int i_line_center = i_visible_lines>>1; const int i_col_center = i_visible_pitch>>1; @@ -352,16 +353,16 @@ static picture_t *FilterPacked( filter_t *p_filter, picture_t *p_pic ) if( 0 <= i_col_orig && i_col_orig < i_visible_pitch && 0 <= i_line_orig && i_line_orig < i_visible_lines ) { - p_out[i_line*i_pitch+2*i_col] = p_in[i_line_orig*i_pitch+2*i_col_orig]; + p_out[i_line*i_out_pitch+2*i_col] = p_in[i_line_orig*i_in_pitch+2*i_col_orig]; i_col_orig /= 2; - p_out_u[i_line*i_pitch+2*i_col] = p_in_u[i_line_orig*i_pitch+4*i_col_orig]; - p_out_v[i_line*i_pitch+2*i_col] = p_in_v[i_line_orig*i_pitch+4*i_col_orig]; + p_out_u[i_line*i_out_pitch+2*i_col] = p_in_u[i_line_orig*i_in_pitch+4*i_col_orig]; + p_out_v[i_line*i_out_pitch+2*i_col] = p_in_v[i_line_orig*i_in_pitch+4*i_col_orig]; } else { - p_out[i_line*i_pitch+2*i_col] = 0x00; - p_out_u[i_line*i_pitch+2*i_col] = 0x80; - p_out_v[i_line*i_pitch+2*i_col] = 0x80; + p_out[i_line*i_out_pitch+2*i_col] = 0x00; + p_out_u[i_line*i_out_pitch+2*i_col] = 0x80; + p_out_v[i_line*i_out_pitch+2*i_col] = 0x80; } /* Handle "2nd Y" */ @@ -378,11 +379,11 @@ static picture_t *FilterPacked( filter_t *p_filter, picture_t *p_pic ) if( 0 <= i_col_orig && i_col_orig < i_visible_pitch && 0 <= i_line_orig && i_line_orig < i_visible_lines ) { - p_out[i_line*i_pitch+2*i_col] = p_in[i_line_orig*i_pitch+2*i_col_orig]; + p_out[i_line*i_out_pitch+2*i_col] = p_in[i_line_orig*i_in_pitch+2*i_col_orig]; } else { - p_out[i_line*i_pitch+2*i_col] = 0x00; + p_out[i_line*i_out_pitch+2*i_col] = 0x00; } } } -- 2.39.2