ret = ff_set_dimensions(avctx, s->coded_width, s->coded_height);
if (ret < 0)
return ret;
- if (s->cropped_height)
- avctx->height = s->cropped_height << (avctx->pix_fmt == AV_PIX_FMT_BAYER_RGGB16);
+ if (s->cropped_height) {
+ unsigned height = s->cropped_height << (avctx->pix_fmt == AV_PIX_FMT_BAYER_RGGB16);
+ if (avctx->height < height)
+ return AVERROR_INVALIDDATA;
+ avctx->height = height;
+ }
frame.f->width =
frame.f->height = 0;
}
low = s->plane[plane].l_h[6];
high = s->plane[plane].l_h[7];
+
+ if (avctx->pix_fmt == AV_PIX_FMT_BAYER_RGGB16 &&
+ (lowpass_height * 2 > avctx->coded_height / 2 ||
+ lowpass_width * 2 > avctx->coded_width / 2 )
+ ) {
+ ret = AVERROR_INVALIDDATA;
+ goto end;
+ }
+
for (i = 0; i < lowpass_height * 2; i++) {
if (avctx->pix_fmt == AV_PIX_FMT_BAYER_RGGB16)
horiz_filter_clip_bayer(dst, low, high, lowpass_width, s->bpc);