vaapi_h264: Scale log2_max_pic_order_cnt_lsb with max_b_frames

Before this change, it was possible to overflow pic_order_cnt_lsb and
generate a stream with invalid POC numbering.  This makes sure that
the field is large enough that a single IDR B* P sequence uses fewer
than half the available POC lsb values.

(cherry picked from commit 89725a8512721fffd190021ded2d3f5b42e20e2a)

diff --git a/libavcodec/vaapi_encode_h264.c b/libavcodec/vaapi_encode_h264.c
index 00d8e6a3bf041acf26c764eb971c3d7e2a3a9159..b98052791751f12bddbf58f4b52958c537f580e2 100644 (file)
--- a/libavcodec/vaapi_encode_h264.c
+++ b/libavcodec/vaapi_encode_h264.c
@@ -800,6 +800,8 @@ static int vaapi_encode_h264_init_sequence_params(AVCodecContext *avctx)
         vseq->seq_fields.bits.direct_8x8_inference_flag = 1;
         vseq->seq_fields.bits.log2_max_frame_num_minus4 = 4;
         vseq->seq_fields.bits.pic_order_cnt_type = 0;
+        vseq->seq_fields.bits.log2_max_pic_order_cnt_lsb_minus4 =
+            av_clip(av_log2(avctx->max_b_frames + 1) - 2, 0, 12);
 
         if (avctx->width  != ctx->surface_width ||
             avctx->height != ctx->surface_height) {