2 * Intel MediaSDK QSV encoder utility functions
4 * copyright (c) 2013 Yukinori Yamazoe
5 * copyright (c) 2015 Anton Khirnov
7 * This file is part of FFmpeg.
9 * FFmpeg is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public
11 * License as published by the Free Software Foundation; either
12 * version 2.1 of the License, or (at your option) any later version.
14 * FFmpeg is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with FFmpeg; if not, write to the Free Software
21 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
25 #include <sys/types.h>
26 #include <mfx/mfxvideo.h>
28 #include "libavutil/common.h"
29 #include "libavutil/mem.h"
30 #include "libavutil/log.h"
31 #include "libavutil/time.h"
32 #include "libavutil/imgutils.h"
37 #include "qsv_internal.h"
40 static int init_video_param(AVCodecContext *avctx, QSVEncContext *q)
42 const char *ratecontrol_desc;
47 ret = ff_qsv_codec_id_to_mfx(avctx->codec_id);
50 q->param.mfx.CodecId = ret;
52 q->width_align = avctx->codec_id == AV_CODEC_ID_HEVC ? 32 : 16;
55 q->param.mfx.CodecLevel = avctx->level;
57 q->param.mfx.CodecProfile = q->profile;
58 q->param.mfx.TargetUsage = q->preset;
59 q->param.mfx.GopPicSize = FFMAX(0, avctx->gop_size);
60 q->param.mfx.GopRefDist = FFMAX(-1, avctx->max_b_frames) + 1;
61 q->param.mfx.GopOptFlag = avctx->flags & AV_CODEC_FLAG_CLOSED_GOP ?
63 q->param.mfx.IdrInterval = q->idr_interval;
64 q->param.mfx.NumSlice = avctx->slices;
65 q->param.mfx.NumRefFrame = FFMAX(0, avctx->refs);
66 q->param.mfx.EncodedOrder = 0;
67 q->param.mfx.BufferSizeInKB = 0;
69 q->param.mfx.FrameInfo.FourCC = MFX_FOURCC_NV12;
70 q->param.mfx.FrameInfo.CropX = 0;
71 q->param.mfx.FrameInfo.CropY = 0;
72 q->param.mfx.FrameInfo.CropW = avctx->width;
73 q->param.mfx.FrameInfo.CropH = avctx->height;
74 q->param.mfx.FrameInfo.AspectRatioW = avctx->sample_aspect_ratio.num;
75 q->param.mfx.FrameInfo.AspectRatioH = avctx->sample_aspect_ratio.den;
76 q->param.mfx.FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV420;
77 q->param.mfx.FrameInfo.BitDepthLuma = 8;
78 q->param.mfx.FrameInfo.BitDepthChroma = 8;
79 q->param.mfx.FrameInfo.Width = FFALIGN(avctx->width, q->width_align);
81 if (avctx->flags & AV_CODEC_FLAG_INTERLACED_DCT) {
82 /* A true field layout (TFF or BFF) is not important here,
83 it will specified later during frame encoding. But it is important
84 to specify is frame progressive or not because allowed heigh alignment
87 q->param.mfx.FrameInfo.PicStruct = MFX_PICSTRUCT_FIELD_TFF;
90 q->param.mfx.FrameInfo.PicStruct = MFX_PICSTRUCT_PROGRESSIVE;
93 q->param.mfx.FrameInfo.Height = FFALIGN(avctx->height, q->height_align);
95 if (avctx->framerate.den > 0 && avctx->framerate.num > 0) {
96 q->param.mfx.FrameInfo.FrameRateExtN = avctx->framerate.num;
97 q->param.mfx.FrameInfo.FrameRateExtD = avctx->framerate.den;
99 q->param.mfx.FrameInfo.FrameRateExtN = avctx->time_base.den;
100 q->param.mfx.FrameInfo.FrameRateExtD = avctx->time_base.num;
103 if (avctx->flags & AV_CODEC_FLAG_QSCALE) {
104 q->param.mfx.RateControlMethod = MFX_RATECONTROL_CQP;
105 ratecontrol_desc = "constant quantization parameter (CQP)";
106 } else if (avctx->rc_max_rate == avctx->bit_rate) {
107 q->param.mfx.RateControlMethod = MFX_RATECONTROL_CBR;
108 ratecontrol_desc = "constant bitrate (CBR)";
109 } else if (!avctx->rc_max_rate) {
110 #if QSV_VERSION_ATLEAST(1,7)
112 q->param.mfx.RateControlMethod = MFX_RATECONTROL_LA;
113 ratecontrol_desc = "lookahead (LA)";
117 q->param.mfx.RateControlMethod = MFX_RATECONTROL_AVBR;
118 ratecontrol_desc = "average variable bitrate (AVBR)";
121 q->param.mfx.RateControlMethod = MFX_RATECONTROL_VBR;
122 ratecontrol_desc = "variable bitrate (VBR)";
125 av_log(avctx, AV_LOG_VERBOSE, "Using the %s ratecontrol method\n", ratecontrol_desc);
127 switch (q->param.mfx.RateControlMethod) {
128 case MFX_RATECONTROL_CBR:
129 case MFX_RATECONTROL_VBR:
130 q->param.mfx.InitialDelayInKB = avctx->rc_initial_buffer_occupancy / 1000;
131 q->param.mfx.TargetKbps = avctx->bit_rate / 1000;
132 q->param.mfx.MaxKbps = avctx->rc_max_rate / 1000;
134 case MFX_RATECONTROL_CQP:
135 quant = avctx->global_quality / FF_QP2LAMBDA;
137 q->param.mfx.QPI = av_clip(quant * fabs(avctx->i_quant_factor) + avctx->i_quant_offset, 0, 51);
138 q->param.mfx.QPP = av_clip(quant, 0, 51);
139 q->param.mfx.QPB = av_clip(quant * fabs(avctx->b_quant_factor) + avctx->b_quant_offset, 0, 51);
142 case MFX_RATECONTROL_AVBR:
143 #if QSV_VERSION_ATLEAST(1,7)
144 case MFX_RATECONTROL_LA:
146 q->param.mfx.TargetKbps = avctx->bit_rate / 1000;
147 q->param.mfx.Convergence = q->avbr_convergence;
148 q->param.mfx.Accuracy = q->avbr_accuracy;
152 // the HEVC encoder plugin currently fails if coding options
154 if (avctx->codec_id != AV_CODEC_ID_HEVC) {
155 q->extco.Header.BufferId = MFX_EXTBUFF_CODING_OPTION;
156 q->extco.Header.BufferSz = sizeof(q->extco);
157 q->extco.CAVLC = avctx->coder_type == FF_CODER_TYPE_VLC ?
158 MFX_CODINGOPTION_ON : MFX_CODINGOPTION_UNKNOWN;
160 q->extco.PicTimingSEI = q->pic_timing_sei ?
161 MFX_CODINGOPTION_ON : MFX_CODINGOPTION_UNKNOWN;
163 q->extparam_internal[q->nb_extparam_internal++] = (mfxExtBuffer *)&q->extco;
165 #if QSV_VERSION_ATLEAST(1,6)
166 q->extco2.Header.BufferId = MFX_EXTBUFF_CODING_OPTION2;
167 q->extco2.Header.BufferSz = sizeof(q->extco2);
169 #if QSV_VERSION_ATLEAST(1,7)
170 // valid value range is from 10 to 100 inclusive
171 // to instruct the encoder to use the default value this should be set to zero
172 q->extco2.LookAheadDepth = q->look_ahead_depth != 0 ? FFMAX(10, q->look_ahead_depth) : 0;
174 #if QSV_VERSION_ATLEAST(1,8)
175 q->extco2.LookAheadDS = q->look_ahead_downsampling;
178 q->extparam_internal[q->nb_extparam_internal++] = (mfxExtBuffer *)&q->extco2;
186 static int qsv_retrieve_enc_params(AVCodecContext *avctx, QSVEncContext *q)
188 uint8_t sps_buf[128];
189 uint8_t pps_buf[128];
191 mfxExtCodingOptionSPSPPS extradata = {
192 .Header.BufferId = MFX_EXTBUFF_CODING_OPTION_SPSPPS,
193 .Header.BufferSz = sizeof(extradata),
194 .SPSBuffer = sps_buf, .SPSBufSize = sizeof(sps_buf),
195 .PPSBuffer = pps_buf, .PPSBufSize = sizeof(pps_buf)
198 mfxExtBuffer *ext_buffers[] = {
199 (mfxExtBuffer*)&extradata,
202 int need_pps = avctx->codec_id != AV_CODEC_ID_MPEG2VIDEO;
205 q->param.ExtParam = ext_buffers;
206 q->param.NumExtParam = FF_ARRAY_ELEMS(ext_buffers);
208 ret = MFXVideoENCODE_GetVideoParam(q->session, &q->param);
210 return ff_qsv_error(ret);
212 q->packet_size = q->param.mfx.BufferSizeInKB * 1000;
214 if (!extradata.SPSBufSize || (need_pps && !extradata.PPSBufSize)) {
215 av_log(avctx, AV_LOG_ERROR, "No extradata returned from libmfx.\n");
216 return AVERROR_UNKNOWN;
219 avctx->extradata = av_malloc(extradata.SPSBufSize + need_pps * extradata.PPSBufSize +
220 AV_INPUT_BUFFER_PADDING_SIZE);
221 if (!avctx->extradata)
222 return AVERROR(ENOMEM);
224 memcpy(avctx->extradata, sps_buf, extradata.SPSBufSize);
226 memcpy(avctx->extradata + extradata.SPSBufSize, pps_buf, extradata.PPSBufSize);
227 avctx->extradata_size = extradata.SPSBufSize + need_pps * extradata.PPSBufSize;
228 memset(avctx->extradata + avctx->extradata_size, 0, AV_INPUT_BUFFER_PADDING_SIZE);
233 static int qsv_init_opaque_alloc(AVCodecContext *avctx, QSVEncContext *q)
235 AVQSVContext *qsv = avctx->hwaccel_context;
236 mfxFrameSurface1 *surfaces;
239 nb_surfaces = qsv->nb_opaque_surfaces + q->req.NumFrameSuggested + q->async_depth;
241 q->opaque_alloc_buf = av_buffer_allocz(sizeof(*surfaces) * nb_surfaces);
242 if (!q->opaque_alloc_buf)
243 return AVERROR(ENOMEM);
245 q->opaque_surfaces = av_malloc_array(nb_surfaces, sizeof(*q->opaque_surfaces));
246 if (!q->opaque_surfaces)
247 return AVERROR(ENOMEM);
249 surfaces = (mfxFrameSurface1*)q->opaque_alloc_buf->data;
250 for (i = 0; i < nb_surfaces; i++) {
251 surfaces[i].Info = q->req.Info;
252 q->opaque_surfaces[i] = surfaces + i;
255 q->opaque_alloc.Header.BufferId = MFX_EXTBUFF_OPAQUE_SURFACE_ALLOCATION;
256 q->opaque_alloc.Header.BufferSz = sizeof(q->opaque_alloc);
257 q->opaque_alloc.In.Surfaces = q->opaque_surfaces;
258 q->opaque_alloc.In.NumSurface = nb_surfaces;
259 q->opaque_alloc.In.Type = q->req.Type;
261 q->extparam_internal[q->nb_extparam_internal++] = (mfxExtBuffer *)&q->opaque_alloc;
263 qsv->nb_opaque_surfaces = nb_surfaces;
264 qsv->opaque_surfaces = q->opaque_alloc_buf;
265 qsv->opaque_alloc_type = q->req.Type;
270 int ff_qsv_enc_init(AVCodecContext *avctx, QSVEncContext *q)
272 int opaque_alloc = 0;
275 q->param.IOPattern = MFX_IOPATTERN_IN_SYSTEM_MEMORY;
276 q->param.AsyncDepth = q->async_depth;
278 q->async_fifo = av_fifo_alloc((1 + q->async_depth) *
279 (sizeof(AVPacket) + sizeof(mfxSyncPoint) + sizeof(mfxBitstream*)));
281 return AVERROR(ENOMEM);
283 if (avctx->hwaccel_context) {
284 AVQSVContext *qsv = avctx->hwaccel_context;
286 q->session = qsv->session;
287 q->param.IOPattern = qsv->iopattern;
289 opaque_alloc = qsv->opaque_alloc;
293 ret = ff_qsv_init_internal_session(avctx, &q->internal_qs,
298 q->session = q->internal_qs.session;
301 ret = init_video_param(avctx, q);
305 ret = MFXVideoENCODE_Query(q->session, &q->param,&q->param);
306 if (MFX_WRN_PARTIAL_ACCELERATION==ret) {
307 av_log(avctx, AV_LOG_WARNING, "Encoder will work with partial HW acceleration\n");
308 } else if (ret < 0) {
309 av_log(avctx, AV_LOG_ERROR, "Error %d querying encoder params\n", ret);
310 return ff_qsv_error(ret);
313 ret = MFXVideoENCODE_QueryIOSurf(q->session, &q->param, &q->req);
315 av_log(avctx, AV_LOG_ERROR, "Error querying the encoding parameters\n");
316 return ff_qsv_error(ret);
320 ret = qsv_init_opaque_alloc(avctx, q);
325 if (avctx->hwaccel_context) {
326 AVQSVContext *qsv = avctx->hwaccel_context;
329 q->extparam = av_mallocz_array(qsv->nb_ext_buffers + q->nb_extparam_internal,
330 sizeof(*q->extparam));
332 return AVERROR(ENOMEM);
334 q->param.ExtParam = q->extparam;
335 for (i = 0; i < qsv->nb_ext_buffers; i++)
336 q->param.ExtParam[i] = qsv->ext_buffers[i];
337 q->param.NumExtParam = qsv->nb_ext_buffers;
339 for (i = 0; i < q->nb_extparam_internal; i++) {
340 for (j = 0; j < qsv->nb_ext_buffers; j++) {
341 if (qsv->ext_buffers[j]->BufferId == q->extparam_internal[i]->BufferId)
344 if (j < qsv->nb_ext_buffers)
347 q->param.ExtParam[q->param.NumExtParam++] = q->extparam_internal[i];
350 q->param.ExtParam = q->extparam_internal;
351 q->param.NumExtParam = q->nb_extparam_internal;
354 ret = MFXVideoENCODE_Init(q->session, &q->param);
355 if (MFX_WRN_PARTIAL_ACCELERATION==ret) {
356 av_log(avctx, AV_LOG_WARNING, "Encoder will work with partial HW acceleration\n");
357 } else if (ret < 0) {
358 av_log(avctx, AV_LOG_ERROR, "Error initializing the encoder\n");
359 return ff_qsv_error(ret);
362 ret = qsv_retrieve_enc_params(avctx, q);
364 av_log(avctx, AV_LOG_ERROR, "Error retrieving encoding parameters.\n");
373 static void clear_unused_frames(QSVEncContext *q)
375 QSVFrame *cur = q->work_frames;
377 if (cur->surface && !cur->surface->Data.Locked) {
379 av_frame_unref(cur->frame);
385 static int get_free_frame(QSVEncContext *q, QSVFrame **f)
387 QSVFrame *frame, **last;
389 clear_unused_frames(q);
391 frame = q->work_frames;
392 last = &q->work_frames;
394 if (!frame->surface) {
403 frame = av_mallocz(sizeof(*frame));
405 return AVERROR(ENOMEM);
406 frame->frame = av_frame_alloc();
409 return AVERROR(ENOMEM);
418 static int submit_frame(QSVEncContext *q, const AVFrame *frame,
419 mfxFrameSurface1 **surface)
424 ret = get_free_frame(q, &qf);
428 if (frame->format == AV_PIX_FMT_QSV) {
429 ret = av_frame_ref(qf->frame, frame);
433 qf->surface = (mfxFrameSurface1*)qf->frame->data[3];
435 /* make a copy if the input is not padded as libmfx requires */
436 if ( frame->height & (q->height_align - 1) ||
437 frame->linesize[0] & (q->width_align - 1)) {
438 qf->frame->height = FFALIGN(frame->height, q->height_align);
439 qf->frame->width = FFALIGN(frame->width, q->width_align);
441 ret = ff_get_buffer(q->avctx, qf->frame, AV_GET_BUFFER_FLAG_REF);
445 qf->frame->height = frame->height;
446 qf->frame->width = frame->width;
447 ret = av_frame_copy(qf->frame, frame);
449 av_frame_unref(qf->frame);
453 ret = av_frame_ref(qf->frame, frame);
458 qf->surface_internal.Info = q->param.mfx.FrameInfo;
460 qf->surface_internal.Info.PicStruct =
461 !frame->interlaced_frame ? MFX_PICSTRUCT_PROGRESSIVE :
462 frame->top_field_first ? MFX_PICSTRUCT_FIELD_TFF :
463 MFX_PICSTRUCT_FIELD_BFF;
464 if (frame->repeat_pict == 1)
465 qf->surface_internal.Info.PicStruct |= MFX_PICSTRUCT_FIELD_REPEATED;
466 else if (frame->repeat_pict == 2)
467 qf->surface_internal.Info.PicStruct |= MFX_PICSTRUCT_FRAME_DOUBLING;
468 else if (frame->repeat_pict == 4)
469 qf->surface_internal.Info.PicStruct |= MFX_PICSTRUCT_FRAME_TRIPLING;
471 qf->surface_internal.Data.PitchLow = qf->frame->linesize[0];
472 qf->surface_internal.Data.Y = qf->frame->data[0];
473 qf->surface_internal.Data.UV = qf->frame->data[1];
475 qf->surface = &qf->surface_internal;
478 qf->surface->Data.TimeStamp = av_rescale_q(frame->pts, q->avctx->time_base, (AVRational){1, 90000});
480 *surface = qf->surface;
485 static void print_interlace_msg(AVCodecContext *avctx, QSVEncContext *q)
487 if (q->param.mfx.CodecId == MFX_CODEC_AVC) {
488 if (q->param.mfx.CodecProfile == MFX_PROFILE_AVC_BASELINE ||
489 q->param.mfx.CodecLevel < MFX_LEVEL_AVC_21 ||
490 q->param.mfx.CodecLevel > MFX_LEVEL_AVC_41)
491 av_log(avctx, AV_LOG_WARNING,
492 "Interlaced coding is supported"
493 " at Main/High Profile Level 2.1-4.1\n");
497 int ff_qsv_encode(AVCodecContext *avctx, QSVEncContext *q,
498 AVPacket *pkt, const AVFrame *frame, int *got_packet)
500 AVPacket new_pkt = { 0 };
503 mfxFrameSurface1 *surf = NULL;
504 mfxSyncPoint sync = NULL;
508 ret = submit_frame(q, frame, &surf);
510 av_log(avctx, AV_LOG_ERROR, "Error submitting the frame for encoding.\n");
515 ret = av_new_packet(&new_pkt, q->packet_size);
517 av_log(avctx, AV_LOG_ERROR, "Error allocating the output packet\n");
521 bs = av_mallocz(sizeof(*bs));
523 av_packet_unref(&new_pkt);
524 return AVERROR(ENOMEM);
526 bs->Data = new_pkt.data;
527 bs->MaxLength = new_pkt.size;
530 ret = MFXVideoENCODE_EncodeFrameAsync(q->session, NULL, surf, bs, &sync);
531 if (ret == MFX_WRN_DEVICE_BUSY) {
539 av_packet_unref(&new_pkt);
541 if (ret == MFX_ERR_MORE_DATA)
543 av_log(avctx, AV_LOG_ERROR, "EncodeFrameAsync returned %d\n", ret);
544 return ff_qsv_error(ret);
547 if (ret == MFX_WRN_INCOMPATIBLE_VIDEO_PARAM) {
548 if (frame->interlaced_frame)
549 print_interlace_msg(avctx, q);
551 av_log(avctx, AV_LOG_WARNING,
552 "EncodeFrameAsync returned 'incompatible param' code\n");
555 av_fifo_generic_write(q->async_fifo, &new_pkt, sizeof(new_pkt), NULL);
556 av_fifo_generic_write(q->async_fifo, &sync, sizeof(sync), NULL);
557 av_fifo_generic_write(q->async_fifo, &bs, sizeof(bs), NULL);
559 av_packet_unref(&new_pkt);
563 if (!av_fifo_space(q->async_fifo) ||
564 (!frame && av_fifo_size(q->async_fifo))) {
565 av_fifo_generic_read(q->async_fifo, &new_pkt, sizeof(new_pkt), NULL);
566 av_fifo_generic_read(q->async_fifo, &sync, sizeof(sync), NULL);
567 av_fifo_generic_read(q->async_fifo, &bs, sizeof(bs), NULL);
569 MFXVideoCORE_SyncOperation(q->session, sync, 60000);
571 new_pkt.dts = av_rescale_q(bs->DecodeTimeStamp, (AVRational){1, 90000}, avctx->time_base);
572 new_pkt.pts = av_rescale_q(bs->TimeStamp, (AVRational){1, 90000}, avctx->time_base);
573 new_pkt.size = bs->DataLength;
575 if (bs->FrameType & MFX_FRAMETYPE_IDR ||
576 bs->FrameType & MFX_FRAMETYPE_xIDR)
577 new_pkt.flags |= AV_PKT_FLAG_KEY;
579 #if FF_API_CODED_FRAME
580 FF_DISABLE_DEPRECATION_WARNINGS
581 if (bs->FrameType & MFX_FRAMETYPE_I || bs->FrameType & MFX_FRAMETYPE_xI)
582 avctx->coded_frame->pict_type = AV_PICTURE_TYPE_I;
583 else if (bs->FrameType & MFX_FRAMETYPE_P || bs->FrameType & MFX_FRAMETYPE_xP)
584 avctx->coded_frame->pict_type = AV_PICTURE_TYPE_P;
585 else if (bs->FrameType & MFX_FRAMETYPE_B || bs->FrameType & MFX_FRAMETYPE_xB)
586 avctx->coded_frame->pict_type = AV_PICTURE_TYPE_B;
587 FF_ENABLE_DEPRECATION_WARNINGS
593 if (pkt->size < new_pkt.size) {
594 av_log(avctx, AV_LOG_ERROR, "Submitted buffer not large enough: %d < %d\n",
595 pkt->size, new_pkt.size);
596 av_packet_unref(&new_pkt);
597 return AVERROR(EINVAL);
600 memcpy(pkt->data, new_pkt.data, new_pkt.size);
601 pkt->size = new_pkt.size;
603 ret = av_packet_copy_props(pkt, &new_pkt);
604 av_packet_unref(&new_pkt);
616 int ff_qsv_enc_close(AVCodecContext *avctx, QSVEncContext *q)
621 MFXVideoENCODE_Close(q->session);
624 ff_qsv_close_internal_session(&q->internal_qs);
626 cur = q->work_frames;
628 q->work_frames = cur->next;
629 av_frame_free(&cur->frame);
631 cur = q->work_frames;
634 while (q->async_fifo && av_fifo_size(q->async_fifo)) {
639 av_fifo_generic_read(q->async_fifo, &pkt, sizeof(pkt), NULL);
640 av_fifo_generic_read(q->async_fifo, &sync, sizeof(sync), NULL);
641 av_fifo_generic_read(q->async_fifo, &bs, sizeof(bs), NULL);
644 av_packet_unref(&pkt);
646 av_fifo_free(q->async_fifo);
647 q->async_fifo = NULL;
649 av_freep(&q->opaque_surfaces);
650 av_buffer_unref(&q->opaque_alloc_buf);
652 av_freep(&q->extparam);