2 * This file is part of FFmpeg.
4 * FFmpeg is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
9 * FFmpeg is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with FFmpeg; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 * deinterlace video filter - QSV
24 #include <mfx/mfxvideo.h>
29 #include "libavutil/avstring.h"
30 #include "libavutil/common.h"
31 #include "libavutil/hwcontext.h"
32 #include "libavutil/hwcontext_qsv.h"
33 #include "libavutil/internal.h"
34 #include "libavutil/mathematics.h"
35 #include "libavutil/opt.h"
36 #include "libavutil/pixdesc.h"
37 #include "libavutil/time.h"
38 #include "libavfilter/qsvvpp.h"
46 QSVDEINT_MORE_OUTPUT = 1,
50 typedef struct QSVFrame {
52 mfxFrameSurface1 surface;
55 struct QSVFrame *next;
58 typedef struct QSVDeintContext {
61 AVBufferRef *hw_frames_ctx;
62 /* a clone of the main session, used internally for deinterlacing */
68 mfxFrameSurface1 **surface_ptrs;
71 mfxExtOpaqueSurfaceAlloc opaque_alloc;
72 mfxExtVPPDeinterlacing deint_conf;
73 mfxExtBuffer *ext_buffers[2];
76 QSVFrame *work_frames;
82 /* option for Deinterlacing algorithm to be used */
86 static av_cold void qsvdeint_uninit(AVFilterContext *ctx)
88 QSVDeintContext *s = ctx->priv;
95 av_buffer_unref(&s->hw_frames_ctx);
99 s->work_frames = cur->next;
100 av_frame_free(&cur->frame);
102 cur = s->work_frames;
105 av_freep(&s->mem_ids);
108 av_freep(&s->surface_ptrs);
109 s->nb_surface_ptrs = 0;
112 static int qsvdeint_query_formats(AVFilterContext *ctx)
114 static const enum AVPixelFormat pixel_formats[] = {
115 AV_PIX_FMT_QSV, AV_PIX_FMT_NONE,
117 AVFilterFormats *pix_fmts = ff_make_format_list(pixel_formats);
120 if ((ret = ff_set_common_formats(ctx, pix_fmts)) < 0)
126 static mfxStatus frame_alloc(mfxHDL pthis, mfxFrameAllocRequest *req,
127 mfxFrameAllocResponse *resp)
129 AVFilterContext *ctx = pthis;
130 QSVDeintContext *s = ctx->priv;
132 if (!(req->Type & MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET) ||
133 !(req->Type & (MFX_MEMTYPE_FROM_VPPIN | MFX_MEMTYPE_FROM_VPPOUT)) ||
134 !(req->Type & MFX_MEMTYPE_EXTERNAL_FRAME))
135 return MFX_ERR_UNSUPPORTED;
137 resp->mids = s->mem_ids;
138 resp->NumFrameActual = s->nb_mem_ids;
143 static mfxStatus frame_free(mfxHDL pthis, mfxFrameAllocResponse *resp)
148 static mfxStatus frame_lock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr)
150 return MFX_ERR_UNSUPPORTED;
153 static mfxStatus frame_unlock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr)
155 return MFX_ERR_UNSUPPORTED;
158 static mfxStatus frame_get_hdl(mfxHDL pthis, mfxMemId mid, mfxHDL *hdl)
164 static const mfxHandleType handle_types[] = {
165 MFX_HANDLE_VA_DISPLAY,
166 MFX_HANDLE_D3D9_DEVICE_MANAGER,
167 MFX_HANDLE_D3D11_DEVICE,
170 static int init_out_session(AVFilterContext *ctx)
173 QSVDeintContext *s = ctx->priv;
174 AVHWFramesContext *hw_frames_ctx = (AVHWFramesContext*)s->hw_frames_ctx->data;
175 AVQSVFramesContext *hw_frames_hwctx = hw_frames_ctx->hwctx;
176 AVQSVDeviceContext *device_hwctx = hw_frames_ctx->device_ctx->hwctx;
178 int opaque = !!(hw_frames_hwctx->frame_type & MFX_MEMTYPE_OPAQUE_FRAME);
180 mfxHDL handle = NULL;
181 mfxHandleType handle_type;
188 /* extract the properties of the "master" session given to us */
189 err = MFXQueryIMPL(device_hwctx->session, &impl);
190 if (err == MFX_ERR_NONE)
191 err = MFXQueryVersion(device_hwctx->session, &ver);
192 if (err != MFX_ERR_NONE) {
193 av_log(ctx, AV_LOG_ERROR, "Error querying the session attributes\n");
194 return AVERROR_UNKNOWN;
197 for (i = 0; i < FF_ARRAY_ELEMS(handle_types); i++) {
198 err = MFXVideoCORE_GetHandle(device_hwctx->session, handle_types[i], &handle);
199 if (err == MFX_ERR_NONE) {
200 handle_type = handle_types[i];
206 return ff_qsvvpp_print_error(ctx, err, "Error getting the session handle");
208 ff_qsvvpp_print_warning(ctx, err, "Warning in getting the session handle");
209 return AVERROR_UNKNOWN;
212 /* create a "slave" session with those same properties, to be used for
213 * actual deinterlacing */
214 err = MFXInit(impl, &ver, &s->session);
216 return ff_qsvvpp_print_error(ctx, err, "Error initializing a session for deinterlacing");
218 ff_qsvvpp_print_warning(ctx, err, "Warning in session initialization");
219 return AVERROR_UNKNOWN;
223 err = MFXVideoCORE_SetHandle(s->session, handle_type, handle);
224 if (err != MFX_ERR_NONE)
225 return AVERROR_UNKNOWN;
228 if (QSV_RUNTIME_VERSION_ATLEAST(ver, 1, 25)) {
229 err = MFXJoinSession(device_hwctx->session, s->session);
230 if (err != MFX_ERR_NONE)
231 return AVERROR_UNKNOWN;
234 memset(&par, 0, sizeof(par));
236 s->deint_conf.Header.BufferId = MFX_EXTBUFF_VPP_DEINTERLACING;
237 s->deint_conf.Header.BufferSz = sizeof(s->deint_conf);
238 s->deint_conf.Mode = s->mode;
240 s->ext_buffers[s->num_ext_buffers++] = (mfxExtBuffer *)&s->deint_conf;
243 s->surface_ptrs = av_mallocz_array(hw_frames_hwctx->nb_surfaces,
244 sizeof(*s->surface_ptrs));
245 if (!s->surface_ptrs)
246 return AVERROR(ENOMEM);
247 for (i = 0; i < hw_frames_hwctx->nb_surfaces; i++)
248 s->surface_ptrs[i] = hw_frames_hwctx->surfaces + i;
249 s->nb_surface_ptrs = hw_frames_hwctx->nb_surfaces;
251 s->opaque_alloc.In.Surfaces = s->surface_ptrs;
252 s->opaque_alloc.In.NumSurface = s->nb_surface_ptrs;
253 s->opaque_alloc.In.Type = hw_frames_hwctx->frame_type;
255 s->opaque_alloc.Out = s->opaque_alloc.In;
257 s->opaque_alloc.Header.BufferId = MFX_EXTBUFF_OPAQUE_SURFACE_ALLOCATION;
258 s->opaque_alloc.Header.BufferSz = sizeof(s->opaque_alloc);
260 s->ext_buffers[s->num_ext_buffers++] = (mfxExtBuffer *)&s->opaque_alloc;
262 par.IOPattern = MFX_IOPATTERN_IN_OPAQUE_MEMORY | MFX_IOPATTERN_OUT_OPAQUE_MEMORY;
264 mfxFrameAllocator frame_allocator = {
266 .Alloc = frame_alloc,
268 .Unlock = frame_unlock,
269 .GetHDL = frame_get_hdl,
273 s->mem_ids = av_mallocz_array(hw_frames_hwctx->nb_surfaces,
274 sizeof(*s->mem_ids));
276 return AVERROR(ENOMEM);
277 for (i = 0; i < hw_frames_hwctx->nb_surfaces; i++)
278 s->mem_ids[i] = hw_frames_hwctx->surfaces[i].Data.MemId;
279 s->nb_mem_ids = hw_frames_hwctx->nb_surfaces;
281 err = MFXVideoCORE_SetFrameAllocator(s->session, &frame_allocator);
282 if (err != MFX_ERR_NONE)
283 return AVERROR_UNKNOWN;
285 par.IOPattern = MFX_IOPATTERN_IN_VIDEO_MEMORY | MFX_IOPATTERN_OUT_VIDEO_MEMORY;
288 par.ExtParam = s->ext_buffers;
289 par.NumExtParam = s->num_ext_buffers;
291 par.AsyncDepth = 1; // TODO async
293 par.vpp.In = hw_frames_hwctx->surfaces[0].Info;
295 par.vpp.In.CropW = ctx->inputs[0]->w;
296 par.vpp.In.CropH = ctx->inputs[0]->h;
298 if (ctx->inputs[0]->frame_rate.num) {
299 par.vpp.In.FrameRateExtN = ctx->inputs[0]->frame_rate.num;
300 par.vpp.In.FrameRateExtD = ctx->inputs[0]->frame_rate.den;
302 par.vpp.In.FrameRateExtN = ctx->inputs[0]->time_base.num;
303 par.vpp.In.FrameRateExtD = ctx->inputs[0]->time_base.den;
306 par.vpp.Out = par.vpp.In;
308 if (ctx->outputs[0]->frame_rate.num) {
309 par.vpp.Out.FrameRateExtN = ctx->outputs[0]->frame_rate.num;
310 par.vpp.Out.FrameRateExtD = ctx->outputs[0]->frame_rate.den;
312 par.vpp.Out.FrameRateExtN = ctx->outputs[0]->time_base.num;
313 par.vpp.Out.FrameRateExtD = ctx->outputs[0]->time_base.den;
316 /* Print input memory mode */
317 ff_qsvvpp_print_iopattern(ctx, par.IOPattern & 0x0F, "VPP");
318 /* Print output memory mode */
319 ff_qsvvpp_print_iopattern(ctx, par.IOPattern & 0xF0, "VPP");
320 err = MFXVideoVPP_Init(s->session, &par);
322 return ff_qsvvpp_print_error(ctx, err,
323 "Error opening the VPP for deinterlacing");
325 ff_qsvvpp_print_warning(ctx, err,
326 "Warning in VPP initialization");
327 return AVERROR_UNKNOWN;
333 static int qsvdeint_config_props(AVFilterLink *outlink)
335 AVFilterContext *ctx = outlink->src;
336 AVFilterLink *inlink = ctx->inputs[0];
337 QSVDeintContext *s = ctx->priv;
340 qsvdeint_uninit(ctx);
342 s->last_pts = AV_NOPTS_VALUE;
343 outlink->frame_rate = av_mul_q(inlink->frame_rate,
344 (AVRational){ 2, 1 });
345 outlink->time_base = av_mul_q(inlink->time_base,
346 (AVRational){ 1, 2 });
348 /* check that we have a hw context */
349 if (!inlink->hw_frames_ctx) {
350 av_log(ctx, AV_LOG_ERROR, "No hw context provided on input\n");
351 return AVERROR(EINVAL);
354 s->hw_frames_ctx = av_buffer_ref(inlink->hw_frames_ctx);
355 if (!s->hw_frames_ctx)
356 return AVERROR(ENOMEM);
358 av_buffer_unref(&outlink->hw_frames_ctx);
359 outlink->hw_frames_ctx = av_buffer_ref(inlink->hw_frames_ctx);
360 if (!outlink->hw_frames_ctx) {
361 qsvdeint_uninit(ctx);
362 return AVERROR(ENOMEM);
365 ret = init_out_session(ctx);
373 static void clear_unused_frames(QSVDeintContext *s)
375 QSVFrame *cur = s->work_frames;
377 if (!cur->surface.Data.Locked) {
378 av_frame_free(&cur->frame);
385 static int get_free_frame(QSVDeintContext *s, QSVFrame **f)
387 QSVFrame *frame, **last;
389 clear_unused_frames(s);
391 frame = s->work_frames;
392 last = &s->work_frames;
403 frame = av_mallocz(sizeof(*frame));
405 return AVERROR(ENOMEM);
412 static int submit_frame(AVFilterContext *ctx, AVFrame *frame,
413 mfxFrameSurface1 **surface)
415 QSVDeintContext *s = ctx->priv;
419 ret = get_free_frame(s, &qf);
425 qf->surface = *(mfxFrameSurface1*)qf->frame->data[3];
427 qf->surface.Data.Locked = 0;
428 qf->surface.Info.CropW = qf->frame->width;
429 qf->surface.Info.CropH = qf->frame->height;
431 qf->surface.Info.PicStruct = !qf->frame->interlaced_frame ? MFX_PICSTRUCT_PROGRESSIVE :
432 (qf->frame->top_field_first ? MFX_PICSTRUCT_FIELD_TFF :
433 MFX_PICSTRUCT_FIELD_BFF);
434 if (qf->frame->repeat_pict == 1) {
435 qf->surface.Info.PicStruct |= MFX_PICSTRUCT_FIELD_REPEATED;
436 qf->surface.Info.PicStruct |= qf->frame->top_field_first ? MFX_PICSTRUCT_FIELD_TFF :
437 MFX_PICSTRUCT_FIELD_BFF;
438 } else if (qf->frame->repeat_pict == 2)
439 qf->surface.Info.PicStruct |= MFX_PICSTRUCT_FRAME_DOUBLING;
440 else if (qf->frame->repeat_pict == 4)
441 qf->surface.Info.PicStruct |= MFX_PICSTRUCT_FRAME_TRIPLING;
443 if (ctx->inputs[0]->frame_rate.num) {
444 qf->surface.Info.FrameRateExtN = ctx->inputs[0]->frame_rate.num;
445 qf->surface.Info.FrameRateExtD = ctx->inputs[0]->frame_rate.den;
447 qf->surface.Info.FrameRateExtN = ctx->inputs[0]->time_base.num;
448 qf->surface.Info.FrameRateExtD = ctx->inputs[0]->time_base.den;
451 qf->surface.Data.TimeStamp = av_rescale_q(qf->frame->pts,
452 ctx->inputs[0]->time_base,
453 (AVRational){1, 90000});
455 *surface = &qf->surface;
461 static int process_frame(AVFilterContext *ctx, const AVFrame *in,
462 mfxFrameSurface1 *surf_in)
464 QSVDeintContext *s = ctx->priv;
465 AVFilterLink *inlink = ctx->inputs[0];
466 AVFilterLink *outlink = ctx->outputs[0];
469 mfxFrameSurface1 *surf_out;
470 mfxSyncPoint sync = NULL;
474 out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
476 ret = AVERROR(ENOMEM);
480 surf_out = (mfxFrameSurface1*)out->data[3];
481 surf_out->Info.CropW = outlink->w;
482 surf_out->Info.CropH = outlink->h;
483 surf_out->Info.PicStruct = MFX_PICSTRUCT_PROGRESSIVE;
486 err = MFXVideoVPP_RunFrameVPPAsync(s->session, surf_in, surf_out,
488 if (err == MFX_WRN_DEVICE_BUSY)
490 } while (err == MFX_WRN_DEVICE_BUSY);
492 if (err == MFX_ERR_MORE_DATA) {
494 return QSVDEINT_MORE_INPUT;
497 if (err < 0 && err != MFX_ERR_MORE_SURFACE) {
498 ret = ff_qsvvpp_print_error(ctx, err, "Error during deinterlacing");
503 av_log(ctx, AV_LOG_ERROR, "No sync during deinterlacing\n");
504 ret = AVERROR_UNKNOWN;
507 if (err == MFX_ERR_MORE_SURFACE)
511 err = MFXVideoCORE_SyncOperation(s->session, sync, 1000);
512 } while (err == MFX_WRN_IN_EXECUTION);
514 ret = ff_qsvvpp_print_error(ctx, err, "Error synchronizing the operation");
518 ret = av_frame_copy_props(out, in);
522 out->width = outlink->w;
523 out->height = outlink->h;
524 out->interlaced_frame = 0;
526 out->pts = av_rescale_q(out->pts, inlink->time_base, outlink->time_base);
527 if (out->pts == s->last_pts)
529 s->last_pts = out->pts;
531 ret = ff_filter_frame(outlink, out);
535 return again ? QSVDEINT_MORE_OUTPUT : 0;
541 static int qsvdeint_filter_frame(AVFilterLink *link, AVFrame *in)
543 AVFilterContext *ctx = link->dst;
545 mfxFrameSurface1 *surf_in;
548 ret = submit_frame(ctx, in, &surf_in);
555 ret = process_frame(ctx, in, surf_in);
558 } while (ret == QSVDEINT_MORE_OUTPUT);
563 static int qsvdeint_request_frame(AVFilterLink *outlink)
565 AVFilterContext *ctx = outlink->src;
567 return ff_request_frame(ctx->inputs[0]);
570 #define OFFSET(x) offsetof(QSVDeintContext, x)
571 #define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM
572 static const AVOption options[] = {
573 { "mode", "set deinterlace mode", OFFSET(mode), AV_OPT_TYPE_INT, {.i64 = MFX_DEINTERLACING_ADVANCED}, MFX_DEINTERLACING_BOB, MFX_DEINTERLACING_ADVANCED, FLAGS, "mode"},
574 { "bob", "bob algorithm", 0, AV_OPT_TYPE_CONST, {.i64 = MFX_DEINTERLACING_BOB}, MFX_DEINTERLACING_BOB, MFX_DEINTERLACING_ADVANCED, FLAGS, "mode"},
575 { "advanced", "Motion adaptive algorithm", 0, AV_OPT_TYPE_CONST, {.i64 = MFX_DEINTERLACING_ADVANCED}, MFX_DEINTERLACING_BOB, MFX_DEINTERLACING_ADVANCED, FLAGS, "mode"},
579 static const AVClass qsvdeint_class = {
580 .class_name = "deinterlace_qsv",
581 .item_name = av_default_item_name,
583 .version = LIBAVUTIL_VERSION_INT,
586 static const AVFilterPad qsvdeint_inputs[] = {
589 .type = AVMEDIA_TYPE_VIDEO,
590 .filter_frame = qsvdeint_filter_frame,
595 static const AVFilterPad qsvdeint_outputs[] = {
598 .type = AVMEDIA_TYPE_VIDEO,
599 .config_props = qsvdeint_config_props,
600 .request_frame = qsvdeint_request_frame,
605 AVFilter ff_vf_deinterlace_qsv = {
606 .name = "deinterlace_qsv",
607 .description = NULL_IF_CONFIG_SMALL("QuickSync video deinterlacing"),
609 .uninit = qsvdeint_uninit,
610 .query_formats = qsvdeint_query_formats,
612 .priv_size = sizeof(QSVDeintContext),
613 .priv_class = &qsvdeint_class,
615 .inputs = qsvdeint_inputs,
616 .outputs = qsvdeint_outputs,
618 .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,