2 * Copyright (c) 2019 Xuewei Meng
4 * This file is part of FFmpeg.
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 * Filter implementing image derain filter using deep convolutional networks.
24 * http://openaccess.thecvf.com/content_ECCV_2018/html/Xia_Li_Recurrent_Squeeze-and-Excitation_Context_ECCV_2018_paper.html
27 #include "libavformat/avio.h"
28 #include "libavutil/opt.h"
30 #include "dnn_interface.h"
34 typedef struct DRContext {
39 DNNBackendType backend_type;
40 DNNModule *dnn_module;
44 #define OFFSET(x) offsetof(DRContext, x)
45 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM
46 static const AVOption derain_options[] = {
47 { "filter_type", "filter type(derain/dehaze)", OFFSET(filter_type), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, FLAGS, "type" },
48 { "derain", "derain filter flag", 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, 0, 0, FLAGS, "type" },
49 { "dehaze", "dehaze filter flag", 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, 0, 0, FLAGS, "type" },
50 { "dnn_backend", "DNN backend", OFFSET(backend_type), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, FLAGS, "backend" },
51 { "native", "native backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, 0, 0, FLAGS, "backend" },
52 #if (CONFIG_LIBTENSORFLOW == 1)
53 { "tensorflow", "tensorflow backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, 0, 0, FLAGS, "backend" },
55 { "model", "path to model file", OFFSET(model_filename), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
59 AVFILTER_DEFINE_CLASS(derain);
61 static int query_formats(AVFilterContext *ctx)
63 AVFilterFormats *formats;
64 const enum AVPixelFormat pixel_fmts[] = {
69 formats = ff_make_format_list(pixel_fmts);
71 return ff_set_common_formats(ctx, formats);
74 static int filter_frame(AVFilterLink *inlink, AVFrame *in)
76 AVFilterContext *ctx = inlink->dst;
77 AVFilterLink *outlink = ctx->outputs[0];
78 DRContext *dr_context = ctx->priv;
79 DNNReturnType dnn_result;
80 const char *model_output_name = "y";
83 out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
85 av_log(ctx, AV_LOG_ERROR, "could not allocate memory for output frame\n");
87 return AVERROR(ENOMEM);
89 av_frame_copy_props(out, in);
91 dnn_result = (dr_context->dnn_module->execute_model)(dr_context->model, "x", in, &model_output_name, 1, out);
92 if (dnn_result != DNN_SUCCESS){
93 av_log(ctx, AV_LOG_ERROR, "failed to execute model\n");
100 return ff_filter_frame(outlink, out);
103 static av_cold int init(AVFilterContext *ctx)
105 DRContext *dr_context = ctx->priv;
107 dr_context->dnn_module = ff_get_dnn_module(dr_context->backend_type);
108 if (!dr_context->dnn_module) {
109 av_log(ctx, AV_LOG_ERROR, "could not create DNN module for requested backend\n");
110 return AVERROR(ENOMEM);
112 if (!dr_context->model_filename) {
113 av_log(ctx, AV_LOG_ERROR, "model file for network is not specified\n");
114 return AVERROR(EINVAL);
116 if (!dr_context->dnn_module->load_model) {
117 av_log(ctx, AV_LOG_ERROR, "load_model for network is not specified\n");
118 return AVERROR(EINVAL);
121 dr_context->model = (dr_context->dnn_module->load_model)(dr_context->model_filename, NULL, NULL);
122 if (!dr_context->model) {
123 av_log(ctx, AV_LOG_ERROR, "could not load DNN model\n");
124 return AVERROR(EINVAL);
130 static av_cold void uninit(AVFilterContext *ctx)
132 DRContext *dr_context = ctx->priv;
134 if (dr_context->dnn_module) {
135 (dr_context->dnn_module->free_model)(&dr_context->model);
136 av_freep(&dr_context->dnn_module);
140 static const AVFilterPad derain_inputs[] = {
143 .type = AVMEDIA_TYPE_VIDEO,
144 .filter_frame = filter_frame,
149 static const AVFilterPad derain_outputs[] = {
152 .type = AVMEDIA_TYPE_VIDEO,
157 AVFilter ff_vf_derain = {
159 .description = NULL_IF_CONFIG_SMALL("Apply derain filter to the input."),
160 .priv_size = sizeof(DRContext),
163 .query_formats = query_formats,
164 .inputs = derain_inputs,
165 .outputs = derain_outputs,
166 .priv_class = &derain_class,
167 .flags = AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC,