2 * Copyright (c) 2018 Sergey Lavrushkin
4 * This file is part of FFmpeg.
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 * DNN inference engine interface.
26 #ifndef AVFILTER_DNN_INTERFACE_H
27 #define AVFILTER_DNN_INTERFACE_H
30 #include "libavutil/frame.h"
32 typedef enum {DNN_SUCCESS, DNN_ERROR} DNNReturnType;
34 typedef enum {DNN_NATIVE, DNN_TF, DNN_OV} DNNBackendType;
36 typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType;
38 typedef struct DNNData{
41 int width, height, channels;
44 typedef struct DNNModel{
45 // Stores model that can be different for different backends.
47 // Stores options when the model is executed by the backend
49 // Stores userdata used for the interaction between AVFrame and DNNData
51 // Gets model input information
52 // Just reuse struct DNNData here, actually the DNNData.data field is not needed.
53 DNNReturnType (*get_input)(void *model, DNNData *input, const char *input_name);
54 // Gets model output width/height with given input w/h
55 DNNReturnType (*get_output)(void *model, const char *input_name, int input_width, int input_height,
56 const char *output_name, int *output_width, int *output_height);
57 // set the pre process to transfer data from AVFrame to DNNData
58 // the default implementation within DNN is used if it is not provided by the filter
59 int (*pre_proc)(AVFrame *frame_in, DNNData *model_input, void *user_data);
60 // set the post process to transfer data from DNNData to AVFrame
61 // the default implementation within DNN is used if it is not provided by the filter
62 int (*post_proc)(AVFrame *frame_out, DNNData *model_output, void *user_data);
65 // Stores pointers to functions for loading, executing, freeing DNN models for one of the backends.
66 typedef struct DNNModule{
67 // Loads model and parameters from given file. Returns NULL if it is not possible.
68 DNNModel *(*load_model)(const char *model_filename, const char *options, void *userdata);
69 // Executes model with specified input and output. Returns DNN_ERROR otherwise.
70 DNNReturnType (*execute_model)(const DNNModel *model, const char *input_name, AVFrame *in_frame,
71 const char **output_names, uint32_t nb_output, AVFrame *out_frame);
72 // Frees memory allocated for model.
73 void (*free_model)(DNNModel **model);
76 // Initializes DNNModule depending on chosen backend.
77 DNNModule *ff_get_dnn_module(DNNBackendType backend_type);