#ifndef AVFILTER_DNN_INTERFACE_H
#define AVFILTER_DNN_INTERFACE_H
+#include <stdint.h>
+#include "libavutil/frame.h"
+#include "avfilter.h"
+
typedef enum {DNN_SUCCESS, DNN_ERROR} DNNReturnType;
-typedef enum {DNN_NATIVE, DNN_TF} DNNBackendType;
+typedef enum {DNN_NATIVE, DNN_TF, DNN_OV} DNNBackendType;
+
+typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType;
+
+typedef enum {
+ DCO_NONE,
+ DCO_BGR,
+} DNNColorOrder;
+
+typedef enum {
+ DAST_FAIL, // something wrong
+ DAST_EMPTY_QUEUE, // no more inference result to get
+ DAST_NOT_READY, // all queued inferences are not finished
+ DAST_SUCCESS // got a result frame successfully
+} DNNAsyncStatusType;
+
+typedef enum {
+ DFT_NONE,
+ DFT_PROCESS_FRAME, // process the whole frame
+ DFT_ANALYTICS_DETECT, // detect from the whole frame
+ // we can add more such as detect_from_crop, classify_from_bbox, etc.
+}DNNFunctionType;
typedef struct DNNData{
- float *data;
+ void *data;
int width, height, channels;
+ // dt and order together decide the color format
+ DNNDataType dt;
+ DNNColorOrder order;
} DNNData;
+typedef int (*FramePrePostProc)(AVFrame *frame, DNNData *model, AVFilterContext *filter_ctx);
+typedef int (*DetectPostProc)(AVFrame *frame, DNNData *output, uint32_t nb, AVFilterContext *filter_ctx);
+
typedef struct DNNModel{
// Stores model that can be different for different backends.
void *model;
- // Sets model input and output, while allocating additional memory for intermediate calculations.
- // Should be called at least once before model execution.
- DNNReturnType (*set_input_output)(void *model, DNNData *input, DNNData *output);
+ // Stores options when the model is executed by the backend
+ const char *options;
+ // Stores FilterContext used for the interaction between AVFrame and DNNData
+ AVFilterContext *filter_ctx;
+ // Stores function type of the model
+ DNNFunctionType func_type;
+ // Gets model input information
+ // Just reuse struct DNNData here, actually the DNNData.data field is not needed.
+ DNNReturnType (*get_input)(void *model, DNNData *input, const char *input_name);
+ // Gets model output width/height with given input w/h
+ DNNReturnType (*get_output)(void *model, const char *input_name, int input_width, int input_height,
+ const char *output_name, int *output_width, int *output_height);
+ // set the pre process to transfer data from AVFrame to DNNData
+ // the default implementation within DNN is used if it is not provided by the filter
+ FramePrePostProc frame_pre_proc;
+ // set the post process to transfer data from DNNData to AVFrame
+ // the default implementation within DNN is used if it is not provided by the filter
+ FramePrePostProc frame_post_proc;
+ // set the post process to interpret detect result from DNNData
+ DetectPostProc detect_post_proc;
} DNNModel;
// Stores pointers to functions for loading, executing, freeing DNN models for one of the backends.
typedef struct DNNModule{
// Loads model and parameters from given file. Returns NULL if it is not possible.
- DNNModel *(*load_model)(const char *model_filename);
+ DNNModel *(*load_model)(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx);
// Executes model with specified input and output. Returns DNN_ERROR otherwise.
- DNNReturnType (*execute_model)(const DNNModel *model);
+ DNNReturnType (*execute_model)(const DNNModel *model, const char *input_name, AVFrame *in_frame,
+ const char **output_names, uint32_t nb_output, AVFrame *out_frame);
+ // Executes model with specified input and output asynchronously. Returns DNN_ERROR otherwise.
+ DNNReturnType (*execute_model_async)(const DNNModel *model, const char *input_name, AVFrame *in_frame,
+ const char **output_names, uint32_t nb_output, AVFrame *out_frame);
+ // Retrieve inference result.
+ DNNAsyncStatusType (*get_async_result)(const DNNModel *model, AVFrame **in, AVFrame **out);
+ // Flush all the pending tasks.
+ DNNReturnType (*flush)(const DNNModel *model);
// Frees memory allocated for model.
void (*free_model)(DNNModel **model);
} DNNModule;