#include "dnn_backend_native_layer_conv2d.h"
#include "dnn_backend_native_layer_depth2space.h"
#include "libavformat/avio.h"
+#include "libavformat/internal.h"
#include "libavutil/avassert.h"
+#include "../internal.h"
#include "dnn_backend_native_layer_pad.h"
#include "dnn_backend_native_layer_maximum.h"
#include "dnn_io_proc.h"
#include <tensorflow/c/c_api.h>
+typedef struct TFOptions{
+ char *sess_config;
+} TFOptions;
+
typedef struct TFContext {
const AVClass *class;
+ TFOptions options;
} TFContext;
typedef struct TFModel{
TF_Status *status;
} TFModel;
-static const AVClass dnn_tensorflow_class = {
- .class_name = "dnn_tensorflow",
- .item_name = av_default_item_name,
- .option = NULL,
- .version = LIBAVUTIL_VERSION_INT,
- .category = AV_CLASS_CATEGORY_FILTER,
+#define OFFSET(x) offsetof(TFContext, x)
+#define FLAGS AV_OPT_FLAG_FILTERING_PARAM
+static const AVOption dnn_tensorflow_options[] = {
+ { "sess_config", "config for SessionOptions", OFFSET(options.sess_config), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
+ { NULL }
};
+AVFILTER_DEFINE_CLASS(dnn_tensorflow);
+
static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_name, AVFrame *in_frame,
const char **output_names, uint32_t nb_output, AVFrame *out_frame,
int do_ioproc);
}
graph_buf = TF_NewBuffer();
- graph_buf->data = (void *)graph_data;
+ graph_buf->data = graph_data;
graph_buf->length = size;
graph_buf->data_deallocator = free_buffer;
static DNNReturnType get_input_tf(void *model, DNNData *input, const char *input_name)
{
- TFModel *tf_model = (TFModel *)model;
+ TFModel *tf_model = model;
TFContext *ctx = &tf_model->ctx;
TF_Status *status;
int64_t dims[4];
const char *output_name, int *output_width, int *output_height)
{
DNNReturnType ret;
- TFModel *tf_model = (TFModel *)model;
+ TFModel *tf_model = model;
TFContext *ctx = &tf_model->ctx;
AVFrame *in_frame = av_frame_alloc();
AVFrame *out_frame = NULL;
TF_ImportGraphDefOptions *graph_opts;
TF_SessionOptions *sess_opts;
const TF_Operation *init_op;
+ uint8_t *sess_config = NULL;
+ int sess_config_length = 0;
+
+ // prepare the sess config data
+ if (tf_model->ctx.options.sess_config != NULL) {
+ const char *config;
+ /*
+ tf_model->ctx.options.sess_config is hex to present the serialized proto
+ required by TF_SetConfig below, so we need to first generate the serialized
+ proto in a python script, tools/python/tf_sess_config.py is a script example
+ to generate the configs of sess_config.
+ */
+ if (strncmp(tf_model->ctx.options.sess_config, "0x", 2) != 0) {
+ av_log(ctx, AV_LOG_ERROR, "sess_config should start with '0x'\n");
+ return DNN_ERROR;
+ }
+ config = tf_model->ctx.options.sess_config + 2;
+ sess_config_length = ff_hex_to_data(NULL, config);
+
+ sess_config = av_mallocz(sess_config_length + AV_INPUT_BUFFER_PADDING_SIZE);
+ if (!sess_config) {
+ av_log(ctx, AV_LOG_ERROR, "failed to allocate memory\n");
+ return DNN_ERROR;
+ }
+ ff_hex_to_data(sess_config, config);
+ }
graph_def = read_graph(model_filename);
if (!graph_def){
av_log(ctx, AV_LOG_ERROR, "Failed to read model \"%s\" graph\n", model_filename);
+ av_freep(&sess_config);
return DNN_ERROR;
}
tf_model->graph = TF_NewGraph();
TF_DeleteGraph(tf_model->graph);
TF_DeleteStatus(tf_model->status);
av_log(ctx, AV_LOG_ERROR, "Failed to import serialized graph to model graph\n");
+ av_freep(&sess_config);
return DNN_ERROR;
}
init_op = TF_GraphOperationByName(tf_model->graph, "init");
sess_opts = TF_NewSessionOptions();
+
+ if (sess_config) {
+ TF_SetConfig(sess_opts, sess_config, sess_config_length,tf_model->status);
+ av_freep(&sess_config);
+ if (TF_GetCode(tf_model->status) != TF_OK) {
+ TF_DeleteGraph(tf_model->graph);
+ TF_DeleteStatus(tf_model->status);
+ TF_DeleteSessionOptions(sess_opts);
+ av_log(ctx, AV_LOG_ERROR, "Failed to set config for sess options with %s\n",
+ tf_model->ctx.options.sess_config);
+ return DNN_ERROR;
+ }
+ }
+
tf_model->session = TF_NewSession(tf_model->graph, sess_opts, tf_model->status);
TF_DeleteSessionOptions(sess_opts);
if (TF_GetCode(tf_model->status) != TF_OK)
{
+ TF_DeleteGraph(tf_model->graph);
+ TF_DeleteStatus(tf_model->status);
av_log(ctx, AV_LOG_ERROR, "Failed to create new session with model graph\n");
return DNN_ERROR;
}
&init_op, 1, NULL, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK)
{
+ TF_DeleteSession(tf_model->session, tf_model->status);
+ TF_DeleteGraph(tf_model->graph);
+ TF_DeleteStatus(tf_model->status);
av_log(ctx, AV_LOG_ERROR, "Failed to run session when initializing\n");
return DNN_ERROR;
}
TF_OperationDescription *op_desc;
TF_Output input;
int64_t strides[] = {1, 1, 1, 1};
- TF_Tensor *tensor;
+ TF_Tensor *kernel_tensor = NULL, *biases_tensor = NULL;
int64_t dims[4];
int dims_len;
char name_buffer[NAME_BUFFER_SIZE];
dims[2] = params->kernel_size;
dims[3] = params->input_num;
dims_len = 4;
- tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, size * sizeof(float));
- memcpy(TF_TensorData(tensor), params->kernel, size * sizeof(float));
- TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
+ kernel_tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, size * sizeof(float));
+ memcpy(TF_TensorData(kernel_tensor), params->kernel, size * sizeof(float));
+ TF_SetAttrTensor(op_desc, "value", kernel_tensor, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
- av_log(ctx, AV_LOG_ERROR, "Failed to set value for kernel of conv layer %d\n", layer);
- return DNN_ERROR;
+ goto err;
}
op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
- av_log(ctx, AV_LOG_ERROR, "Failed to add kernel to conv layer %d\n", layer);
- return DNN_ERROR;
+ goto err;
}
snprintf(name_buffer, NAME_BUFFER_SIZE, "transpose%d", layer);
TF_SetAttrType(op_desc, "Tperm", TF_INT32);
op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
- av_log(ctx, AV_LOG_ERROR, "Failed to add transpose to conv layer %d\n", layer);
- return DNN_ERROR;
+ goto err;
}
snprintf(name_buffer, NAME_BUFFER_SIZE, "conv2d%d", layer);
TF_SetAttrString(op_desc, "padding", "VALID", 5);
*cur_op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
- av_log(ctx, AV_LOG_ERROR, "Failed to add conv2d to conv layer %d\n", layer);
- return DNN_ERROR;
+ goto err;
}
snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_biases%d", layer);
TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
dims[0] = params->output_num;
dims_len = 1;
- tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, params->output_num * sizeof(float));
- memcpy(TF_TensorData(tensor), params->biases, params->output_num * sizeof(float));
- TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
+ biases_tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, params->output_num * sizeof(float));
+ memcpy(TF_TensorData(biases_tensor), params->biases, params->output_num * sizeof(float));
+ TF_SetAttrTensor(op_desc, "value", biases_tensor, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
- av_log(ctx, AV_LOG_ERROR, "Failed to set value for conv_biases of conv layer %d\n", layer);
- return DNN_ERROR;
+ goto err;
}
op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
- av_log(ctx, AV_LOG_ERROR, "Failed to add conv_biases to conv layer %d\n", layer);
- return DNN_ERROR;
+ goto err;
}
snprintf(name_buffer, NAME_BUFFER_SIZE, "bias_add%d", layer);
TF_SetAttrType(op_desc, "T", TF_FLOAT);
*cur_op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
- av_log(ctx, AV_LOG_ERROR, "Failed to add bias_add to conv layer %d\n", layer);
- return DNN_ERROR;
+ goto err;
}
snprintf(name_buffer, NAME_BUFFER_SIZE, "activation%d", layer);
op_desc = TF_NewOperation(tf_model->graph, "Sigmoid", name_buffer);
break;
default:
- av_log(ctx, AV_LOG_ERROR, "Unsupported convolutional activation function\n");
+ avpriv_report_missing_feature(ctx, "convolutional activation function %d", params->activation);
return DNN_ERROR;
}
input.oper = *cur_op;
TF_SetAttrType(op_desc, "T", TF_FLOAT);
*cur_op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
- av_log(ctx, AV_LOG_ERROR, "Failed to add activation function to conv layer %d\n", layer);
- return DNN_ERROR;
+ goto err;
}
return DNN_SUCCESS;
+err:
+ TF_DeleteTensor(kernel_tensor);
+ TF_DeleteTensor(biases_tensor);
+ av_log(ctx, AV_LOG_ERROR, "Failed to add conv layer %d\n", layer);
+ return DNN_ERROR;
}
static DNNReturnType add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op,
pads[7] = params->paddings[3][1];
TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
+ TF_DeleteTensor(tensor);
av_log(ctx, AV_LOG_ERROR, "Failed to set value for pad of layer %d\n", layer);
return DNN_ERROR;
}
op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
+ TF_DeleteTensor(tensor);
av_log(ctx, AV_LOG_ERROR, "Failed to add pad to layer %d\n", layer);
return DNN_ERROR;
}
TF_SetAttrString(op_desc, "mode", "SYMMETRIC", 9);
*cur_op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
+ TF_DeleteTensor(tensor);
av_log(ctx, AV_LOG_ERROR, "Failed to add mirror_pad to layer %d\n", layer);
return DNN_ERROR;
}
*y = params->val.y;
TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
+ TF_DeleteTensor(tensor);
av_log(ctx, AV_LOG_ERROR, "Failed to set value for maximum/y of layer %d", layer);
return DNN_ERROR;
}
op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
+ TF_DeleteTensor(tensor);
av_log(ctx, AV_LOG_ERROR, "Failed to add maximum/y to layer %d\n", layer);
return DNN_ERROR;
}
TF_SetAttrType(op_desc, "T", TF_FLOAT);
*cur_op = TF_FinishOperation(op_desc, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK){
+ TF_DeleteTensor(tensor);
av_log(ctx, AV_LOG_ERROR, "Failed to add maximum to layer %d\n", layer);
return DNN_ERROR;
}
TF_OperationDescription *op_desc;
TF_Operation *op;
TF_Operation *transpose_op;
- TF_Tensor *tensor;
+ TF_Tensor *tensor = NULL;
TF_Output input;
int32_t *transpose_perm;
int64_t transpose_perm_shape[] = {4};
DNNModel *model = NULL;
NativeModel *native_model;
- model = ff_dnn_load_model_native(model_filename, NULL, NULL);
+ model = ff_dnn_load_model_native(model_filename, DFT_PROCESS_FRAME, NULL, NULL);
if (!model){
av_log(ctx, AV_LOG_ERROR, "Failed to load native model\n");
return DNN_ERROR;
}
- native_model = (NativeModel *)model->model;
+ native_model = model->model;
tf_model->graph = TF_NewGraph();
tf_model->status = TF_NewStatus();
#define CLEANUP_ON_ERROR(tf_model) \
{ \
+ TF_DeleteTensor(tensor); \
TF_DeleteGraph(tf_model->graph); \
TF_DeleteStatus(tf_model->status); \
av_log(ctx, AV_LOG_ERROR, "Failed to set value or add operator to layer\n"); \
CLEANUP_ON_ERROR(tf_model);
}
transpose_op = TF_FinishOperation(op_desc, tf_model->status);
+ if (TF_GetCode(tf_model->status) != TF_OK){
+ CLEANUP_ON_ERROR(tf_model);
+ }
for (layer = 0; layer < native_model->layers_num; ++layer){
switch (native_model->layers[layer].type){
return DNN_SUCCESS;
}
-DNNModel *ff_dnn_load_model_tf(const char *model_filename, const char *options, void *userdata)
+DNNModel *ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
{
DNNModel *model = NULL;
TFModel *tf_model = NULL;
tf_model->ctx.class = &dnn_tensorflow_class;
tf_model->model = model;
+ //parse options
+ av_opt_set_defaults(&tf_model->ctx);
+ if (av_opt_set_from_string(&tf_model->ctx, options, NULL, "=", "&") < 0) {
+ av_log(&tf_model->ctx, AV_LOG_ERROR, "Failed to parse options \"%s\"\n", options);
+ av_freep(&tf_model);
+ av_freep(&model);
+ return NULL;
+ }
+
if (load_tf_model(tf_model, model_filename) != DNN_SUCCESS){
if (load_native_model(tf_model, model_filename) != DNN_SUCCESS){
av_freep(&tf_model);
}
}
- model->model = (void *)tf_model;
+ model->model = tf_model;
model->get_input = &get_input_tf;
model->get_output = &get_output_tf;
model->options = options;
- model->userdata = userdata;
+ model->filter_ctx = filter_ctx;
+ model->func_type = func_type;
return model;
}
int do_ioproc)
{
TF_Output *tf_outputs;
- TFModel *tf_model = (TFModel *)model->model;
+ TFModel *tf_model = model->model;
TFContext *ctx = &tf_model->ctx;
DNNData input, output;
TF_Tensor **output_tensors;
input.data = (float *)TF_TensorData(input_tensor);
if (do_ioproc) {
- if (tf_model->model->pre_proc != NULL) {
- tf_model->model->pre_proc(in_frame, &input, tf_model->model->userdata);
+ if (tf_model->model->frame_pre_proc != NULL) {
+ tf_model->model->frame_pre_proc(in_frame, &input, tf_model->model->filter_ctx);
} else {
- proc_from_frame_to_dnn(in_frame, &input, ctx);
+ ff_proc_from_frame_to_dnn(in_frame, &input, tf_model->model->func_type, ctx);
}
}
if (nb_output != 1) {
// currently, the filter does not need multiple outputs,
// so we just pending the support until we really need it.
- av_log(ctx, AV_LOG_ERROR, "do not support multiple outputs\n");
+ TF_DeleteTensor(input_tensor);
+ avpriv_report_missing_feature(ctx, "multiple outputs");
return DNN_ERROR;
}
tf_outputs = av_malloc_array(nb_output, sizeof(*tf_outputs));
if (tf_outputs == NULL) {
+ TF_DeleteTensor(input_tensor);
av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for *tf_outputs\n"); \
return DNN_ERROR;
}
output_tensors = av_mallocz_array(nb_output, sizeof(*output_tensors));
if (!output_tensors) {
+ TF_DeleteTensor(input_tensor);
av_freep(&tf_outputs);
av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for output tensor\n"); \
return DNN_ERROR;
for (int i = 0; i < nb_output; ++i) {
tf_outputs[i].oper = TF_GraphOperationByName(tf_model->graph, output_names[i]);
if (!tf_outputs[i].oper) {
+ TF_DeleteTensor(input_tensor);
av_freep(&tf_outputs);
av_freep(&output_tensors);
av_log(ctx, AV_LOG_ERROR, "Could not find output \"%s\" in model\n", output_names[i]); \
tf_outputs, output_tensors, nb_output,
NULL, 0, NULL, tf_model->status);
if (TF_GetCode(tf_model->status) != TF_OK) {
+ TF_DeleteTensor(input_tensor);
av_freep(&tf_outputs);
av_freep(&output_tensors);
av_log(ctx, AV_LOG_ERROR, "Failed to run session when executing model\n");
output.dt = TF_TensorType(output_tensors[i]);
if (do_ioproc) {
- if (tf_model->model->post_proc != NULL) {
- tf_model->model->post_proc(out_frame, &output, tf_model->model->userdata);
+ if (tf_model->model->frame_post_proc != NULL) {
+ tf_model->model->frame_post_proc(out_frame, &output, tf_model->model->filter_ctx);
} else {
- proc_from_dnn_to_frame(out_frame, &output, ctx);
+ ff_proc_from_dnn_to_frame(out_frame, &output, ctx);
}
} else {
out_frame->width = output.width;
DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, const char *input_name, AVFrame *in_frame,
const char **output_names, uint32_t nb_output, AVFrame *out_frame)
{
- TFModel *tf_model = (TFModel *)model->model;
+ TFModel *tf_model = model->model;
TFContext *ctx = &tf_model->ctx;
if (!in_frame) {
TFModel *tf_model;
if (*model){
- tf_model = (TFModel *)(*model)->model;
+ tf_model = (*model)->model;
if (tf_model->graph){
TF_DeleteGraph(tf_model->graph);
}
projects / ffmpeg / blobdiff