2 * Copyright (c) 2018 Sergey Lavrushkin
4 * This file is part of FFmpeg.
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 * DNN native backend implementation.
26 #include "dnn_backend_native.h"
27 #include "libavutil/avassert.h"
28 #include "dnn_backend_native_layer_conv2d.h"
29 #include "dnn_backend_native_layers.h"
30 #include "dnn_io_proc.h"
32 #define OFFSET(x) offsetof(NativeContext, x)
33 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM
34 static const AVOption dnn_native_options[] = {
35 { "conv2d_threads", "threads num for conv2d layer", OFFSET(options.conv2d_threads), AV_OPT_TYPE_INT, { .i64 = 0 }, INT_MIN, INT_MAX, FLAGS },
39 static const AVClass dnn_native_class = {
40 .class_name = "dnn_native",
41 .item_name = av_default_item_name,
42 .option = dnn_native_options,
43 .version = LIBAVUTIL_VERSION_INT,
44 .category = AV_CLASS_CATEGORY_FILTER,
47 static DNNReturnType execute_model_native(const DNNModel *model, const char *input_name, AVFrame *in_frame,
48 const char **output_names, uint32_t nb_output, AVFrame *out_frame,
51 static DNNReturnType get_input_native(void *model, DNNData *input, const char *input_name)
53 NativeModel *native_model = model;
54 NativeContext *ctx = &native_model->ctx;
56 for (int i = 0; i < native_model->operands_num; ++i) {
57 DnnOperand *oprd = &native_model->operands[i];
58 if (strcmp(oprd->name, input_name) == 0) {
59 if (oprd->type != DOT_INPUT) {
60 av_log(ctx, AV_LOG_ERROR, "Found \"%s\" in model, but it is not input node\n", input_name);
63 input->dt = oprd->data_type;
64 av_assert0(oprd->dims[0] == 1);
65 input->height = oprd->dims[1];
66 input->width = oprd->dims[2];
67 input->channels = oprd->dims[3];
72 // do not find the input operand
73 av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name);
77 static DNNReturnType get_output_native(void *model, const char *input_name, int input_width, int input_height,
78 const char *output_name, int *output_width, int *output_height)
81 NativeModel *native_model = model;
82 NativeContext *ctx = &native_model->ctx;
83 AVFrame *in_frame = av_frame_alloc();
84 AVFrame *out_frame = NULL;
87 av_log(ctx, AV_LOG_ERROR, "Could not allocate memory for input frame\n");
91 out_frame = av_frame_alloc();
94 av_log(ctx, AV_LOG_ERROR, "Could not allocate memory for output frame\n");
95 av_frame_free(&in_frame);
99 in_frame->width = input_width;
100 in_frame->height = input_height;
102 ret = execute_model_native(native_model->model, input_name, in_frame, &output_name, 1, out_frame, 0);
103 *output_width = out_frame->width;
104 *output_height = out_frame->height;
106 av_frame_free(&out_frame);
107 av_frame_free(&in_frame);
111 // Loads model and its parameters that are stored in a binary file with following structure:
112 // layers_num,layer_type,layer_parameterss,layer_type,layer_parameters...
113 // For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases
114 // For DEPTH_TO_SPACE layer: block_size
115 DNNModel *ff_dnn_load_model_native(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
117 DNNModel *model = NULL;
118 char header_expected[] = "FFMPEGDNNNATIVE";
121 int version, header_size, major_version_expected = 1;
122 NativeModel *native_model = NULL;
123 AVIOContext *model_file_context;
124 int file_size, dnn_size, parsed_size;
126 DNNLayerType layer_type;
128 if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
131 file_size = avio_size(model_file_context);
133 model = av_mallocz(sizeof(DNNModel));
139 * check file header with string and version
141 size = sizeof(header_expected);
142 buf = av_malloc(size);
147 // size - 1 to skip the ending '\0' which is not saved in file
148 avio_get_str(model_file_context, size - 1, buf, size);
150 if (strncmp(buf, header_expected, size) != 0) {
156 version = (int32_t)avio_rl32(model_file_context);
158 if (version != major_version_expected) {
162 // currently no need to check minor version
163 version = (int32_t)avio_rl32(model_file_context);
165 header_size = dnn_size;
167 native_model = av_mallocz(sizeof(NativeModel));
172 native_model->ctx.class = &dnn_native_class;
173 model->options = options;
174 if (av_opt_set_from_string(&native_model->ctx, model->options, NULL, "=", "&") < 0)
176 model->model = (void *)native_model;
177 native_model->model = model;
179 #if !HAVE_PTHREAD_CANCEL
180 if (native_model->ctx.options.conv2d_threads > 1){
181 av_log(&native_model->ctx, AV_LOG_WARNING, "'conv2d_threads' option was set but it is not supported "
182 "on this build (pthread support is required)\n");
186 avio_seek(model_file_context, file_size - 8, SEEK_SET);
187 native_model->layers_num = (int32_t)avio_rl32(model_file_context);
188 native_model->operands_num = (int32_t)avio_rl32(model_file_context);
190 avio_seek(model_file_context, header_size, SEEK_SET);
192 native_model->layers = av_mallocz(native_model->layers_num * sizeof(Layer));
193 if (!native_model->layers){
197 native_model->operands = av_mallocz(native_model->operands_num * sizeof(DnnOperand));
198 if (!native_model->operands){
202 for (layer = 0; layer < native_model->layers_num; ++layer){
203 layer_type = (int32_t)avio_rl32(model_file_context);
206 if (layer_type >= DLT_COUNT) {
210 native_model->layers[layer].type = layer_type;
211 parsed_size = ff_layer_funcs[layer_type].pf_load(&native_model->layers[layer], model_file_context, file_size, native_model->operands_num);
215 dnn_size += parsed_size;
218 for (int32_t i = 0; i < native_model->operands_num; ++i){
221 int32_t operand_index = (int32_t)avio_rl32(model_file_context);
224 if (operand_index >= native_model->operands_num) {
228 oprd = &native_model->operands[operand_index];
229 name_len = (int32_t)avio_rl32(model_file_context);
232 avio_get_str(model_file_context, name_len, oprd->name, sizeof(oprd->name));
233 dnn_size += name_len;
235 oprd->type = (int32_t)avio_rl32(model_file_context);
238 oprd->data_type = (int32_t)avio_rl32(model_file_context);
241 for (int32_t dim = 0; dim < 4; ++dim) {
242 oprd->dims[dim] = (int32_t)avio_rl32(model_file_context);
249 avio_closep(&model_file_context);
251 if (dnn_size != file_size){
252 ff_dnn_free_model_native(&model);
256 model->get_input = &get_input_native;
257 model->get_output = &get_output_native;
258 model->filter_ctx = filter_ctx;
259 model->func_type = func_type;
264 ff_dnn_free_model_native(&model);
265 avio_closep(&model_file_context);
269 static DNNReturnType execute_model_native(const DNNModel *model, const char *input_name, AVFrame *in_frame,
270 const char **output_names, uint32_t nb_output, AVFrame *out_frame,
273 NativeModel *native_model = model->model;
274 NativeContext *ctx = &native_model->ctx;
276 DNNData input, output;
277 DnnOperand *oprd = NULL;
279 if (native_model->layers_num <= 0 || native_model->operands_num <= 0) {
280 av_log(ctx, AV_LOG_ERROR, "No operands or layers in model\n");
284 for (int i = 0; i < native_model->operands_num; ++i) {
285 oprd = &native_model->operands[i];
286 if (strcmp(oprd->name, input_name) == 0) {
287 if (oprd->type != DOT_INPUT) {
288 av_log(ctx, AV_LOG_ERROR, "Found \"%s\" in model, but it is not input node\n", input_name);
296 av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name);
300 oprd->dims[1] = in_frame->height;
301 oprd->dims[2] = in_frame->width;
303 av_freep(&oprd->data);
304 oprd->length = ff_calculate_operand_data_length(oprd);
305 if (oprd->length <= 0) {
306 av_log(ctx, AV_LOG_ERROR, "The input data length overflow\n");
309 oprd->data = av_malloc(oprd->length);
311 av_log(ctx, AV_LOG_ERROR, "Failed to malloc memory for input data\n");
315 input.height = oprd->dims[1];
316 input.width = oprd->dims[2];
317 input.channels = oprd->dims[3];
318 input.data = oprd->data;
319 input.dt = oprd->data_type;
321 if (native_model->model->pre_proc != NULL) {
322 native_model->model->pre_proc(in_frame, &input, native_model->model->filter_ctx);
324 ff_proc_from_frame_to_dnn(in_frame, &input, ctx);
328 if (nb_output != 1) {
329 // currently, the filter does not need multiple outputs,
330 // so we just pending the support until we really need it.
331 avpriv_report_missing_feature(ctx, "multiple outputs");
335 for (layer = 0; layer < native_model->layers_num; ++layer){
336 DNNLayerType layer_type = native_model->layers[layer].type;
337 if (ff_layer_funcs[layer_type].pf_exec(native_model->operands,
338 native_model->layers[layer].input_operand_indexes,
339 native_model->layers[layer].output_operand_index,
340 native_model->layers[layer].params,
341 &native_model->ctx) == DNN_ERROR) {
342 av_log(ctx, AV_LOG_ERROR, "Failed to execuet model\n");
347 for (uint32_t i = 0; i < nb_output; ++i) {
348 DnnOperand *oprd = NULL;
349 const char *output_name = output_names[i];
350 for (int j = 0; j < native_model->operands_num; ++j) {
351 if (strcmp(native_model->operands[j].name, output_name) == 0) {
352 oprd = &native_model->operands[j];
358 av_log(ctx, AV_LOG_ERROR, "Could not find output in model\n");
362 output.data = oprd->data;
363 output.height = oprd->dims[1];
364 output.width = oprd->dims[2];
365 output.channels = oprd->dims[3];
366 output.dt = oprd->data_type;
369 if (native_model->model->post_proc != NULL) {
370 native_model->model->post_proc(out_frame, &output, native_model->model->filter_ctx);
372 ff_proc_from_dnn_to_frame(out_frame, &output, ctx);
375 out_frame->width = output.width;
376 out_frame->height = output.height;
383 DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, const char *input_name, AVFrame *in_frame,
384 const char **output_names, uint32_t nb_output, AVFrame *out_frame)
386 NativeModel *native_model = model->model;
387 NativeContext *ctx = &native_model->ctx;
390 av_log(ctx, AV_LOG_ERROR, "in frame is NULL when execute model.\n");
395 av_log(ctx, AV_LOG_ERROR, "out frame is NULL when execute model.\n");
399 return execute_model_native(model, input_name, in_frame, output_names, nb_output, out_frame, 1);
402 int32_t ff_calculate_operand_dims_count(const DnnOperand *oprd)
405 for (int i = 0; i < 4; ++i)
406 result *= oprd->dims[i];
411 int32_t ff_calculate_operand_data_length(const DnnOperand* oprd)
413 // currently, we just support DNN_FLOAT
414 uint64_t len = sizeof(float);
415 for (int i = 0; i < 4; i++) {
416 len *= oprd->dims[i];
423 void ff_dnn_free_model_native(DNNModel **model)
425 NativeModel *native_model;
426 ConvolutionalParams *conv_params;
431 if ((*model)->model) {
432 native_model = (*model)->model;
433 if (native_model->layers) {
434 for (layer = 0; layer < native_model->layers_num; ++layer){
435 if (native_model->layers[layer].type == DLT_CONV2D){
436 conv_params = (ConvolutionalParams *)native_model->layers[layer].params;
437 av_freep(&conv_params->kernel);
438 av_freep(&conv_params->biases);
440 av_freep(&native_model->layers[layer].params);
442 av_freep(&native_model->layers);
445 if (native_model->operands) {
446 for (uint32_t operand = 0; operand < native_model->operands_num; ++operand)
447 av_freep(&native_model->operands[operand].data);
448 av_freep(&native_model->operands);
451 av_freep(&native_model);