2 * Copyright (c) 2018 Sergey Lavrushkin
4 * This file is part of FFmpeg.
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 * DNN native backend implementation.
26 #include "dnn_backend_native.h"
27 #include "libavutil/avassert.h"
28 #include "dnn_backend_native_layer_conv2d.h"
29 #include "dnn_backend_native_layers.h"
30 #include "dnn_io_proc.h"
32 #define OFFSET(x) offsetof(NativeContext, x)
33 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM
34 static const AVOption dnn_native_options[] = {
35 { "conv2d_threads", "threads num for conv2d layer", OFFSET(options.conv2d_threads), AV_OPT_TYPE_INT, { .i64 = 0 }, INT_MIN, INT_MAX, FLAGS },
39 const AVClass dnn_native_class = {
40 .class_name = "dnn_native",
41 .item_name = av_default_item_name,
42 .option = dnn_native_options,
43 .version = LIBAVUTIL_VERSION_INT,
44 .category = AV_CLASS_CATEGORY_FILTER,
47 static DNNReturnType execute_model_native(const DNNModel *model, const char *input_name, AVFrame *in_frame,
48 const char **output_names, uint32_t nb_output, AVFrame *out_frame,
51 static DNNReturnType get_input_native(void *model, DNNData *input, const char *input_name)
53 NativeModel *native_model = (NativeModel *)model;
54 NativeContext *ctx = &native_model->ctx;
56 for (int i = 0; i < native_model->operands_num; ++i) {
57 DnnOperand *oprd = &native_model->operands[i];
58 if (strcmp(oprd->name, input_name) == 0) {
59 if (oprd->type != DOT_INPUT) {
60 av_log(ctx, AV_LOG_ERROR, "Found \"%s\" in model, but it is not input node\n", input_name);
63 input->dt = oprd->data_type;
64 av_assert0(oprd->dims[0] == 1);
65 input->height = oprd->dims[1];
66 input->width = oprd->dims[2];
67 input->channels = oprd->dims[3];
72 // do not find the input operand
73 av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name);
77 static DNNReturnType get_output_native(void *model, const char *input_name, int input_width, int input_height,
78 const char *output_name, int *output_width, int *output_height)
81 NativeModel *native_model = (NativeModel *)model;
82 AVFrame *in_frame = av_frame_alloc();
83 AVFrame *out_frame = av_frame_alloc();
84 in_frame->width = input_width;
85 in_frame->height = input_height;
87 ret = execute_model_native(native_model->model, input_name, in_frame, &output_name, 1, out_frame, 0);
88 *output_width = out_frame->width;
89 *output_height = out_frame->height;
91 av_frame_free(&out_frame);
92 av_frame_free(&in_frame);
96 // Loads model and its parameters that are stored in a binary file with following structure:
97 // layers_num,layer_type,layer_parameterss,layer_type,layer_parameters...
98 // For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases
99 // For DEPTH_TO_SPACE layer: block_size
100 DNNModel *ff_dnn_load_model_native(const char *model_filename, const char *options, void *userdata)
102 DNNModel *model = NULL;
103 char header_expected[] = "FFMPEGDNNNATIVE";
106 int version, header_size, major_version_expected = 1;
107 NativeModel *native_model = NULL;
108 AVIOContext *model_file_context;
109 int file_size, dnn_size, parsed_size;
111 DNNLayerType layer_type;
113 if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
116 file_size = avio_size(model_file_context);
118 model = av_mallocz(sizeof(DNNModel));
124 * check file header with string and version
126 size = sizeof(header_expected);
127 buf = av_malloc(size);
132 // size - 1 to skip the ending '\0' which is not saved in file
133 avio_get_str(model_file_context, size - 1, buf, size);
135 if (strncmp(buf, header_expected, size) != 0) {
141 version = (int32_t)avio_rl32(model_file_context);
143 if (version != major_version_expected) {
147 // currently no need to check minor version
148 version = (int32_t)avio_rl32(model_file_context);
150 header_size = dnn_size;
152 native_model = av_mallocz(sizeof(NativeModel));
157 native_model->ctx.class = &dnn_native_class;
158 model->options = options;
159 if (av_opt_set_from_string(&native_model->ctx, model->options, NULL, "=", "&") < 0)
161 model->model = (void *)native_model;
162 native_model->model = model;
164 #if !HAVE_PTHREAD_CANCEL
165 if (native_model->ctx.options.conv2d_threads > 1){
166 av_log(&native_model->ctx, AV_LOG_WARNING, "'conv2d_threads' option was set but it is not supported "
167 "on this build (pthread support is required)\n");
171 avio_seek(model_file_context, file_size - 8, SEEK_SET);
172 native_model->layers_num = (int32_t)avio_rl32(model_file_context);
173 native_model->operands_num = (int32_t)avio_rl32(model_file_context);
175 avio_seek(model_file_context, header_size, SEEK_SET);
177 native_model->layers = av_mallocz(native_model->layers_num * sizeof(Layer));
178 if (!native_model->layers){
182 native_model->operands = av_mallocz(native_model->operands_num * sizeof(DnnOperand));
183 if (!native_model->operands){
187 for (layer = 0; layer < native_model->layers_num; ++layer){
188 layer_type = (int32_t)avio_rl32(model_file_context);
191 if (layer_type >= DLT_COUNT) {
195 native_model->layers[layer].type = layer_type;
196 parsed_size = layer_funcs[layer_type].pf_load(&native_model->layers[layer], model_file_context, file_size, native_model->operands_num);
200 dnn_size += parsed_size;
203 for (int32_t i = 0; i < native_model->operands_num; ++i){
206 int32_t operand_index = (int32_t)avio_rl32(model_file_context);
209 if (operand_index >= native_model->operands_num) {
213 oprd = &native_model->operands[operand_index];
214 name_len = (int32_t)avio_rl32(model_file_context);
217 avio_get_str(model_file_context, name_len, oprd->name, sizeof(oprd->name));
218 dnn_size += name_len;
220 oprd->type = (int32_t)avio_rl32(model_file_context);
223 oprd->data_type = (int32_t)avio_rl32(model_file_context);
226 for (int32_t dim = 0; dim < 4; ++dim) {
227 oprd->dims[dim] = (int32_t)avio_rl32(model_file_context);
234 avio_closep(&model_file_context);
236 if (dnn_size != file_size){
237 ff_dnn_free_model_native(&model);
241 model->get_input = &get_input_native;
242 model->get_output = &get_output_native;
243 model->userdata = userdata;
248 ff_dnn_free_model_native(&model);
249 avio_closep(&model_file_context);
253 static DNNReturnType execute_model_native(const DNNModel *model, const char *input_name, AVFrame *in_frame,
254 const char **output_names, uint32_t nb_output, AVFrame *out_frame,
257 NativeModel *native_model = (NativeModel *)model->model;
258 NativeContext *ctx = &native_model->ctx;
260 DNNData input, output;
261 DnnOperand *oprd = NULL;
263 if (native_model->layers_num <= 0 || native_model->operands_num <= 0) {
264 av_log(ctx, AV_LOG_ERROR, "No operands or layers in model\n");
268 for (int i = 0; i < native_model->operands_num; ++i) {
269 oprd = &native_model->operands[i];
270 if (strcmp(oprd->name, input_name) == 0) {
271 if (oprd->type != DOT_INPUT) {
272 av_log(ctx, AV_LOG_ERROR, "Found \"%s\" in model, but it is not input node\n", input_name);
280 av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name);
284 oprd->dims[1] = in_frame->height;
285 oprd->dims[2] = in_frame->width;
287 av_freep(&oprd->data);
288 oprd->length = calculate_operand_data_length(oprd);
289 if (oprd->length <= 0) {
290 av_log(ctx, AV_LOG_ERROR, "The input data length overflow\n");
293 oprd->data = av_malloc(oprd->length);
295 av_log(ctx, AV_LOG_ERROR, "Failed to malloc memory for input data\n");
299 input.height = oprd->dims[1];
300 input.width = oprd->dims[2];
301 input.channels = oprd->dims[3];
302 input.data = oprd->data;
303 input.dt = oprd->data_type;
305 if (native_model->model->pre_proc != NULL) {
306 native_model->model->pre_proc(in_frame, &input, native_model->model->userdata);
308 proc_from_frame_to_dnn(in_frame, &input, ctx);
312 if (nb_output != 1) {
313 // currently, the filter does not need multiple outputs,
314 // so we just pending the support until we really need it.
315 av_log(ctx, AV_LOG_ERROR, "do not support multiple outputs\n");
319 for (layer = 0; layer < native_model->layers_num; ++layer){
320 DNNLayerType layer_type = native_model->layers[layer].type;
321 if (layer_funcs[layer_type].pf_exec(native_model->operands,
322 native_model->layers[layer].input_operand_indexes,
323 native_model->layers[layer].output_operand_index,
324 native_model->layers[layer].params,
325 &native_model->ctx) == DNN_ERROR) {
326 av_log(ctx, AV_LOG_ERROR, "Failed to execuet model\n");
331 for (uint32_t i = 0; i < nb_output; ++i) {
332 DnnOperand *oprd = NULL;
333 const char *output_name = output_names[i];
334 for (int j = 0; j < native_model->operands_num; ++j) {
335 if (strcmp(native_model->operands[j].name, output_name) == 0) {
336 oprd = &native_model->operands[j];
342 av_log(ctx, AV_LOG_ERROR, "Could not find output in model\n");
346 output.data = oprd->data;
347 output.height = oprd->dims[1];
348 output.width = oprd->dims[2];
349 output.channels = oprd->dims[3];
350 output.dt = oprd->data_type;
353 if (native_model->model->post_proc != NULL) {
354 native_model->model->post_proc(out_frame, &output, native_model->model->userdata);
356 proc_from_dnn_to_frame(out_frame, &output, ctx);
359 out_frame->width = output.width;
360 out_frame->height = output.height;
367 DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, const char *input_name, AVFrame *in_frame,
368 const char **output_names, uint32_t nb_output, AVFrame *out_frame)
370 NativeModel *native_model = (NativeModel *)model->model;
371 NativeContext *ctx = &native_model->ctx;
374 av_log(ctx, AV_LOG_ERROR, "in frame is NULL when execute model.\n");
379 av_log(ctx, AV_LOG_ERROR, "out frame is NULL when execute model.\n");
383 return execute_model_native(model, input_name, in_frame, output_names, nb_output, out_frame, 1);
386 int32_t calculate_operand_dims_count(const DnnOperand *oprd)
389 for (int i = 0; i < 4; ++i)
390 result *= oprd->dims[i];
395 int32_t calculate_operand_data_length(const DnnOperand* oprd)
397 // currently, we just support DNN_FLOAT
398 uint64_t len = sizeof(float);
399 for (int i = 0; i < 4; i++) {
400 len *= oprd->dims[i];
407 void ff_dnn_free_model_native(DNNModel **model)
409 NativeModel *native_model;
410 ConvolutionalParams *conv_params;
415 if ((*model)->model) {
416 native_model = (NativeModel *)(*model)->model;
417 if (native_model->layers) {
418 for (layer = 0; layer < native_model->layers_num; ++layer){
419 if (native_model->layers[layer].type == DLT_CONV2D){
420 conv_params = (ConvolutionalParams *)native_model->layers[layer].params;
421 av_freep(&conv_params->kernel);
422 av_freep(&conv_params->biases);
424 av_freep(&native_model->layers[layer].params);
426 av_freep(&native_model->layers);
429 if (native_model->operands) {
430 for (uint32_t operand = 0; operand < native_model->operands_num; ++operand)
431 av_freep(&native_model->operands[operand].data);
432 av_freep(&native_model->operands);
435 av_freep(&native_model);