2 * Copyright (c) 2018 Sergey Lavrushkin
4 * This file is part of FFmpeg.
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 * DNN native backend implementation.
26 #include "dnn_backend_native.h"
27 #include "libavutil/avassert.h"
28 #include "dnn_backend_native_layer_conv2d.h"
29 #include "dnn_backend_native_layers.h"
30 #include "dnn_io_proc.h"
32 #define OFFSET(x) offsetof(NativeContext, x)
33 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM
34 static const AVOption dnn_native_options[] = {
35 { "conv2d_threads", "threads num for conv2d layer", OFFSET(options.conv2d_threads), AV_OPT_TYPE_INT, { .i64 = 0 }, INT_MIN, INT_MAX, FLAGS },
39 static const AVClass dnn_native_class = {
40 .class_name = "dnn_native",
41 .item_name = av_default_item_name,
42 .option = dnn_native_options,
43 .version = LIBAVUTIL_VERSION_INT,
44 .category = AV_CLASS_CATEGORY_FILTER,
47 static DNNReturnType execute_model_native(const DNNModel *model, const char *input_name, AVFrame *in_frame,
48 const char **output_names, uint32_t nb_output, AVFrame *out_frame,
51 static DNNReturnType get_input_native(void *model, DNNData *input, const char *input_name)
53 NativeModel *native_model = model;
54 NativeContext *ctx = &native_model->ctx;
56 for (int i = 0; i < native_model->operands_num; ++i) {
57 DnnOperand *oprd = &native_model->operands[i];
58 if (strcmp(oprd->name, input_name) == 0) {
59 if (oprd->type != DOT_INPUT) {
60 av_log(ctx, AV_LOG_ERROR, "Found \"%s\" in model, but it is not input node\n", input_name);
63 input->dt = oprd->data_type;
64 av_assert0(oprd->dims[0] == 1);
65 input->height = oprd->dims[1];
66 input->width = oprd->dims[2];
67 input->channels = oprd->dims[3];
72 // do not find the input operand
73 av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name);
77 static DNNReturnType get_output_native(void *model, const char *input_name, int input_width, int input_height,
78 const char *output_name, int *output_width, int *output_height)
81 NativeModel *native_model = model;
82 NativeContext *ctx = &native_model->ctx;
83 AVFrame *in_frame = av_frame_alloc();
84 AVFrame *out_frame = NULL;
87 av_log(ctx, AV_LOG_ERROR, "Could not allocate memory for input frame\n");
91 out_frame = av_frame_alloc();
94 av_log(ctx, AV_LOG_ERROR, "Could not allocate memory for output frame\n");
95 av_frame_free(&in_frame);
99 in_frame->width = input_width;
100 in_frame->height = input_height;
102 ret = execute_model_native(native_model->model, input_name, in_frame, &output_name, 1, out_frame, 0);
103 *output_width = out_frame->width;
104 *output_height = out_frame->height;
106 av_frame_free(&out_frame);
107 av_frame_free(&in_frame);
111 // Loads model and its parameters that are stored in a binary file with following structure:
112 // layers_num,layer_type,layer_parameterss,layer_type,layer_parameters...
113 // For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases
114 // For DEPTH_TO_SPACE layer: block_size
115 DNNModel *ff_dnn_load_model_native(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
117 #define DNN_NATIVE_MAGIC "FFMPEGDNNNATIVE"
118 DNNModel *model = NULL;
119 // sizeof - 1 to skip the terminating '\0' which is not written in the file
120 char buf[sizeof(DNN_NATIVE_MAGIC) - 1];
121 int version, header_size, major_version_expected = 1;
122 NativeModel *native_model = NULL;
123 AVIOContext *model_file_context;
124 int file_size, dnn_size, parsed_size;
126 DNNLayerType layer_type;
128 if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
131 file_size = avio_size(model_file_context);
133 model = av_mallocz(sizeof(DNNModel));
139 * check file header with string and version
141 if (avio_read(model_file_context, buf, sizeof(buf)) != sizeof(buf) ||
142 memcmp(buf, DNN_NATIVE_MAGIC, sizeof(buf)))
144 dnn_size = sizeof(buf);
146 version = (int32_t)avio_rl32(model_file_context);
148 if (version != major_version_expected) {
152 // currently no need to check minor version
153 version = (int32_t)avio_rl32(model_file_context);
155 header_size = dnn_size;
157 native_model = av_mallocz(sizeof(NativeModel));
161 model->model = native_model;
163 native_model->ctx.class = &dnn_native_class;
164 model->options = options;
165 if (av_opt_set_from_string(&native_model->ctx, model->options, NULL, "=", "&") < 0)
167 native_model->model = model;
169 #if !HAVE_PTHREAD_CANCEL
170 if (native_model->ctx.options.conv2d_threads > 1){
171 av_log(&native_model->ctx, AV_LOG_WARNING, "'conv2d_threads' option was set but it is not supported "
172 "on this build (pthread support is required)\n");
176 avio_seek(model_file_context, file_size - 8, SEEK_SET);
177 native_model->layers_num = (int32_t)avio_rl32(model_file_context);
178 native_model->operands_num = (int32_t)avio_rl32(model_file_context);
180 avio_seek(model_file_context, header_size, SEEK_SET);
182 native_model->layers = av_mallocz(native_model->layers_num * sizeof(Layer));
183 if (!native_model->layers){
187 native_model->operands = av_mallocz(native_model->operands_num * sizeof(DnnOperand));
188 if (!native_model->operands){
192 for (layer = 0; layer < native_model->layers_num; ++layer){
193 layer_type = (int32_t)avio_rl32(model_file_context);
196 if (layer_type >= DLT_COUNT) {
200 native_model->layers[layer].type = layer_type;
201 parsed_size = ff_layer_funcs[layer_type].pf_load(&native_model->layers[layer], model_file_context, file_size, native_model->operands_num);
205 dnn_size += parsed_size;
208 for (int32_t i = 0; i < native_model->operands_num; ++i){
211 int32_t operand_index = (int32_t)avio_rl32(model_file_context);
214 if (operand_index >= native_model->operands_num) {
218 oprd = &native_model->operands[operand_index];
219 name_len = (int32_t)avio_rl32(model_file_context);
222 avio_get_str(model_file_context, name_len, oprd->name, sizeof(oprd->name));
223 dnn_size += name_len;
225 oprd->type = (int32_t)avio_rl32(model_file_context);
228 oprd->data_type = (int32_t)avio_rl32(model_file_context);
231 for (int32_t dim = 0; dim < 4; ++dim) {
232 oprd->dims[dim] = (int32_t)avio_rl32(model_file_context);
235 if (oprd->type == DOT_INPUT && oprd->dims[0] != 1)
241 avio_closep(&model_file_context);
243 if (dnn_size != file_size){
244 ff_dnn_free_model_native(&model);
248 model->get_input = &get_input_native;
249 model->get_output = &get_output_native;
250 model->filter_ctx = filter_ctx;
251 model->func_type = func_type;
256 ff_dnn_free_model_native(&model);
257 avio_closep(&model_file_context);
261 static DNNReturnType execute_model_native(const DNNModel *model, const char *input_name, AVFrame *in_frame,
262 const char **output_names, uint32_t nb_output, AVFrame *out_frame,
265 NativeModel *native_model = model->model;
266 NativeContext *ctx = &native_model->ctx;
268 DNNData input, output;
269 DnnOperand *oprd = NULL;
271 if (native_model->layers_num <= 0 || native_model->operands_num <= 0) {
272 av_log(ctx, AV_LOG_ERROR, "No operands or layers in model\n");
276 for (int i = 0; i < native_model->operands_num; ++i) {
277 oprd = &native_model->operands[i];
278 if (strcmp(oprd->name, input_name) == 0) {
279 if (oprd->type != DOT_INPUT) {
280 av_log(ctx, AV_LOG_ERROR, "Found \"%s\" in model, but it is not input node\n", input_name);
288 av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name);
292 oprd->dims[1] = in_frame->height;
293 oprd->dims[2] = in_frame->width;
295 av_freep(&oprd->data);
296 oprd->length = ff_calculate_operand_data_length(oprd);
297 if (oprd->length <= 0) {
298 av_log(ctx, AV_LOG_ERROR, "The input data length overflow\n");
301 oprd->data = av_malloc(oprd->length);
303 av_log(ctx, AV_LOG_ERROR, "Failed to malloc memory for input data\n");
307 input.height = oprd->dims[1];
308 input.width = oprd->dims[2];
309 input.channels = oprd->dims[3];
310 input.data = oprd->data;
311 input.dt = oprd->data_type;
313 if (native_model->model->pre_proc != NULL) {
314 native_model->model->pre_proc(in_frame, &input, native_model->model->filter_ctx);
316 ff_proc_from_frame_to_dnn(in_frame, &input, native_model->model->func_type, ctx);
320 if (nb_output != 1) {
321 // currently, the filter does not need multiple outputs,
322 // so we just pending the support until we really need it.
323 avpriv_report_missing_feature(ctx, "multiple outputs");
327 for (layer = 0; layer < native_model->layers_num; ++layer){
328 DNNLayerType layer_type = native_model->layers[layer].type;
329 if (ff_layer_funcs[layer_type].pf_exec(native_model->operands,
330 native_model->layers[layer].input_operand_indexes,
331 native_model->layers[layer].output_operand_index,
332 native_model->layers[layer].params,
333 &native_model->ctx) == DNN_ERROR) {
334 av_log(ctx, AV_LOG_ERROR, "Failed to execute model\n");
339 for (uint32_t i = 0; i < nb_output; ++i) {
340 DnnOperand *oprd = NULL;
341 const char *output_name = output_names[i];
342 for (int j = 0; j < native_model->operands_num; ++j) {
343 if (strcmp(native_model->operands[j].name, output_name) == 0) {
344 oprd = &native_model->operands[j];
350 av_log(ctx, AV_LOG_ERROR, "Could not find output in model\n");
354 output.data = oprd->data;
355 output.height = oprd->dims[1];
356 output.width = oprd->dims[2];
357 output.channels = oprd->dims[3];
358 output.dt = oprd->data_type;
361 if (native_model->model->post_proc != NULL) {
362 native_model->model->post_proc(out_frame, &output, native_model->model->filter_ctx);
364 ff_proc_from_dnn_to_frame(out_frame, &output, ctx);
367 out_frame->width = output.width;
368 out_frame->height = output.height;
375 DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, const char *input_name, AVFrame *in_frame,
376 const char **output_names, uint32_t nb_output, AVFrame *out_frame)
378 NativeModel *native_model = model->model;
379 NativeContext *ctx = &native_model->ctx;
382 av_log(ctx, AV_LOG_ERROR, "in frame is NULL when execute model.\n");
387 av_log(ctx, AV_LOG_ERROR, "out frame is NULL when execute model.\n");
391 return execute_model_native(model, input_name, in_frame, output_names, nb_output, out_frame, 1);
394 int32_t ff_calculate_operand_dims_count(const DnnOperand *oprd)
397 for (int i = 0; i < 4; ++i)
398 result *= oprd->dims[i];
403 int32_t ff_calculate_operand_data_length(const DnnOperand* oprd)
405 // currently, we just support DNN_FLOAT
406 uint64_t len = sizeof(float);
407 for (int i = 0; i < 4; i++) {
408 len *= oprd->dims[i];
415 void ff_dnn_free_model_native(DNNModel **model)
417 NativeModel *native_model;
418 ConvolutionalParams *conv_params;
423 if ((*model)->model) {
424 native_model = (*model)->model;
425 if (native_model->layers) {
426 for (layer = 0; layer < native_model->layers_num; ++layer){
427 if (native_model->layers[layer].type == DLT_CONV2D){
428 conv_params = (ConvolutionalParams *)native_model->layers[layer].params;
429 av_freep(&conv_params->kernel);
430 av_freep(&conv_params->biases);
432 av_freep(&native_model->layers[layer].params);
434 av_freep(&native_model->layers);
437 if (native_model->operands) {
438 for (uint32_t operand = 0; operand < native_model->operands_num; ++operand)
439 av_freep(&native_model->operands[operand].data);
440 av_freep(&native_model->operands);
443 av_freep(&native_model);