git.sesse.net Git - ffmpeg/blob - libavfilter/dnn/dnn_backend_native.c

   1 /*
   2  * Copyright (c) 2018 Sergey Lavrushkin
   3  *
   4  * This file is part of FFmpeg.
   5  *
   6  * FFmpeg is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * FFmpeg is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with FFmpeg; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  19  */
  20
  21 /**
  22  * @file
  23  * DNN native backend implementation.
  24  */
  25
  26 #include "dnn_backend_native.h"
  27 #include "libavutil/avassert.h"
  28 #include "dnn_backend_native_layer_pad.h"
  29 #include "dnn_backend_native_layer_conv2d.h"
  30 #include "dnn_backend_native_layer_depth2space.h"
  31 #include "dnn_backend_native_layer_maximum.h"
  32
  33 static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output)
  34 {
  35     ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
  36
  37     if (network->layers_num <= 0 || network->operands_num <= 0)
  38         return DNN_ERROR;
  39
  40     av_assert0(input->dt == DNN_FLOAT);
  41
  42     /**
  43      * as the first step, suppose network->operands[0] is the input operand.
  44      */
  45     network->operands[0].dims[0] = 1;
  46     network->operands[0].dims[1] = input->height;
  47     network->operands[0].dims[2] = input->width;
  48     network->operands[0].dims[3] = input->channels;
  49     network->operands[0].type = DOT_INPUT;
  50     network->operands[0].data_type = DNN_FLOAT;
  51     network->operands[0].isNHWC = 1;
  52
  53     av_freep(&network->operands[0].data);
  54     network->operands[0].length = calculate_operand_data_length(&network->operands[0]);
  55     network->operands[0].data = av_malloc(network->operands[0].length);
  56     if (!network->operands[0].data)
  57         return DNN_ERROR;
  58
  59     input->data = network->operands[0].data;
  60     return DNN_SUCCESS;
  61 }
  62
  63 // Loads model and its parameters that are stored in a binary file with following structure:
  64 // layers_num,layer_type,layer_parameterss,layer_type,layer_parameters...
  65 // For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases
  66 // For DEPTH_TO_SPACE layer: block_size
  67 DNNModel *ff_dnn_load_model_native(const char *model_filename)
  68 {
  69     DNNModel *model = NULL;
  70     char header_expected[] = "FFMPEGDNNNATIVE";
  71     char *buf;
  72     size_t size;
  73     int version, header_size, major_version_expected = 0;
  74     ConvolutionalNetwork *network = NULL;
  75     AVIOContext *model_file_context;
  76     int file_size, dnn_size, kernel_size, i;
  77     int32_t layer;
  78     DNNLayerType layer_type;
  79     ConvolutionalParams *conv_params;
  80     DepthToSpaceParams *depth_to_space_params;
  81     LayerPadParams *pad_params;
  82     DnnLayerMaximumParams *maximum_params;
  83
  84     model = av_malloc(sizeof(DNNModel));
  85     if (!model){
  86         return NULL;
  87     }
  88
  89     if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
  90         av_freep(&model);
  91         return NULL;
  92     }
  93     file_size = avio_size(model_file_context);
  94
  95     /**
  96      * check file header with string and version
  97      */
  98     size = sizeof(header_expected);
  99     buf = av_malloc(size);
 100     if (!buf) {
 101         avio_closep(&model_file_context);
 102         av_freep(&model);
 103         return NULL;
 104     }
 105
 106     // size - 1 to skip the ending '\0' which is not saved in file
 107     avio_get_str(model_file_context, size - 1, buf, size);
 108     dnn_size = size - 1;
 109     if (strncmp(buf, header_expected, size) != 0) {
 110         av_freep(&buf);
 111         avio_closep(&model_file_context);
 112         av_freep(&model);
 113         return NULL;
 114     }
 115     av_freep(&buf);
 116
 117     version = (int32_t)avio_rl32(model_file_context);
 118     dnn_size += 4;
 119     if (version != major_version_expected) {
 120         avio_closep(&model_file_context);
 121         av_freep(&model);
 122         return NULL;
 123     }
 124
 125     // currently no need to check minor version
 126     version = (int32_t)avio_rl32(model_file_context);
 127     dnn_size += 4;
 128     header_size = dnn_size;
 129
 130     network = av_mallocz(sizeof(ConvolutionalNetwork));
 131     if (!network){
 132         avio_closep(&model_file_context);
 133         av_freep(&model);
 134         return NULL;
 135     }
 136     model->model = (void *)network;
 137
 138     avio_seek(model_file_context, file_size - 8, SEEK_SET);
 139     network->layers_num = (int32_t)avio_rl32(model_file_context);
 140     network->operands_num = (int32_t)avio_rl32(model_file_context);
 141     dnn_size += 8;
 142     avio_seek(model_file_context, header_size, SEEK_SET);
 143
 144     network->layers = av_mallocz(network->layers_num * sizeof(Layer));
 145     if (!network->layers){
 146         avio_closep(&model_file_context);
 147         ff_dnn_free_model_native(&model);
 148         return NULL;
 149     }
 150
 151     network->operands = av_mallocz(network->operands_num * sizeof(DnnOperand));
 152     if (!network->operands){
 153         avio_closep(&model_file_context);
 154         ff_dnn_free_model_native(&model);
 155         return NULL;
 156     }
 157
 158     for (layer = 0; layer < network->layers_num; ++layer){
 159         layer_type = (int32_t)avio_rl32(model_file_context);
 160         dnn_size += 4;
 161         switch (layer_type){
 162         case CONV:
 163             conv_params = av_malloc(sizeof(ConvolutionalParams));
 164             if (!conv_params){
 165                 avio_closep(&model_file_context);
 166                 ff_dnn_free_model_native(&model);
 167                 return NULL;
 168             }
 169             conv_params->dilation = (int32_t)avio_rl32(model_file_context);
 170             conv_params->padding_method = (int32_t)avio_rl32(model_file_context);
 171             conv_params->activation = (int32_t)avio_rl32(model_file_context);
 172             conv_params->input_num = (int32_t)avio_rl32(model_file_context);
 173             conv_params->output_num = (int32_t)avio_rl32(model_file_context);
 174             conv_params->kernel_size = (int32_t)avio_rl32(model_file_context);
 175             kernel_size = conv_params->input_num * conv_params->output_num *
 176                           conv_params->kernel_size * conv_params->kernel_size;
 177             dnn_size += 24 + (kernel_size + conv_params->output_num << 2);
 178             if (dnn_size > file_size || conv_params->input_num <= 0 ||
 179                 conv_params->output_num <= 0 || conv_params->kernel_size <= 0){
 180                 avio_closep(&model_file_context);
 181                 av_freep(&conv_params);
 182                 ff_dnn_free_model_native(&model);
 183                 return NULL;
 184             }
 185             conv_params->kernel = av_malloc(kernel_size * sizeof(float));
 186             conv_params->biases = av_malloc(conv_params->output_num * sizeof(float));
 187             if (!conv_params->kernel || !conv_params->biases){
 188                 avio_closep(&model_file_context);
 189                 av_freep(&conv_params->kernel);
 190                 av_freep(&conv_params->biases);
 191                 av_freep(&conv_params);
 192                 ff_dnn_free_model_native(&model);
 193                 return NULL;
 194             }
 195             for (i = 0; i < kernel_size; ++i){
 196                 conv_params->kernel[i] = av_int2float(avio_rl32(model_file_context));
 197             }
 198             for (i = 0; i < conv_params->output_num; ++i){
 199                 conv_params->biases[i] = av_int2float(avio_rl32(model_file_context));
 200             }
 201             network->layers[layer].input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
 202             network->layers[layer].output_operand_index = (int32_t)avio_rl32(model_file_context);
 203             dnn_size += 8;
 204             network->layers[layer].type = CONV;
 205             network->layers[layer].params = conv_params;
 206             break;
 207         case DEPTH_TO_SPACE:
 208             depth_to_space_params = av_malloc(sizeof(DepthToSpaceParams));
 209             if (!depth_to_space_params){
 210                 avio_closep(&model_file_context);
 211                 ff_dnn_free_model_native(&model);
 212                 return NULL;
 213             }
 214             depth_to_space_params->block_size = (int32_t)avio_rl32(model_file_context);
 215             dnn_size += 4;
 216             network->layers[layer].input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
 217             network->layers[layer].output_operand_index = (int32_t)avio_rl32(model_file_context);
 218             dnn_size += 8;
 219             network->layers[layer].type = DEPTH_TO_SPACE;
 220             network->layers[layer].params = depth_to_space_params;
 221             break;
 222         case MIRROR_PAD:
 223             pad_params = av_malloc(sizeof(LayerPadParams));
 224             if (!pad_params){
 225                 avio_closep(&model_file_context);
 226                 ff_dnn_free_model_native(&model);
 227                 return NULL;
 228             }
 229             pad_params->mode = (int32_t)avio_rl32(model_file_context);
 230             dnn_size += 4;
 231             for (i = 0; i < 4; ++i) {
 232                 pad_params->paddings[i][0] = avio_rl32(model_file_context);
 233                 pad_params->paddings[i][1] = avio_rl32(model_file_context);
 234                 dnn_size += 8;
 235             }
 236             network->layers[layer].input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
 237             network->layers[layer].output_operand_index = (int32_t)avio_rl32(model_file_context);
 238             dnn_size += 8;
 239             network->layers[layer].type = MIRROR_PAD;
 240             network->layers[layer].params = pad_params;
 241             break;
 242         case MAXIMUM:
 243             maximum_params = av_malloc(sizeof(*maximum_params));
 244             if (!maximum_params){
 245                 avio_closep(&model_file_context);
 246                 ff_dnn_free_model_native(&model);
 247                 return NULL;
 248             }
 249             maximum_params->val.u32 = avio_rl32(model_file_context);
 250             dnn_size += 4;
 251             network->layers[layer].type = MAXIMUM;
 252             network->layers[layer].params = maximum_params;
 253             network->layers[layer].input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
 254             network->layers[layer].output_operand_index = (int32_t)avio_rl32(model_file_context);
 255             dnn_size += 8;
 256             break;
 257         default:
 258             avio_closep(&model_file_context);
 259             ff_dnn_free_model_native(&model);
 260             return NULL;
 261         }
 262     }
 263
 264     for (int32_t i = 0; i < network->operands_num; ++i){
 265         DnnOperand *oprd;
 266         int32_t name_len;
 267         int32_t operand_index = (int32_t)avio_rl32(model_file_context);
 268         dnn_size += 4;
 269
 270         oprd = &network->operands[operand_index];
 271         name_len = (int32_t)avio_rl32(model_file_context);
 272         dnn_size += 4;
 273
 274         avio_get_str(model_file_context, name_len, oprd->name, sizeof(oprd->name));
 275         dnn_size += name_len;
 276
 277         oprd->type = (int32_t)avio_rl32(model_file_context);
 278         dnn_size += 4;
 279
 280         oprd->data_type = (int32_t)avio_rl32(model_file_context);
 281         dnn_size += 4;
 282
 283         for (int32_t dim = 0; dim < 4; ++dim) {
 284             oprd->dims[dim] = (int32_t)avio_rl32(model_file_context);
 285             dnn_size += 4;
 286         }
 287
 288         oprd->isNHWC = 1;
 289     }
 290
 291     avio_closep(&model_file_context);
 292
 293     if (dnn_size != file_size){
 294         ff_dnn_free_model_native(&model);
 295         return NULL;
 296     }
 297
 298     model->set_input_output = &set_input_output_native;
 299
 300     return model;
 301 }
 302
 303 DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output)
 304 {
 305     ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model;
 306     int32_t layer;
 307     ConvolutionalParams *conv_params;
 308     DepthToSpaceParams *depth_to_space_params;
 309     LayerPadParams *pad_params;
 310     DnnLayerMaximumParams *maximum_params;
 311
 312     if (network->layers_num <= 0 || network->operands_num <= 0)
 313         return DNN_ERROR;
 314     if (!network->operands[0].data)
 315         return DNN_ERROR;
 316
 317     for (layer = 0; layer < network->layers_num; ++layer){
 318         switch (network->layers[layer].type){
 319         case CONV:
 320             conv_params = (ConvolutionalParams *)network->layers[layer].params;
 321             convolve(network->operands, network->layers[layer].input_operand_indexes,
 322                      network->layers[layer].output_operand_index, conv_params);
 323             break;
 324         case DEPTH_TO_SPACE:
 325             depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params;
 326             depth_to_space(network->operands, network->layers[layer].input_operand_indexes,
 327                            network->layers[layer].output_operand_index, depth_to_space_params->block_size);
 328             break;
 329         case MIRROR_PAD:
 330             pad_params = (LayerPadParams *)network->layers[layer].params;
 331             dnn_execute_layer_pad(network->operands, network->layers[layer].input_operand_indexes,
 332                                   network->layers[layer].output_operand_index, pad_params);
 333             break;
 334         case MAXIMUM:
 335             maximum_params = (DnnLayerMaximumParams *)network->layers[layer].params;
 336             dnn_execute_layer_maximum(network->operands, network->layers[layer].input_operand_indexes,
 337                                   network->layers[layer].output_operand_index, maximum_params);
 338             break;
 339         case INPUT:
 340             return DNN_ERROR;
 341         }
 342     }
 343
 344     // native mode does not support multiple outputs yet
 345     if (nb_output > 1)
 346         return DNN_ERROR;
 347
 348     /**
 349      * as the first step, suppose network->operands[network->operands_num - 1] is the output operand.
 350      */
 351     outputs[0].data = network->operands[network->operands_num - 1].data;
 352     outputs[0].height = network->operands[network->operands_num - 1].dims[1];
 353     outputs[0].width = network->operands[network->operands_num - 1].dims[2];
 354     outputs[0].channels = network->operands[network->operands_num - 1].dims[3];
 355
 356     return DNN_SUCCESS;
 357 }
 358
 359 int32_t calculate_operand_dims_count(const DnnOperand *oprd)
 360 {
 361     int32_t result = 1;
 362     for (int i = 0; i < 4; ++i)
 363         result *= oprd->dims[i];
 364
 365     return result;
 366 }
 367
 368 int32_t calculate_operand_data_length(const DnnOperand* oprd)
 369 {
 370     // currently, we just support DNN_FLOAT
 371     return oprd->dims[0] * oprd->dims[1] * oprd->dims[2] * oprd->dims[3] * sizeof(float);
 372 }
 373
 374 void ff_dnn_free_model_native(DNNModel **model)
 375 {
 376     ConvolutionalNetwork *network;
 377     ConvolutionalParams *conv_params;
 378     int32_t layer;
 379
 380     if (*model)
 381     {
 382         network = (ConvolutionalNetwork *)(*model)->model;
 383         for (layer = 0; layer < network->layers_num; ++layer){
 384             if (network->layers[layer].type == CONV){
 385                 conv_params = (ConvolutionalParams *)network->layers[layer].params;
 386                 av_freep(&conv_params->kernel);
 387                 av_freep(&conv_params->biases);
 388             }
 389             av_freep(&network->layers[layer].params);
 390         }
 391         av_freep(&network->layers);
 392
 393         for (uint32_t operand = 0; operand < network->operands_num; ++operand)
 394             av_freep(&network->operands[operand].data);
 395         av_freep(&network->operands);
 396
 397         av_freep(&network);
 398         av_freep(model);
 399     }
 400 }