dnn_backend_native: Add overflow check for length calculation.

We should not silently allocate an incorrect sized buffer.
Fixes trac issue #8718.

Signed-off-by: Reimar Döffinger <Reimar.Doeffinger@gmx.de>
Reviewed-by: Michael Niedermayer <michael@niedermayer.cc>
Reviewed-by: Guo, Yejun <yejun.guo@intel.com>

diff --git a/libavfilter/dnn/dnn_backend_native.c b/libavfilter/dnn/dnn_backend_native.c
index 35236fc66fbb528b2ef96f258bef09fb8fbb7992..a685efb09236ae94879ad244359321d120a25295 100644 (file)
--- a/libavfilter/dnn/dnn_backend_native.c
+++ b/libavfilter/dnn/dnn_backend_native.c
@@ -79,6 +79,8 @@ static DNNReturnType set_input_output_native(void *model, DNNData *input, const
 
     av_freep(&oprd->data);
     oprd->length = calculate_operand_data_length(oprd);
+    if (oprd->length <= 0)
+        return DNN_ERROR;
     oprd->data = av_malloc(oprd->length);
     if (!oprd->data)
         return DNN_ERROR;
@@ -295,7 +297,13 @@ int32_t calculate_operand_dims_count(const DnnOperand *oprd)
 int32_t calculate_operand_data_length(const DnnOperand* oprd)
 {
     // currently, we just support DNN_FLOAT
-    return oprd->dims[0] * oprd->dims[1] * oprd->dims[2] * oprd->dims[3] * sizeof(float);
+    uint64_t len = sizeof(float);
+    for (int i = 0; i < 4; i++) {
+        len *= oprd->dims[i];
+        if (len > INT32_MAX)
+            return 0;
+    }
+    return len;
 }
 
 void ff_dnn_free_model_native(DNNModel **model)

diff --git a/libavfilter/dnn/dnn_backend_native.h b/libavfilter/dnn/dnn_backend_native.h
index bec63be45008f18d417186aa81cd3f411febc84d..62191ffe883dbed50ac9d34982bf1ed864a47ee8 100644 (file)
--- a/libavfilter/dnn/dnn_backend_native.h
+++ b/libavfilter/dnn/dnn_backend_native.h
@@ -120,6 +120,8 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output
 
 void ff_dnn_free_model_native(DNNModel **model);
 
+// NOTE: User must check for error (return value <= 0) to handle
+// case like integer overflow.
 int32_t calculate_operand_data_length(const DnnOperand *oprd);
 int32_t calculate_operand_dims_count(const DnnOperand *oprd);
 #endif

diff --git a/libavfilter/dnn/dnn_backend_native_layer_conv2d.c b/libavfilter/dnn/dnn_backend_native_layer_conv2d.c
index c05bb5eca92743ea72cb2d15435f83023b682c68..a2202e40738d3a8975ad66addc79419ad0cbb8ae 100644 (file)
--- a/libavfilter/dnn/dnn_backend_native_layer_conv2d.c
+++ b/libavfilter/dnn/dnn_backend_native_layer_conv2d.c
@@ -113,6 +113,8 @@ int dnn_execute_layer_conv2d(DnnOperand *operands, const int32_t *input_operand_
     output_operand->dims[3] = conv_params->output_num;
     output_operand->data_type = operands[input_operand_index].data_type;
     output_operand->length = calculate_operand_data_length(output_operand);
+    if (output_operand->length <= 0)
+        return -1;
     output_operand->data = av_realloc(output_operand->data, output_operand->length);
     if (!output_operand->data)
         return -1;

diff --git a/libavfilter/dnn/dnn_backend_native_layer_depth2space.c b/libavfilter/dnn/dnn_backend_native_layer_depth2space.c
index 324871cecaf48d48c0c853fb000bfaa951e8ee73..2c8bddf23dae95b63676d79ae417b30369b26427 100644 (file)
--- a/libavfilter/dnn/dnn_backend_native_layer_depth2space.c
+++ b/libavfilter/dnn/dnn_backend_native_layer_depth2space.c
@@ -75,6 +75,8 @@ int dnn_execute_layer_depth2space(DnnOperand *operands, const int32_t *input_ope
     output_operand->dims[3] = new_channels;
     output_operand->data_type = operands[input_operand_index].data_type;
     output_operand->length = calculate_operand_data_length(output_operand);
+    if (output_operand->length <= 0)
+        return -1;
     output_operand->data = av_realloc(output_operand->data, output_operand->length);
     if (!output_operand->data)
         return -1;

diff --git a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c
index b239a2005843deeb645bf268c65c7fe7c98edbc1..dd42c329a9d9da718dffbc5377154bb5281570cf 100644 (file)
--- a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c
+++ b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c
@@ -91,6 +91,8 @@ int dnn_execute_layer_math_binary(DnnOperand *operands, const int32_t *input_ope
 
     output->data_type = input->data_type;
     output->length = calculate_operand_data_length(output);
+    if (output->length <= 0)
+        return DNN_ERROR;
     output->data = av_realloc(output->data, output->length);
     if (!output->data)
         return DNN_ERROR;

diff --git a/libavfilter/dnn/dnn_backend_native_layer_mathunary.c b/libavfilter/dnn/dnn_backend_native_layer_mathunary.c
index c83d50db64c6237907c38d45f2df6834effad42e..c5f0f7adec4b1b9a0be6176054e3c663b15a6fd2 100644 (file)
--- a/libavfilter/dnn/dnn_backend_native_layer_mathunary.c
+++ b/libavfilter/dnn/dnn_backend_native_layer_mathunary.c
@@ -67,6 +67,8 @@ int dnn_execute_layer_math_unary(DnnOperand *operands, const int32_t *input_oper
 
     output->data_type = input->data_type;
     output->length = calculate_operand_data_length(output);
+    if (output->length <= 0)
+        return DNN_ERROR;
     output->data = av_realloc(output->data, output->length);
     if (!output->data)
         return DNN_ERROR;

diff --git a/libavfilter/dnn/dnn_backend_native_layer_maximum.c b/libavfilter/dnn/dnn_backend_native_layer_maximum.c
index af16e08b951b227b868e44a1189e53877661dcb8..cdddfdd87b20f4680782275c053f50d44164ec1b 100644 (file)
--- a/libavfilter/dnn/dnn_backend_native_layer_maximum.c
+++ b/libavfilter/dnn/dnn_backend_native_layer_maximum.c
@@ -64,6 +64,8 @@ int dnn_execute_layer_maximum(DnnOperand *operands, const int32_t *input_operand
 
     output->data_type = input->data_type;
     output->length = calculate_operand_data_length(output);
+    if (output->length <= 0)
+        return DNN_ERROR;
     output->data = av_realloc(output->data, output->length);
     if (!output->data)
         return DNN_ERROR;

diff --git a/libavfilter/dnn/dnn_backend_native_layer_pad.c b/libavfilter/dnn/dnn_backend_native_layer_pad.c
index dfbd20445620d49993b24188c23a42a680538535..feaab001e897bcee104dde141b76a544951cfdec 100644 (file)
--- a/libavfilter/dnn/dnn_backend_native_layer_pad.c
+++ b/libavfilter/dnn/dnn_backend_native_layer_pad.c
@@ -111,6 +111,8 @@ int dnn_execute_layer_pad(DnnOperand *operands, const int32_t *input_operand_ind
     output_operand->dims[3] = new_channel;
     output_operand->data_type = operands[input_operand_index].data_type;
     output_operand->length = calculate_operand_data_length(output_operand);
+    if (output_operand->length <= 0)
+        return -1;
     output_operand->data = av_realloc(output_operand->data, output_operand->length);
     if (!output_operand->data)
         return -1;