]> git.sesse.net Git - ffmpeg/blobdiff - libavfilter/formats.c
avfilter/formats: Make check for buffer overflow redundant
[ffmpeg] / libavfilter / formats.c
index 31ee445c494ee38fdd75b6e6517274ed70a31a6e..c5a64c14c641dd4feee662bde17a96429b262333 100644 (file)
 
 /**
  * Add all refs from a to ret and destroy a.
+ * ret->refs must have enough spare room left for this.
  */
-#define MERGE_REF(ret, a, fmts, type, fail)                                \
+#define MERGE_REF_NO_ALLOC(ret, a, fmts)                                   \
 do {                                                                       \
-    type ***tmp;                                                           \
     int i;                                                                 \
-                                                                           \
-    if (!(tmp = av_realloc_array(ret->refs, ret->refcount + a->refcount,   \
-                                 sizeof(*tmp))))                           \
-        goto fail;                                                         \
-    ret->refs = tmp;                                                       \
-                                                                           \
     for (i = 0; i < a->refcount; i ++) {                                   \
         ret->refs[ret->refcount] = a->refs[i];                             \
         *ret->refs[ret->refcount++] = ret;                                 \
@@ -54,13 +48,25 @@ do {                                                                       \
     av_freep(&a);                                                          \
 } while (0)
 
+#define MERGE_REF(ret, a, fmts, type, fail_statement)                      \
+do {                                                                       \
+    type ***tmp;                                                           \
+                                                                           \
+    if (!(tmp = av_realloc_array(ret->refs, ret->refcount + a->refcount,   \
+                                 sizeof(*tmp))))                           \
+        { fail_statement }                                                 \
+    ret->refs = tmp;                                                       \
+    MERGE_REF_NO_ALLOC(ret, a, fmts);                                      \
+} while (0)
+
 /**
  * Add all formats common for a and b to ret, copy the refs and destroy
  * a and b.
  */
 #define MERGE_FORMATS(ret, a, b, fmts, nb, type, fail)                          \
 do {                                                                            \
-    int i, j, k = 0, count = FFMIN(a->nb, b->nb);                               \
+    int i, j, k = 0, count = a->nb;                                             \
+    type ***tmp;                                                                \
                                                                                 \
     if (!(ret = av_mallocz(sizeof(*ret))))                                      \
         goto fail;                                                              \
@@ -71,13 +77,8 @@ do {
         for (i = 0; i < a->nb; i++)                                             \
             for (j = 0; j < b->nb; j++)                                         \
                 if (a->fmts[i] == b->fmts[j]) {                                 \
-                    if(k >= FFMIN(a->nb, b->nb)){                               \
-                        av_log(NULL, AV_LOG_ERROR, "Duplicate formats in %s detected\n", __FUNCTION__); \
-                        av_free(ret->fmts);                                     \
-                        av_free(ret);                                           \
-                        return NULL;                                            \
-                    }                                                           \
                     ret->fmts[k++] = a->fmts[i];                                \
+                    break;                                                      \
                 }                                                               \
     }                                                                           \
     ret->nb = k;                                                                \
@@ -85,8 +86,13 @@ do {
     if (!ret->nb)                                                               \
         goto fail;                                                              \
                                                                                 \
-    MERGE_REF(ret, a, fmts, type, fail);                                        \
-    MERGE_REF(ret, b, fmts, type, fail);                                        \
+    tmp = av_realloc_array(NULL, a->refcount + b->refcount, sizeof(*tmp));      \
+    if (!tmp)                                                                   \
+        goto fail;                                                              \
+    ret->refs = tmp;                                                            \
+                                                                                \
+    MERGE_REF_NO_ALLOC(ret, a, fmts);                                           \
+    MERGE_REF_NO_ALLOC(ret, b, fmts);                                           \
 } while (0)
 
 AVFilterFormats *ff_merge_formats(AVFilterFormats *a, AVFilterFormats *b,
@@ -129,10 +135,10 @@ AVFilterFormats *ff_merge_formats(AVFilterFormats *a, AVFilterFormats *b,
     return ret;
 fail:
     if (ret) {
-        av_freep(&ret->refs);
+        av_assert1(!ret->refs);
         av_freep(&ret->formats);
+        av_freep(&ret);
     }
-    av_freep(&ret);
     return NULL;
 }
 
@@ -146,27 +152,27 @@ AVFilterFormats *ff_merge_samplerates(AVFilterFormats *a,
     if (a->nb_formats && b->nb_formats) {
         MERGE_FORMATS(ret, a, b, formats, nb_formats, AVFilterFormats, fail);
     } else if (a->nb_formats) {
-        MERGE_REF(a, b, formats, AVFilterFormats, fail);
+        MERGE_REF(a, b, formats, AVFilterFormats, return NULL;);
         ret = a;
     } else {
-        MERGE_REF(b, a, formats, AVFilterFormats, fail);
+        MERGE_REF(b, a, formats, AVFilterFormats, return NULL;);
         ret = b;
     }
 
     return ret;
 fail:
     if (ret) {
-        av_freep(&ret->refs);
+        av_assert1(!ret->refs);
         av_freep(&ret->formats);
+        av_freep(&ret);
     }
-    av_freep(&ret);
     return NULL;
 }
 
 AVFilterChannelLayouts *ff_merge_channel_layouts(AVFilterChannelLayouts *a,
                                                  AVFilterChannelLayouts *b)
 {
-    AVFilterChannelLayouts *ret = NULL;
+    uint64_t *channel_layouts;
     unsigned a_all = a->all_layouts + a->all_counts;
     unsigned b_all = b->all_layouts + b->all_counts;
     int ret_max, ret_nb = 0, i, j, round;
@@ -190,15 +196,13 @@ AVFilterChannelLayouts *ff_merge_channel_layouts(AVFilterChannelLayouts *a,
                 return NULL;
             b->nb_channel_layouts = j;
         }
-        MERGE_REF(b, a, channel_layouts, AVFilterChannelLayouts, fail);
+        MERGE_REF(b, a, channel_layouts, AVFilterChannelLayouts, return NULL;);
         return b;
     }
 
     ret_max = a->nb_channel_layouts + b->nb_channel_layouts;
-    if (!(ret = av_mallocz(sizeof(*ret))) ||
-        !(ret->channel_layouts = av_malloc_array(ret_max,
-                                                 sizeof(*ret->channel_layouts))))
-        goto fail;
+    if (!(channel_layouts = av_malloc_array(ret_max, sizeof(*channel_layouts))))
+        return NULL;
 
     /* a[known] intersect b[known] */
     for (i = 0; i < a->nb_channel_layouts; i++) {
@@ -206,8 +210,9 @@ AVFilterChannelLayouts *ff_merge_channel_layouts(AVFilterChannelLayouts *a,
             continue;
         for (j = 0; j < b->nb_channel_layouts; j++) {
             if (a->channel_layouts[i] == b->channel_layouts[j]) {
-                ret->channel_layouts[ret_nb++] = a->channel_layouts[i];
+                channel_layouts[ret_nb++] = a->channel_layouts[i];
                 a->channel_layouts[i] = b->channel_layouts[j] = 0;
+                break;
             }
         }
     }
@@ -221,7 +226,7 @@ AVFilterChannelLayouts *ff_merge_channel_layouts(AVFilterChannelLayouts *a,
             bfmt = FF_COUNT2LAYOUT(av_get_channel_layout_nb_channels(fmt));
             for (j = 0; j < b->nb_channel_layouts; j++)
                 if (b->channel_layouts[j] == bfmt)
-                    ret->channel_layouts[ret_nb++] = a->channel_layouts[i];
+                    channel_layouts[ret_nb++] = a->channel_layouts[i];
         }
         /* 1st round: swap to prepare 2nd round; 2nd round: put it back */
         FFSWAP(AVFilterChannelLayouts *, a, b);
@@ -232,22 +237,23 @@ AVFilterChannelLayouts *ff_merge_channel_layouts(AVFilterChannelLayouts *a,
             continue;
         for (j = 0; j < b->nb_channel_layouts; j++)
             if (a->channel_layouts[i] == b->channel_layouts[j])
-                ret->channel_layouts[ret_nb++] = a->channel_layouts[i];
+                channel_layouts[ret_nb++] = a->channel_layouts[i];
     }
 
-    ret->nb_channel_layouts = ret_nb;
-    if (!ret->nb_channel_layouts)
+    if (!ret_nb)
         goto fail;
-    MERGE_REF(ret, a, channel_layouts, AVFilterChannelLayouts, fail);
-    MERGE_REF(ret, b, channel_layouts, AVFilterChannelLayouts, fail);
-    return ret;
+
+    if (a->refcount > b->refcount)
+        FFSWAP(AVFilterChannelLayouts *, a, b);
+
+    MERGE_REF(b, a, channel_layouts, AVFilterChannelLayouts, goto fail;);
+    av_freep(&b->channel_layouts);
+    b->channel_layouts    = channel_layouts;
+    b->nb_channel_layouts = ret_nb;
+    return b;
 
 fail:
-    if (ret) {
-        av_freep(&ret->refs);
-        av_freep(&ret->channel_layouts);
-    }
-    av_freep(&ret);
+    av_free(channel_layouts);
     return NULL;
 }
 
@@ -289,7 +295,7 @@ AVFilterFormats *ff_make_format_list(const int *fmts)
     return formats;
 }
 
-AVFilterChannelLayouts *ff_make_formatu64_list(const uint64_t *fmts)
+AVFilterChannelLayouts *ff_make_format64_list(const int64_t *fmts)
 {
     MAKE_FORMAT_LIST(AVFilterChannelLayouts,
                      channel_layouts, nb_channel_layouts);
@@ -300,16 +306,12 @@ AVFilterChannelLayouts *ff_make_formatu64_list(const uint64_t *fmts)
     return formats;
 }
 
+#if LIBAVFILTER_VERSION_MAJOR < 8
 AVFilterChannelLayouts *avfilter_make_format64_list(const int64_t *fmts)
 {
-    MAKE_FORMAT_LIST(AVFilterChannelLayouts,
-                     channel_layouts, nb_channel_layouts);
-    if (count)
-        memcpy(formats->channel_layouts, fmts,
-               sizeof(*formats->channel_layouts) * count);
-
-    return formats;
+    return ff_make_format64_list(fmts);
 }
+#endif
 
 #define ADD_FORMAT(f, fmt, unref_fn, type, list, nb)        \
 do {                                                        \
@@ -317,7 +319,6 @@ do {                                                        \
     void *oldf = *f;                                        \
                                                             \
     if (!(*f) && !(*f = av_mallocz(sizeof(**f)))) {         \
-        unref_fn(f);                                        \
         return AVERROR(ENOMEM);                             \
     }                                                       \
                                                             \
@@ -369,15 +370,46 @@ AVFilterFormats *ff_all_formats(enum AVMediaType type)
     return ret;
 }
 
-const int64_t avfilter_all_channel_layouts[] = {
-#include "all_channel_layouts.inc"
-    -1
-};
-
-// AVFilterFormats *avfilter_make_all_channel_layouts(void)
-// {
-//     return avfilter_make_format64_list(avfilter_all_channel_layouts);
-// }
+int ff_formats_pixdesc_filter(AVFilterFormats **rfmts, unsigned want, unsigned rej)
+{
+    unsigned nb_formats, fmt, flags;
+    AVFilterFormats *formats = NULL;
+
+    while (1) {
+        nb_formats = 0;
+        for (fmt = 0;; fmt++) {
+            const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt);
+            if (!desc)
+                break;
+            flags = desc->flags;
+            if (!(desc->flags & AV_PIX_FMT_FLAG_HWACCEL) &&
+                !(desc->flags & AV_PIX_FMT_FLAG_PLANAR) &&
+                (desc->log2_chroma_w || desc->log2_chroma_h))
+                flags |= FF_PIX_FMT_FLAG_SW_FLAT_SUB;
+            if ((flags & (want | rej)) != want)
+                continue;
+            if (formats)
+                formats->formats[nb_formats] = fmt;
+            nb_formats++;
+        }
+        if (formats) {
+            av_assert0(formats->nb_formats == nb_formats);
+            *rfmts = formats;
+            return 0;
+        }
+        formats = av_mallocz(sizeof(*formats));
+        if (!formats)
+            return AVERROR(ENOMEM);
+        formats->nb_formats = nb_formats;
+        if (nb_formats) {
+            formats->formats = av_malloc_array(nb_formats, sizeof(*formats->formats));
+            if (!formats->formats) {
+                av_freep(&formats);
+                return AVERROR(ENOMEM);
+            }
+        }
+    }
+}
 
 AVFilterFormats *ff_planar_sample_fmts(void)
 {
@@ -456,7 +488,7 @@ do {                                        \
 do {                                                               \
     int idx = -1;                                                  \
                                                                    \
-    if (!*ref || !(*ref)->refs)                                    \
+    if (!ref || !*ref || !(*ref)->refs)                            \
         return;                                                    \
                                                                    \
     FIND_REF_INDEX(ref, idx);                                      \
@@ -518,7 +550,8 @@ void ff_formats_changeref(AVFilterFormats **oldref, AVFilterFormats **newref)
             int ret = ref_fn(fmts, &ctx->inputs[i]->out_fmts);      \
             if (ret < 0) {                                          \
                 unref_fn(&fmts);                                    \
-                av_freep(&fmts->list);                              \
+                if (fmts)                                           \
+                    av_freep(&fmts->list);                          \
                 av_freep(&fmts);                                    \
                 return ret;                                         \
             }                                                       \
@@ -530,7 +563,8 @@ void ff_formats_changeref(AVFilterFormats **oldref, AVFilterFormats **newref)
             int ret = ref_fn(fmts, &ctx->outputs[i]->in_fmts);      \
             if (ret < 0) {                                          \
                 unref_fn(&fmts);                                    \
-                av_freep(&fmts->list);                              \
+                if (fmts)                                           \
+                    av_freep(&fmts->list);                          \
                 av_freep(&fmts);                                    \
                 return ret;                                         \
             }                                                       \