avcodec/apedec: fix decoding 24bit insane files with recent versions

[ffmpeg] / libavcodec / h264_metadata_bsf.c

diff --git a/libavcodec/h264_metadata_bsf.c b/libavcodec/h264_metadata_bsf.c
index 3684e6bf7f5830aed427fe2736ed2954ff6ac50b..eb1503159b275d6e62bb030c64988df7c320c749 100644 (file)
--- a/libavcodec/h264_metadata_bsf.c
+++ b/libavcodec/h264_metadata_bsf.c
@@ -22,6 +22,7 @@
 #include "libavutil/opt.h"
 
 #include "bsf.h"
+#include "bsf_internal.h"
 #include "cbs.h"
 #include "cbs_h264.h"
 #include "h264.h"
@@ -48,7 +49,8 @@ enum {
 typedef struct H264MetadataContext {
     const AVClass *class;
 
-    CodedBitstreamContext *cbc;
+    CodedBitstreamContext *input;
+    CodedBitstreamContext *output;
     CodedBitstreamFragment access_unit;
 
     int done_first_au;
@@ -57,6 +59,8 @@ typedef struct H264MetadataContext {
 
     AVRational sample_aspect_ratio;
 
+    int overscan_appropriate_flag;
+
     int video_format;
     int video_full_range_flag;
     int colour_primaries;
@@ -129,6 +133,11 @@ static int h264_metadata_update_sps(AVBSFContext *bsf,
         } \
     } while (0)
 
+    if (ctx->overscan_appropriate_flag >= 0) {
+        SET_VUI_FIELD(overscan_appropriate_flag);
+        sps->vui.overscan_info_present_flag = 1;
+    }
+
     if (ctx->video_format             >= 0 ||
         ctx->video_full_range_flag    >= 0 ||
         ctx->colour_primaries         >= 0 ||
@@ -268,6 +277,49 @@ static int h264_metadata_update_sps(AVBSFContext *bsf,
     return 0;
 }
 
+static int h264_metadata_update_side_data(AVBSFContext *bsf, AVPacket *pkt)
+{
+    H264MetadataContext *ctx = bsf->priv_data;
+    CodedBitstreamFragment *au = &ctx->access_unit;
+    uint8_t *side_data;
+    int side_data_size;
+    int err, i;
+
+    side_data = av_packet_get_side_data(pkt, AV_PKT_DATA_NEW_EXTRADATA,
+                                        &side_data_size);
+    if (!side_data_size)
+        return 0;
+
+    err = ff_cbs_read(ctx->input, au, side_data, side_data_size);
+    if (err < 0) {
+        av_log(bsf, AV_LOG_ERROR, "Failed to read extradata from packet side data.\n");
+        return err;
+    }
+
+    for (i = 0; i < au->nb_units; i++) {
+        if (au->units[i].type == H264_NAL_SPS) {
+            err = h264_metadata_update_sps(bsf, au->units[i].content);
+            if (err < 0)
+                return err;
+        }
+    }
+
+    err = ff_cbs_write_fragment_data(ctx->output, au);
+    if (err < 0) {
+        av_log(bsf, AV_LOG_ERROR, "Failed to write extradata into packet side data.\n");
+        return err;
+    }
+
+    side_data = av_packet_new_side_data(pkt, AV_PKT_DATA_NEW_EXTRADATA, au->data_size);
+    if (!side_data)
+        return AVERROR(ENOMEM);
+    memcpy(side_data, au->data, au->data_size);
+
+    ff_cbs_fragment_reset(au);
+
+    return 0;
+}
+
 static int h264_metadata_filter(AVBSFContext *bsf, AVPacket *pkt)
 {
     H264MetadataContext *ctx = bsf->priv_data;
@@ -279,7 +331,11 @@ static int h264_metadata_filter(AVBSFContext *bsf, AVPacket *pkt)
     if (err < 0)
         return err;
 
-    err = ff_cbs_read_packet(ctx->cbc, au, pkt);
+    err = h264_metadata_update_side_data(bsf, pkt);
+    if (err < 0)
+        goto fail;
+
+    err = ff_cbs_read_packet(ctx->input, au, pkt);
     if (err < 0) {
         av_log(bsf, AV_LOG_ERROR, "Failed to read packet.\n");
         goto fail;
@@ -294,7 +350,7 @@ static int h264_metadata_filter(AVBSFContext *bsf, AVPacket *pkt)
     // If an AUD is present, it must be the first NAL unit.
     if (au->units[0].type == H264_NAL_AUD) {
         if (ctx->aud == REMOVE)
-            ff_cbs_delete_unit(ctx->cbc, au, 0);
+            ff_cbs_delete_unit(au, 0);
     } else {
         if (ctx->aud == INSERT) {
             static const int primary_pic_type_table[] = {
@@ -335,7 +391,7 @@ static int h264_metadata_filter(AVBSFContext *bsf, AVPacket *pkt)
                 .primary_pic_type = j,
             };
 
-            err = ff_cbs_insert_unit_content(ctx->cbc, au,
+            err = ff_cbs_insert_unit_content(au,
                                              0, H264_NAL_AUD, &aud, NULL);
             if (err < 0) {
                 av_log(bsf, AV_LOG_ERROR, "Failed to insert AUD.\n");
@@ -374,7 +430,7 @@ static int h264_metadata_filter(AVBSFContext *bsf, AVPacket *pkt)
             } else {
                 goto invalid_user_data;
             }
-            if (i & 1)
+            if (j & 1)
                 udu->uuid_iso_iec_11578[j / 2] |= v;
             else
                 udu->uuid_iso_iec_11578[j / 2] = v << 4;
@@ -393,7 +449,7 @@ static int h264_metadata_filter(AVBSFContext *bsf, AVPacket *pkt)
             udu->data_length = len + 1;
             memcpy(udu->data, ctx->sei_user_data + i + 1, len + 1);
 
-            err = ff_cbs_h264_add_sei_message(ctx->cbc, au, &payload);
+            err = ff_cbs_h264_add_sei_message(au, &payload);
             if (err < 0) {
                 av_log(bsf, AV_LOG_ERROR, "Failed to add user data SEI "
                        "message to access unit.\n");
@@ -412,7 +468,7 @@ static int h264_metadata_filter(AVBSFContext *bsf, AVPacket *pkt)
     if (ctx->delete_filler) {
         for (i = au->nb_units - 1; i >= 0; i--) {
             if (au->units[i].type == H264_NAL_FILLER_DATA) {
-                ff_cbs_delete_unit(ctx->cbc, au, i);
+                ff_cbs_delete_unit(au, i);
                 continue;
             }
 
@@ -423,8 +479,7 @@ static int h264_metadata_filter(AVBSFContext *bsf, AVPacket *pkt)
                 for (j = sei->payload_count - 1; j >= 0; j--) {
                     if (sei->payload[j].payload_type ==
                         H264_SEI_TYPE_FILLER_PAYLOAD)
-                        ff_cbs_h264_delete_sei_message(ctx->cbc, au,
-                                                       &au->units[i], j);
+                        ff_cbs_h264_delete_sei_message(au, &au->units[i], j);
                 }
             }
         }
@@ -448,8 +503,7 @@ static int h264_metadata_filter(AVBSFContext *bsf, AVPacket *pkt)
 
                 if (ctx->display_orientation == REMOVE ||
                     ctx->display_orientation == INSERT) {
-                    ff_cbs_h264_delete_sei_message(ctx->cbc, au,
-                                                   &au->units[i], j);
+                    ff_cbs_h264_delete_sei_message(au, &au->units[i], j);
                     continue;
                 }
 
@@ -473,7 +527,7 @@ static int h264_metadata_filter(AVBSFContext *bsf, AVPacket *pkt)
                 if (err < 0) {
                     av_log(bsf, AV_LOG_ERROR, "Failed to attach extracted "
                            "displaymatrix side data to packet.\n");
-                    av_freep(matrix);
+                    av_free(matrix);
                     goto fail;
                 }
             }
@@ -540,7 +594,7 @@ static int h264_metadata_filter(AVBSFContext *bsf, AVPacket *pkt)
         if (write) {
             disp->display_orientation_repetition_period = 1;
 
-            err = ff_cbs_h264_add_sei_message(ctx->cbc, au, &payload);
+            err = ff_cbs_h264_add_sei_message(au, &payload);
             if (err < 0) {
                 av_log(bsf, AV_LOG_ERROR, "Failed to add display orientation "
                        "SEI message to access unit.\n");
@@ -549,7 +603,7 @@ static int h264_metadata_filter(AVBSFContext *bsf, AVPacket *pkt)
         }
     }
 
-    err = ff_cbs_write_packet(ctx->cbc, pkt, au);
+    err = ff_cbs_write_packet(ctx->output, pkt, au);
     if (err < 0) {
         av_log(bsf, AV_LOG_ERROR, "Failed to write packet.\n");
         goto fail;
@@ -559,7 +613,7 @@ static int h264_metadata_filter(AVBSFContext *bsf, AVPacket *pkt)
 
     err = 0;
 fail:
-    ff_cbs_fragment_reset(ctx->cbc, au);
+    ff_cbs_fragment_reset(au);
 
     if (err < 0)
         av_packet_unref(pkt);
@@ -573,12 +627,15 @@ static int h264_metadata_init(AVBSFContext *bsf)
     CodedBitstreamFragment *au = &ctx->access_unit;
     int err, i;
 
-    err = ff_cbs_init(&ctx->cbc, AV_CODEC_ID_H264, bsf);
+    err = ff_cbs_init(&ctx->input,  AV_CODEC_ID_H264, bsf);
+    if (err < 0)
+        return err;
+    err = ff_cbs_init(&ctx->output, AV_CODEC_ID_H264, bsf);
     if (err < 0)
         return err;
 
     if (bsf->par_in->extradata) {
-        err = ff_cbs_read_extradata(ctx->cbc, au, bsf->par_in);
+        err = ff_cbs_read_extradata(ctx->input, au, bsf->par_in);
         if (err < 0) {
             av_log(bsf, AV_LOG_ERROR, "Failed to read extradata.\n");
             goto fail;
@@ -592,7 +649,7 @@ static int h264_metadata_init(AVBSFContext *bsf)
             }
         }
 
-        err = ff_cbs_write_extradata(ctx->cbc, bsf->par_out, au);
+        err = ff_cbs_write_extradata(ctx->output, bsf->par_out, au);
         if (err < 0) {
             av_log(bsf, AV_LOG_ERROR, "Failed to write extradata.\n");
             goto fail;
@@ -601,7 +658,7 @@ static int h264_metadata_init(AVBSFContext *bsf)
 
     err = 0;
 fail:
-    ff_cbs_fragment_reset(ctx->cbc, au);
+    ff_cbs_fragment_reset(au);
     return err;
 }
 
@@ -609,8 +666,9 @@ static void h264_metadata_close(AVBSFContext *bsf)
 {
     H264MetadataContext *ctx = bsf->priv_data;
 
-    ff_cbs_fragment_free(ctx->cbc, &ctx->access_unit);
-    ff_cbs_close(&ctx->cbc);
+    ff_cbs_fragment_free(&ctx->access_unit);
+    ff_cbs_close(&ctx->input);
+    ff_cbs_close(&ctx->output);
 }
 
 #define OFFSET(x) offsetof(H264MetadataContext, x)
@@ -630,6 +688,10 @@ static const AVOption h264_metadata_options[] = {
         OFFSET(sample_aspect_ratio), AV_OPT_TYPE_RATIONAL,
         { .dbl = 0.0 }, 0, 65535, FLAGS },
 
+    { "overscan_appropriate_flag", "Set VUI overscan appropriate flag",
+        OFFSET(overscan_appropriate_flag), AV_OPT_TYPE_INT,
+        { .i64 = -1 }, -1, 1, FLAGS },
+
     { "video_format", "Set video format (table E-2)",
         OFFSET(video_format), AV_OPT_TYPE_INT,
         { .i64 = -1 }, -1, 7, FLAGS},