X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=libavcodec%2Fsmacker.c;h=ffd816f72131243572a120b06e45d34883a86355;hb=728c4658563dc82115ade0f1679679eddb7be5ff;hp=32ee39a2d968a474db8d5d1920b5bdf542b568dc;hpb=9676ffba8346791f494451e68d2a3b37a2918a9b;p=ffmpeg diff --git a/libavcodec/smacker.c b/libavcodec/smacker.c index 32ee39a2d96..ffd816f7213 100644 --- a/libavcodec/smacker.c +++ b/libavcodec/smacker.c @@ -31,10 +31,12 @@ #include #include +#include "libavutil/channel_layout.h" #include "avcodec.h" -#include "libavutil/audioconvert.h" +#include "internal.h" +#include "mathops.h" -#define ALT_BITSTREAM_READER_LE +#define BITSTREAM_READER_LE #include "get_bits.h" #include "bytestream.h" @@ -127,18 +129,16 @@ static int smacker_decode_tree(GetBitContext *gb, HuffContext *hc, uint32_t pref */ static int smacker_decode_bigtree(GetBitContext *gb, HuffContext *hc, DBCtx *ctx) { + if (hc->current + 1 >= hc->length) { + av_log(NULL, AV_LOG_ERROR, "Tree size exceeded!\n"); + return -1; + } if(!get_bits1(gb)){ //Leaf - int val, i1, i2, b1, b2; - if(hc->current >= hc->length){ - av_log(NULL, AV_LOG_ERROR, "Tree size exceeded!\n"); - return -1; - } - b1 = get_bits_count(gb); + int val, i1, i2; i1 = ctx->v1->table ? get_vlc2(gb, ctx->v1->table, SMKTREE_BITS, 3) : 0; - b1 = get_bits_count(gb) - b1; - b2 = get_bits_count(gb); i2 = ctx->v2->table ? get_vlc2(gb, ctx->v2->table, SMKTREE_BITS, 3) : 0; - b2 = get_bits_count(gb) - b2; + if (i1 < 0 || i2 < 0) + return -1; val = ctx->recode1[i1] | (ctx->recode2[i2] << 8); if(val == ctx->escapes[0]) { ctx->last[0] = hc->current; @@ -154,7 +154,7 @@ static int smacker_decode_bigtree(GetBitContext *gb, HuffContext *hc, DBCtx *ctx hc->values[hc->current++] = val; return 1; } else { //Node - int r = 0, t; + int r = 0, r_new, t; t = hc->current++; r = smacker_decode_bigtree(gb, hc, ctx); @@ -162,8 +162,10 @@ static int smacker_decode_bigtree(GetBitContext *gb, HuffContext *hc, DBCtx *ctx return r; hc->values[t] = SMK_NODE | r; r++; - r += smacker_decode_bigtree(gb, hc, ctx); - return r; + r_new = smacker_decode_bigtree(gb, hc, ctx); + if (r_new < 0) + return r_new; + return r + r_new; } } @@ -175,9 +177,10 @@ static int smacker_decode_header_tree(SmackVContext *smk, GetBitContext *gb, int int res; HuffContext huff; HuffContext tmp1, tmp2; - VLC vlc[2]; + VLC vlc[2] = { { 0 } }; int escapes[3]; DBCtx ctx; + int err = 0; if(size >= UINT_MAX>>4){ // (((size + 3) >> 2) + 3) << 2 must not overflow av_log(smk->avctx, AV_LOG_ERROR, "size too large\n"); @@ -197,9 +200,11 @@ static int smacker_decode_header_tree(SmackVContext *smk, GetBitContext *gb, int tmp2.bits = av_mallocz(256 * 4); tmp2.lengths = av_mallocz(256 * sizeof(int)); tmp2.values = av_mallocz(256 * sizeof(int)); - - memset(&vlc[0], 0, sizeof(VLC)); - memset(&vlc[1], 0, sizeof(VLC)); + if (!tmp1.bits || !tmp1.lengths || !tmp1.values || + !tmp2.bits || !tmp2.lengths || !tmp2.values) { + err = AVERROR(ENOMEM); + goto error; + } if(get_bits1(gb)) { smacker_decode_tree(gb, &tmp1, 0, 0); @@ -209,7 +214,8 @@ static int smacker_decode_header_tree(SmackVContext *smk, GetBitContext *gb, int tmp1.bits, sizeof(uint32_t), sizeof(uint32_t), INIT_VLC_LE); if(res < 0) { av_log(smk->avctx, AV_LOG_ERROR, "Cannot build VLC table\n"); - return -1; + err = res; + goto error; } } else { av_log(smk->avctx, AV_LOG_ERROR, "Skipping low bytes tree\n"); @@ -222,7 +228,8 @@ static int smacker_decode_header_tree(SmackVContext *smk, GetBitContext *gb, int tmp2.bits, sizeof(uint32_t), sizeof(uint32_t), INIT_VLC_LE); if(res < 0) { av_log(smk->avctx, AV_LOG_ERROR, "Cannot build VLC table\n"); - return -1; + err = res; + goto error; } } else { av_log(smk->avctx, AV_LOG_ERROR, "Skipping high bytes tree\n"); @@ -246,23 +253,35 @@ static int smacker_decode_header_tree(SmackVContext *smk, GetBitContext *gb, int ctx.recode2 = tmp2.values; ctx.last = last; - huff.length = ((size + 3) >> 2) + 3; + huff.length = ((size + 3) >> 2) + 4; huff.maxlength = 0; huff.current = 0; huff.values = av_mallocz(huff.length * sizeof(int)); + if (!huff.values) { + err = AVERROR(ENOMEM); + goto error; + } - smacker_decode_bigtree(gb, &huff, &ctx); + if (smacker_decode_bigtree(gb, &huff, &ctx) < 0) + err = -1; skip_bits1(gb); if(ctx.last[0] == -1) ctx.last[0] = huff.current++; if(ctx.last[1] == -1) ctx.last[1] = huff.current++; if(ctx.last[2] == -1) ctx.last[2] = huff.current++; + if (ctx.last[0] >= huff.length || + ctx.last[1] >= huff.length || + ctx.last[2] >= huff.length) { + av_log(smk->avctx, AV_LOG_ERROR, "Huffman codes out of range\n"); + err = AVERROR_INVALIDDATA; + } *recodes = huff.values; +error: if(vlc[0].table) - free_vlc(&vlc[0]); + ff_free_vlc(&vlc[0]); if(vlc[1].table) - free_vlc(&vlc[1]); + ff_free_vlc(&vlc[1]); av_free(tmp1.bits); av_free(tmp1.lengths); av_free(tmp1.values); @@ -270,7 +289,7 @@ static int smacker_decode_header_tree(SmackVContext *smk, GetBitContext *gb, int av_free(tmp2.lengths); av_free(tmp2.values); - return 0; + return err; } static int decode_header_trees(SmackVContext *smk) { @@ -287,6 +306,8 @@ static int decode_header_trees(SmackVContext *smk) { if(!get_bits1(&gb)) { av_log(smk->avctx, AV_LOG_INFO, "Skipping MMAP tree\n"); smk->mmap_tbl = av_malloc(sizeof(int) * 2); + if (!smk->mmap_tbl) + return AVERROR(ENOMEM); smk->mmap_tbl[0] = 0; smk->mmap_last[0] = smk->mmap_last[1] = smk->mmap_last[2] = 1; } else { @@ -296,6 +317,8 @@ static int decode_header_trees(SmackVContext *smk) { if(!get_bits1(&gb)) { av_log(smk->avctx, AV_LOG_INFO, "Skipping MCLR tree\n"); smk->mclr_tbl = av_malloc(sizeof(int) * 2); + if (!smk->mclr_tbl) + return AVERROR(ENOMEM); smk->mclr_tbl[0] = 0; smk->mclr_last[0] = smk->mclr_last[1] = smk->mclr_last[2] = 1; } else { @@ -305,6 +328,8 @@ static int decode_header_trees(SmackVContext *smk) { if(!get_bits1(&gb)) { av_log(smk->avctx, AV_LOG_INFO, "Skipping FULL tree\n"); smk->full_tbl = av_malloc(sizeof(int) * 2); + if (!smk->full_tbl) + return AVERROR(ENOMEM); smk->full_tbl[0] = 0; smk->full_last[0] = smk->full_last[1] = smk->full_last[2] = 1; } else { @@ -314,6 +339,8 @@ static int decode_header_trees(SmackVContext *smk) { if(!get_bits1(&gb)) { av_log(smk->avctx, AV_LOG_INFO, "Skipping TYPE tree\n"); smk->type_tbl = av_malloc(sizeof(int) * 2); + if (!smk->type_tbl) + return AVERROR(ENOMEM); smk->type_tbl[0] = 0; smk->type_last[0] = smk->type_last[1] = smk->type_last[2] = 1; } else { @@ -331,16 +358,14 @@ static av_always_inline void last_reset(int *recode, int *last) { /* get code and update history */ static av_always_inline int smk_get_code(GetBitContext *gb, int *recode, int *last) { register int *table = recode; - int v, b; + int v; - b = get_bits_count(gb); while(*table & SMK_NODE) { if(get_bits1(gb)) table += (*table) & (~SMK_NODE); table++; } v = *table; - b = get_bits_count(gb) - b; if(v != recode[last[0]]) { recode[last[2]] = recode[last[1]]; @@ -350,47 +375,46 @@ static av_always_inline int smk_get_code(GetBitContext *gb, int *recode, int *la return v; } -static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPacket *avpkt) +static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame, + AVPacket *avpkt) { - const uint8_t *buf = avpkt->data; - int buf_size = avpkt->size; SmackVContext * const smk = avctx->priv_data; uint8_t *out; uint32_t *pal; + GetByteContext gb2; GetBitContext gb; int blocks, blk, bw, bh; - int i; + int i, ret; int stride; + int flags; - if(buf_size <= 769) + if (avpkt->size <= 769) return 0; - smk->pic.reference = 1; - smk->pic.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE; - if(avctx->reget_buffer(avctx, &smk->pic) < 0){ + if ((ret = ff_reget_buffer(avctx, &smk->pic)) < 0) { av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n"); - return -1; + return ret; } /* make the palette available on the way out */ pal = (uint32_t*)smk->pic.data[1]; - smk->pic.palette_has_changed = buf[0] & 1; - smk->pic.key_frame = !!(buf[0] & 2); + bytestream2_init(&gb2, avpkt->data, avpkt->size); + flags = bytestream2_get_byteu(&gb2); + smk->pic.palette_has_changed = flags & 1; + smk->pic.key_frame = !!(flags & 2); if(smk->pic.key_frame) smk->pic.pict_type = AV_PICTURE_TYPE_I; else smk->pic.pict_type = AV_PICTURE_TYPE_P; - buf++; for(i = 0; i < 256; i++) - *pal++ = bytestream_get_be24(&buf); - buf_size -= 769; + *pal++ = bytestream2_get_be24u(&gb2); last_reset(smk->mmap_tbl, smk->mmap_last); last_reset(smk->mclr_tbl, smk->mclr_last); last_reset(smk->full_tbl, smk->full_last); last_reset(smk->type_tbl, smk->type_last); - init_get_bits(&gb, buf, buf_size * 8); + init_get_bits(&gb, avpkt->data + 769, (avpkt->size - 769) * 8); blk = 0; bw = avctx->width >> 2; @@ -497,15 +521,37 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac } - *data_size = sizeof(AVFrame); - *(AVFrame*)data = smk->pic; + if ((ret = av_frame_ref(data, &smk->pic)) < 0) + return ret; + + *got_frame = 1; /* always report that the buffer was completely consumed */ - return buf_size; + return avpkt->size; } +/* + * + * Uninit smacker decoder + * + */ +static av_cold int decode_end(AVCodecContext *avctx) +{ + SmackVContext * const smk = avctx->priv_data; + + av_freep(&smk->mmap_tbl); + av_freep(&smk->mclr_tbl); + av_freep(&smk->full_tbl); + av_freep(&smk->type_tbl); + + av_frame_unref(&smk->pic); + + return 0; +} + + /* * * Init smacker decoder @@ -517,8 +563,8 @@ static av_cold int decode_init(AVCodecContext *avctx) c->avctx = avctx; - avctx->pix_fmt = PIX_FMT_PAL8; - + avctx->pix_fmt = AV_PIX_FMT_PAL8; + avcodec_get_frame_defaults(&c->pic); /* decode huffman trees from extradata */ if(avctx->extradata_size < 16){ @@ -526,78 +572,82 @@ static av_cold int decode_init(AVCodecContext *avctx) return -1; } - if (decode_header_trees(c)) + if (decode_header_trees(c)) { + decode_end(avctx); return -1; + } return 0; } -/* - * - * Uninit smacker decoder - * - */ -static av_cold int decode_end(AVCodecContext *avctx) -{ - SmackVContext * const smk = avctx->priv_data; - - av_freep(&smk->mmap_tbl); - av_freep(&smk->mclr_tbl); - av_freep(&smk->full_tbl); - av_freep(&smk->type_tbl); - - if (smk->pic.data[0]) - avctx->release_buffer(avctx, &smk->pic); - - return 0; -} - - static av_cold int smka_decode_init(AVCodecContext *avctx) { + if (avctx->channels < 1 || avctx->channels > 2) { + av_log(avctx, AV_LOG_ERROR, "invalid number of channels\n"); + return AVERROR(EINVAL); + } avctx->channel_layout = (avctx->channels==2) ? AV_CH_LAYOUT_STEREO : AV_CH_LAYOUT_MONO; avctx->sample_fmt = avctx->bits_per_coded_sample == 8 ? AV_SAMPLE_FMT_U8 : AV_SAMPLE_FMT_S16; + return 0; } /** * Decode Smacker audio data */ -static int smka_decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPacket *avpkt) +static int smka_decode_frame(AVCodecContext *avctx, void *data, + int *got_frame_ptr, AVPacket *avpkt) { + AVFrame *frame = data; const uint8_t *buf = avpkt->data; int buf_size = avpkt->size; GetBitContext gb; - HuffContext h[4]; - VLC vlc[4]; - int16_t *samples = data; - int8_t *samples8 = data; + HuffContext h[4] = { { 0 } }; + VLC vlc[4] = { { 0 } }; + int16_t *samples; + uint8_t *samples8; int val; - int i, res; + int i, res, ret; int unp_size; int bits, stereo; int pred[2] = {0, 0}; + if (buf_size <= 4) { + av_log(avctx, AV_LOG_ERROR, "packet is too small\n"); + return AVERROR(EINVAL); + } + unp_size = AV_RL32(buf); init_get_bits(&gb, buf + 4, (buf_size - 4) * 8); if(!get_bits1(&gb)){ av_log(avctx, AV_LOG_INFO, "Sound: no data\n"); - *data_size = 0; + *got_frame_ptr = 0; return 1; } stereo = get_bits1(&gb); bits = get_bits1(&gb); - if (unp_size & 0xC0000000 || unp_size > *data_size) { - av_log(avctx, AV_LOG_ERROR, "Frame is too large to fit in buffer\n"); - return -1; + if (stereo ^ (avctx->channels != 1)) { + av_log(avctx, AV_LOG_ERROR, "channels mismatch\n"); + return AVERROR(EINVAL); + } + if (bits && avctx->sample_fmt == AV_SAMPLE_FMT_U8) { + av_log(avctx, AV_LOG_ERROR, "sample format mismatch\n"); + return AVERROR(EINVAL); } - memset(vlc, 0, sizeof(VLC) * 4); - memset(h, 0, sizeof(HuffContext) * 4); + /* get output buffer */ + frame->nb_samples = unp_size / (avctx->channels * (bits + 1)); + if ((ret = ff_get_buffer(avctx, frame, 0)) < 0) { + av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n"); + return ret; + } + samples = (int16_t *)frame->data[0]; + samples8 = frame->data[0]; + // Initialize for(i = 0; i < (1 << (bits + stereo)); i++) { h[i].length = 256; @@ -606,8 +656,15 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data, int *data_size, h[i].bits = av_mallocz(256 * 4); h[i].lengths = av_mallocz(256 * sizeof(int)); h[i].values = av_mallocz(256 * sizeof(int)); + if (!h[i].bits || !h[i].lengths || !h[i].values) { + ret = AVERROR(ENOMEM); + goto error; + } skip_bits1(&gb); - smacker_decode_tree(&gb, &h[i], 0, 0); + if (smacker_decode_tree(&gb, &h[i], 0, 0) < 0) { + ret = AVERROR_INVALIDDATA; + goto error; + } skip_bits1(&gb); if(h[i].current > 1) { res = init_vlc(&vlc[i], SMKTREE_BITS, h[i].length, @@ -615,16 +672,18 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data, int *data_size, h[i].bits, sizeof(uint32_t), sizeof(uint32_t), INIT_VLC_LE); if(res < 0) { av_log(avctx, AV_LOG_ERROR, "Cannot build VLC table\n"); - return -1; + ret = AVERROR_INVALIDDATA; + goto error; } } } + /* this codec relies on wraparound instead of clipping audio */ if(bits) { //decode 16-bit data for(i = stereo; i >= 0; i--) - pred[i] = av_bswap16(get_bits(&gb, 16)); - for(i = 0; i < stereo; i++) + pred[i] = sign_extend(av_bswap16(get_bits(&gb, 16)), 16); + for(i = 0; i <= stereo; i++) *samples++ = pred[i]; - for(i = 0; i < unp_size / 2; i++) { + for(; i < unp_size / 2; i++) { if(i & stereo) { if(vlc[2].table) res = get_vlc2(&gb, vlc[2].table, SMKTREE_BITS, 3); @@ -636,7 +695,7 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data, int *data_size, else res = 0; val |= h[3].values[res] << 8; - pred[1] += (int16_t)val; + pred[1] += sign_extend(val, 16); *samples++ = pred[1]; } else { if(vlc[0].table) @@ -649,64 +708,67 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data, int *data_size, else res = 0; val |= h[1].values[res] << 8; - pred[0] += val; + pred[0] += sign_extend(val, 16); *samples++ = pred[0]; } } } else { //8-bit data for(i = stereo; i >= 0; i--) pred[i] = get_bits(&gb, 8); - for(i = 0; i < stereo; i++) + for(i = 0; i <= stereo; i++) *samples8++ = pred[i]; - for(i = 0; i < unp_size; i++) { + for(; i < unp_size; i++) { if(i & stereo){ if(vlc[1].table) res = get_vlc2(&gb, vlc[1].table, SMKTREE_BITS, 3); else res = 0; - pred[1] += (int8_t)h[1].values[res]; + pred[1] += sign_extend(h[1].values[res], 8); *samples8++ = pred[1]; } else { if(vlc[0].table) res = get_vlc2(&gb, vlc[0].table, SMKTREE_BITS, 3); else res = 0; - pred[0] += (int8_t)h[0].values[res]; + pred[0] += sign_extend(h[0].values[res], 8); *samples8++ = pred[0]; } } } + *got_frame_ptr = 1; + ret = buf_size; + +error: for(i = 0; i < 4; i++) { if(vlc[i].table) - free_vlc(&vlc[i]); + ff_free_vlc(&vlc[i]); av_free(h[i].bits); av_free(h[i].lengths); av_free(h[i].values); } - *data_size = unp_size; - return buf_size; + return ret; } AVCodec ff_smacker_decoder = { .name = "smackvid", + .long_name = NULL_IF_CONFIG_SMALL("Smacker video"), .type = AVMEDIA_TYPE_VIDEO, - .id = CODEC_ID_SMACKVIDEO, + .id = AV_CODEC_ID_SMACKVIDEO, .priv_data_size = sizeof(SmackVContext), .init = decode_init, .close = decode_end, .decode = decode_frame, .capabilities = CODEC_CAP_DR1, - .long_name = NULL_IF_CONFIG_SMALL("Smacker video"), }; AVCodec ff_smackaud_decoder = { .name = "smackaud", + .long_name = NULL_IF_CONFIG_SMALL("Smacker audio"), .type = AVMEDIA_TYPE_AUDIO, - .id = CODEC_ID_SMACKAUDIO, + .id = AV_CODEC_ID_SMACKAUDIO, .init = smka_decode_init, .decode = smka_decode_frame, - .long_name = NULL_IF_CONFIG_SMALL("Smacker audio"), + .capabilities = CODEC_CAP_DR1, }; -