From 68af7105a45805cfd8d951313dbe35e678055b29 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Mon, 16 Oct 2017 23:58:36 +0200 Subject: [PATCH] Add more off-by-one fixes. --- rans.shader | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rans.shader b/rans.shader index 0c0bf8b..8e9c422 100644 --- a/rans.shader +++ b/rans.shader @@ -3,7 +3,7 @@ layout(local_size_x = 1) in; -const uint prob_bits = 13; // Note! +const uint prob_bits = 12; const uint prob_scale = 1 << prob_bits; const uint RANS_BYTE_L = (1u << 23); const uint BLOCKS_PER_STREAM = 320; @@ -98,12 +98,12 @@ void encode_coeff(int signed_k, uint sign_bias, inout RansEncoder enc) if (k >= ESCAPE_LIMIT) { // ... boring stuff here - RansEncPut(enc.rans, enc.rans_offset, k, 1, prob_bits - 1); + RansEncPut(enc.rans, enc.rans_offset, k, 1, prob_bits); k = ESCAPE_LIMIT; } - uvec2 sym = ransdist[enc.lut_base + ((k + NUM_SYMS - 1) & (NUM_SYMS - 1))]; - RansEncPut(enc.rans, enc.rans_offset, sym.x, sym.y, prob_bits); + uvec2 sym = ransdist[enc.lut_base + ((k - 1) & (NUM_SYMS - 1))]; + RansEncPut(enc.rans, enc.rans_offset, sym.x, sym.y, prob_bits + 1); if (signed_k < 0) { enc.rans += sign_bias; -- 2.39.2