Save some needless sign extending, and fix an escaping bug.

diff --git a/coded.dat b/coded.dat
index 3980407c36b3a939eda4e5c24eec4998427f9957..bfe60fb37829987db21a3e54e59eb651c1a2fa9e 100644 (file)
Binary files a/coded.dat and b/coded.dat differ

diff --git a/rans.shader b/rans.shader
index 36feb657544e0fbfcb19b4e955bad5c3031ecf8c..b77d2407adab28496e783a1953d76f043f312b51 100644 (file)
--- a/rans.shader
+++ b/rans.shader
@@ -87,16 +87,18 @@ void RansEncFlush(uint rans, inout uint rans_offset)
        rans_output[rans_offset + 3] = uint8_t(rans >> 24);
 }
 
-void encode_coeff(uint coeff, uint bits, uint sign_bias, inout RansEncoder enc)
+int sign_extend(uint coeff, uint bits)
+{
+       return int(coeff << (32 - bits)) >> (32 - bits);
+}
+
+void encode_coeff(int signed_k, uint sign_bias, inout RansEncoder enc)
 {
-       // Sign-extend to recover the coefficient.
-       // FIXME: not needed for the bits == 8 case!
-       int signed_k = int(coeff << (32 - bits)) >> (32 - bits);
        uint k = abs(signed_k);
 
        if (k >= ESCAPE_LIMIT) {
                // ... boring stuff here
-               RansEncPut(enc.rans, enc.rans_offset, k, 1, prob_bits);
+               RansEncPut(enc.rans, enc.rans_offset, k, 1, prob_bits - 1);
                k = ESCAPE_LIMIT;
        }
 
@@ -129,8 +131,8 @@ void encode_9_7(uint streamgroup_num, uint coeff_row, layout(r16ui) restrict rea
                uint y = (streamgroup_num * BLOCKS_PER_STREAM + subblock_idx) / 160;
                uint f = imageLoad(tex, ivec2(x, y * 8 + coeff_row)).x;
 
-               encode_coeff(f & 0x1ffu, 9, sign_bias1, enc1);
-               encode_coeff(f >> 9, 7, sign_bias2, enc2);
+               encode_coeff(sign_extend(f & 0x1ffu, 9), sign_bias1, enc1);
+               encode_coeff(sign_extend(f >> 9, 7), sign_bias2, enc2);
        }
 
        encode_end(enc1);
@@ -150,7 +152,7 @@ void encode_8(uint streamgroup_num, uint coeff_row, layout(r8i) restrict readonl
                uint y = (streamgroup_num * BLOCKS_PER_STREAM + subblock_idx) / 160;
                int f = imageLoad(tex, ivec2(x, y * 8 + coeff_row)).x;
 
-               encode_coeff(f, 8, sign_bias, enc);
+               encode_coeff(f, sign_bias, enc);
        }
 
        encode_end(enc);