rans_output[rans_offset + 3] = uint8_t(rans >> 24);
}
-void encode_coeff(uint coeff, uint bits, uint sign_bias, inout RansEncoder enc)
+int sign_extend(uint coeff, uint bits)
+{
+ return int(coeff << (32 - bits)) >> (32 - bits);
+}
+
+void encode_coeff(int signed_k, uint sign_bias, inout RansEncoder enc)
{
- // Sign-extend to recover the coefficient.
- // FIXME: not needed for the bits == 8 case!
- int signed_k = int(coeff << (32 - bits)) >> (32 - bits);
uint k = abs(signed_k);
if (k >= ESCAPE_LIMIT) {
// ... boring stuff here
- RansEncPut(enc.rans, enc.rans_offset, k, 1, prob_bits);
+ RansEncPut(enc.rans, enc.rans_offset, k, 1, prob_bits - 1);
k = ESCAPE_LIMIT;
}
uint y = (streamgroup_num * BLOCKS_PER_STREAM + subblock_idx) / 160;
uint f = imageLoad(tex, ivec2(x, y * 8 + coeff_row)).x;
- encode_coeff(f & 0x1ffu, 9, sign_bias1, enc1);
- encode_coeff(f >> 9, 7, sign_bias2, enc2);
+ encode_coeff(sign_extend(f & 0x1ffu, 9), sign_bias1, enc1);
+ encode_coeff(sign_extend(f >> 9, 7), sign_bias2, enc2);
}
encode_end(enc1);
uint y = (streamgroup_num * BLOCKS_PER_STREAM + subblock_idx) / 160;
int f = imageLoad(tex, ivec2(x, y * 8 + coeff_row)).x;
- encode_coeff(f, 8, sign_bias, enc);
+ encode_coeff(f, sign_bias, enc);
}
encode_end(enc);