+static inline unsigned read_huffman_symbol(const struct huffman_table* table,
+ struct bit_source* source)
+{
+ possibly_refill(source, DEHUF_TABLE_BITS);
+ return read_huffman_symbol_no_refill(table, source);
+}
+
+// procedure EXTEND (figure F.12)
+
+// Fast lookup table for (1 << (bits - 1)).
+// The table actually helps, since the load can go in parallel with the shift
+// operation below.
+static const int bit_thresholds[16] = {
+ 0, 1 << 0, 1 << 1, 1 << 2, 1 << 3, 1 << 4, 1 << 5, 1 << 6, 1 << 7, 1 << 8, 1 << 9, 1 << 10, 1 << 11, 1 << 12, 1 << 13, 1 << 14
+};
+
+static inline unsigned extend(int val, unsigned bits)
+{
+#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
+ // GCC should ideally be able to figure out that the conditional move is better, but
+ // it doesn't for various reasons, and this is pretty important for speed, so we hardcode.
+ asm("cmp %2, %0 ; cmovl %3, %0"
+ : "=r" (val)
+ : "0" (val),
+ "g" (bit_thresholds[bits]),
+ "r" (val + (-1 << bits) + 1)
+ : "cc");
+ return val;
+#else
+ if (val < bit_thresholds[bits]) {
+ return val + (-1 << bits) + 1;
+ } else {
+ return val;
+ }
+#endif
+}
+