avcodec/cfhd: improve decompanding quality with reference implementation

diff --git a/libavcodec/cfhd.c b/libavcodec/cfhd.c
index aa6e2ebee783e5a0ae47b3471b717fff62e77d91..0327b456ac09b81f4f4e8415ace53ef3c0560a23 100644 (file)
--- a/libavcodec/cfhd.c
+++ b/libavcodec/cfhd.c
@@ -117,8 +117,8 @@ static inline int dequant_and_decompand(int level, int quantisation, int codeboo
 {
     if (codebook == 0 || codebook == 1) {
         int64_t abslevel = abs(level);
-        if (level < 264)
-            return (abslevel + ((768 * abslevel * abslevel * abslevel) / (255 * 255 * 255))) *
+        if (abslevel < 256)
+            return (abslevel + ((768 * abslevel * abslevel * abslevel) / (256 * 256 * 256))) *
                FFSIGN(level) * quantisation;
         else
             return level * quantisation;

diff --git a/tests/ref/fate/cfhd-3 b/tests/ref/fate/cfhd-3
index 59fdc9226090d14b505d6d68869db0dfca239861..79133a14b025993e324fd2d86d2a425b3a2d4181 100644 (file)
--- a/tests/ref/fate/cfhd-3
+++ b/tests/ref/fate/cfhd-3
@@ -3,13 +3,13 @@
 #codec_id 0: rawvideo
 #dimensions 0: 496x241
 #sar 0: 0/1
-0,          0,          0,        1,   478144, 0x48a01dbb
-0,          1,          1,        1,   478144, 0x48a01dbb
-0,          2,          2,        1,   478144, 0x48a01dbb
-0,          3,          3,        1,   478144, 0xb978a72f
-0,          4,          4,        1,   478144, 0x7bbb4679
-0,          5,          5,        1,   478144, 0xc3fd3f59
-0,          6,          6,        1,   478144, 0xfd2a4816
-0,          7,          7,        1,   478144, 0x207f65d3
-0,          8,          8,        1,   478144, 0x207f65d3
-0,          9,          9,        1,   478144, 0x207f65d3
+0,          0,          0,        1,   478144, 0x1e5a0d6c
+0,          1,          1,        1,   478144, 0x1e5a0d6c
+0,          2,          2,        1,   478144, 0x1e5a0d6c
+0,          3,          3,        1,   478144, 0x88788c7d
+0,          4,          4,        1,   478144, 0x78643db8
+0,          5,          5,        1,   478144, 0x84303909
+0,          6,          6,        1,   478144, 0x8ddd4828
+0,          7,          7,        1,   478144, 0xc0845d58
+0,          8,          8,        1,   478144, 0xc0845d58
+0,          9,          9,        1,   478144, 0xc0845d58