Fix the DCT scaling (I believe).

diff --git a/encoder.shader b/encoder.shader
index 54103a093d374cca0fb2fcda37a7add308574268..48c75e645829cabe554894c142cb7629c7d48c7a 100644 (file)
--- a/encoder.shader
+++ b/encoder.shader
@@ -12,6 +12,12 @@ layout(r8ui) uniform restrict readonly uimage2D image_tex;
 
 shared float temp[64];
 
 
 shared float temp[64];
 

+// Scale factors; 1.0 / (sqrt(2.0) * cos(k * M_PI / 16.0)), except for the first which is 1.
+const float sf[8] = {
+       1.0, 0.7209598220069479, 0.765366864730180, 0.8504300947672564,
+       1.0, 1.2727585805728336, 1.847759065022573, 3.6245097854115502
+};
+

 const float W[64] = {
          8, 16, 19, 22, 26, 27, 29, 34,
         16, 16, 22, 24, 27, 29, 34, 37,
 const float W[64] = {
          8, 16, 19, 22, 26, 27, 29, 34,
         16, 16, 22, 24, 27, 29, 34, 37,


@@ -22,18 +28,18 @@ const float W[64] = {
         26, 27, 29, 34, 38, 46, 56, 69,
         27, 29, 35, 38, 46, 56, 69, 83
 };
         26, 27, 29, 34, 38, 46, 56, 69,
         27, 29, 35, 38, 46, 56, 69, 83
 };

-const float S = 4.0;  // whatever?
+const float S = 4.0 * 0.5;  // whatever?

 
 // NOTE: Contains factors to counteract the scaling in the DCT implementation.
 const float quant_matrix[64] = {
 
 // NOTE: Contains factors to counteract the scaling in the DCT implementation.
 const float quant_matrix[64] = {

-        1.0 / 64.0,         1.0 / (W[ 1] * S),  1.0 / (W[ 2] * S),  1.0 / (W[ 3] * S),  1.0 / (W[ 4] * S),  1.0 / (W[ 5] * S),  1.0 / (W[ 6] * S),  1.0 / (W[ 7] * S),
-        1.0 / (W[ 8] * S),  2.0 / (W[ 9] * S),  2.0 / (W[10] * S),  2.0 / (W[11] * S),  2.0 / (W[12] * S),  2.0 / (W[13] * S),  2.0 / (W[14] * S),  2.0 / (W[15] * S),
-        1.0 / (W[16] * S),  2.0 / (W[17] * S),  2.0 / (W[18] * S),  2.0 / (W[19] * S),  2.0 / (W[20] * S),  2.0 / (W[21] * S),  2.0 / (W[22] * S),  2.0 / (W[23] * S),
-        1.0 / (W[24] * S),  2.0 / (W[25] * S),  2.0 / (W[26] * S),  2.0 / (W[27] * S),  2.0 / (W[28] * S),  2.0 / (W[29] * S),  2.0 / (W[30] * S),  2.0 / (W[31] * S),
-        1.0 / (W[32] * S),  2.0 / (W[33] * S),  2.0 / (W[34] * S),  2.0 / (W[35] * S),  2.0 / (W[36] * S),  2.0 / (W[37] * S),  2.0 / (W[38] * S),  2.0 / (W[39] * S),
-        1.0 / (W[40] * S),  2.0 / (W[41] * S),  2.0 / (W[42] * S),  2.0 / (W[43] * S),  2.0 / (W[44] * S),  2.0 / (W[45] * S),  2.0 / (W[46] * S),  2.0 / (W[47] * S),
-        1.0 / (W[48] * S),  2.0 / (W[49] * S),  2.0 / (W[50] * S),  2.0 / (W[51] * S),  2.0 / (W[52] * S),  2.0 / (W[53] * S),  2.0 / (W[54] * S),  2.0 / (W[55] * S),
-        1.0 / (W[56] * S),  2.0 / (W[57] * S),  2.0 / (W[58] * S),  2.0 / (W[59] * S),  2.0 / (W[60] * S),  2.0 / (W[61] * S),  2.0 / (W[62] * S),  2.0 / (W[63] * S)
+        sf[0] * sf[0] / 64.0,         sf[1] * sf[0] / (W[ 1] * S),  sf[2] * sf[0] / (W[ 2] * S),  sf[3] * sf[0] / (W[ 3] * S),  sf[4] * sf[0] / (W[ 4] * S),  sf[5] * sf[0] / (W[ 5] * S),  sf[6] * sf[0] / (W[ 6] * S),  sf[7] * sf[0] / (W[ 7] * S),
+        sf[0] * sf[1] / (W[ 8] * S),  sf[1] * sf[1] / (W[ 9] * S),  sf[2] * sf[1] / (W[10] * S),  sf[3] * sf[1] / (W[11] * S),  sf[4] * sf[1] / (W[12] * S),  sf[5] * sf[1] / (W[13] * S),  sf[6] * sf[1] / (W[14] * S),  sf[7] * sf[1] / (W[15] * S),
+        sf[0] * sf[2] / (W[16] * S),  sf[1] * sf[2] / (W[17] * S),  sf[2] * sf[2] / (W[18] * S),  sf[3] * sf[2] / (W[19] * S),  sf[4] * sf[2] / (W[20] * S),  sf[5] * sf[2] / (W[21] * S),  sf[6] * sf[2] / (W[22] * S),  sf[7] * sf[2] / (W[23] * S),
+        sf[0] * sf[3] / (W[24] * S),  sf[1] * sf[3] / (W[25] * S),  sf[2] * sf[3] / (W[26] * S),  sf[3] * sf[3] / (W[27] * S),  sf[4] * sf[3] / (W[28] * S),  sf[5] * sf[3] / (W[29] * S),  sf[6] * sf[3] / (W[30] * S),  sf[7] * sf[3] / (W[31] * S),
+        sf[0] * sf[4] / (W[32] * S),  sf[1] * sf[4] / (W[33] * S),  sf[2] * sf[4] / (W[34] * S),  sf[3] * sf[4] / (W[35] * S),  sf[4] * sf[4] / (W[36] * S),  sf[5] * sf[4] / (W[37] * S),  sf[6] * sf[4] / (W[38] * S),  sf[7] * sf[4] / (W[39] * S),
+        sf[0] * sf[5] / (W[40] * S),  sf[1] * sf[5] / (W[41] * S),  sf[2] * sf[5] / (W[42] * S),  sf[3] * sf[5] / (W[43] * S),  sf[4] * sf[5] / (W[44] * S),  sf[5] * sf[5] / (W[45] * S),  sf[6] * sf[5] / (W[46] * S),  sf[7] * sf[5] / (W[47] * S),
+        sf[0] * sf[6] / (W[48] * S),  sf[1] * sf[6] / (W[49] * S),  sf[2] * sf[6] / (W[50] * S),  sf[3] * sf[6] / (W[51] * S),  sf[4] * sf[6] / (W[52] * S),  sf[5] * sf[6] / (W[53] * S),  sf[6] * sf[6] / (W[54] * S),  sf[7] * sf[6] / (W[55] * S),
+        sf[0] * sf[7] / (W[56] * S),  sf[1] * sf[7] / (W[57] * S),  sf[2] * sf[7] / (W[58] * S),  sf[3] * sf[7] / (W[59] * S),  sf[4] * sf[7] / (W[60] * S),  sf[5] * sf[7] / (W[61] * S),  sf[6] * sf[7] / (W[62] * S),  sf[7] * sf[7] / (W[63] * S)

 };
 
 // Clamp and pack a 9-bit and a 7-bit signed value into a 16-bit word.
 };
 
 // Clamp and pack a 9-bit and a 7-bit signed value into a 16-bit word.


@@ -42,7 +48,7 @@ uint pack_9_7(int v9, int v7)
        return (uint(clamp(v9, -256, 255)) & 0x1ffu) | ((uint(clamp(v7, -64, 63)) & 0x7fu) << 9);
 }
 
        return (uint(clamp(v9, -256, 255)) & 0x1ffu) | ((uint(clamp(v7, -64, 63)) & 0x7fu) << 9);
 }
 

-// Scaled 1D DCT. y0 output is scaled by 8, everything else is scaled by 16.
+// Scaled 1D DCT (AA&N). y0 is correctly scaled, all other y_k are scaled by sqrt(2) cos(k * Pi / 16).

 void dct_1d(inout float y0, inout float y1, inout float y2, inout float y3, inout float y4, inout float y5, inout float y6, inout float y7)
 {
        const float a1 = 0.7071067811865474;   // sqrt(2)
 void dct_1d(inout float y0, inout float y1, inout float y2, inout float y3, inout float y4, inout float y5, inout float y6, inout float y7)
 {
        const float a1 = 0.7071067811865474;   // sqrt(2)