From: Steinar H. Gunderson Date: Mon, 9 Oct 2017 21:11:44 +0000 (+0200) Subject: Fix the DCT scaling (I believe). X-Git-Url: https://git.sesse.net/?p=narabu;a=commitdiff_plain;h=929ed30933ca58ca55abeea2217756f6b0b9410e Fix the DCT scaling (I believe). --- diff --git a/encoder.shader b/encoder.shader index 54103a0..48c75e6 100644 --- a/encoder.shader +++ b/encoder.shader @@ -12,6 +12,12 @@ layout(r8ui) uniform restrict readonly uimage2D image_tex; shared float temp[64]; +// Scale factors; 1.0 / (sqrt(2.0) * cos(k * M_PI / 16.0)), except for the first which is 1. +const float sf[8] = { + 1.0, 0.7209598220069479, 0.765366864730180, 0.8504300947672564, + 1.0, 1.2727585805728336, 1.847759065022573, 3.6245097854115502 +}; + const float W[64] = { 8, 16, 19, 22, 26, 27, 29, 34, 16, 16, 22, 24, 27, 29, 34, 37, @@ -22,18 +28,18 @@ const float W[64] = { 26, 27, 29, 34, 38, 46, 56, 69, 27, 29, 35, 38, 46, 56, 69, 83 }; -const float S = 4.0; // whatever? +const float S = 4.0 * 0.5; // whatever? // NOTE: Contains factors to counteract the scaling in the DCT implementation. const float quant_matrix[64] = { - 1.0 / 64.0, 1.0 / (W[ 1] * S), 1.0 / (W[ 2] * S), 1.0 / (W[ 3] * S), 1.0 / (W[ 4] * S), 1.0 / (W[ 5] * S), 1.0 / (W[ 6] * S), 1.0 / (W[ 7] * S), - 1.0 / (W[ 8] * S), 2.0 / (W[ 9] * S), 2.0 / (W[10] * S), 2.0 / (W[11] * S), 2.0 / (W[12] * S), 2.0 / (W[13] * S), 2.0 / (W[14] * S), 2.0 / (W[15] * S), - 1.0 / (W[16] * S), 2.0 / (W[17] * S), 2.0 / (W[18] * S), 2.0 / (W[19] * S), 2.0 / (W[20] * S), 2.0 / (W[21] * S), 2.0 / (W[22] * S), 2.0 / (W[23] * S), - 1.0 / (W[24] * S), 2.0 / (W[25] * S), 2.0 / (W[26] * S), 2.0 / (W[27] * S), 2.0 / (W[28] * S), 2.0 / (W[29] * S), 2.0 / (W[30] * S), 2.0 / (W[31] * S), - 1.0 / (W[32] * S), 2.0 / (W[33] * S), 2.0 / (W[34] * S), 2.0 / (W[35] * S), 2.0 / (W[36] * S), 2.0 / (W[37] * S), 2.0 / (W[38] * S), 2.0 / (W[39] * S), - 1.0 / (W[40] * S), 2.0 / (W[41] * S), 2.0 / (W[42] * S), 2.0 / (W[43] * S), 2.0 / (W[44] * S), 2.0 / (W[45] * S), 2.0 / (W[46] * S), 2.0 / (W[47] * S), - 1.0 / (W[48] * S), 2.0 / (W[49] * S), 2.0 / (W[50] * S), 2.0 / (W[51] * S), 2.0 / (W[52] * S), 2.0 / (W[53] * S), 2.0 / (W[54] * S), 2.0 / (W[55] * S), - 1.0 / (W[56] * S), 2.0 / (W[57] * S), 2.0 / (W[58] * S), 2.0 / (W[59] * S), 2.0 / (W[60] * S), 2.0 / (W[61] * S), 2.0 / (W[62] * S), 2.0 / (W[63] * S) + sf[0] * sf[0] / 64.0, sf[1] * sf[0] / (W[ 1] * S), sf[2] * sf[0] / (W[ 2] * S), sf[3] * sf[0] / (W[ 3] * S), sf[4] * sf[0] / (W[ 4] * S), sf[5] * sf[0] / (W[ 5] * S), sf[6] * sf[0] / (W[ 6] * S), sf[7] * sf[0] / (W[ 7] * S), + sf[0] * sf[1] / (W[ 8] * S), sf[1] * sf[1] / (W[ 9] * S), sf[2] * sf[1] / (W[10] * S), sf[3] * sf[1] / (W[11] * S), sf[4] * sf[1] / (W[12] * S), sf[5] * sf[1] / (W[13] * S), sf[6] * sf[1] / (W[14] * S), sf[7] * sf[1] / (W[15] * S), + sf[0] * sf[2] / (W[16] * S), sf[1] * sf[2] / (W[17] * S), sf[2] * sf[2] / (W[18] * S), sf[3] * sf[2] / (W[19] * S), sf[4] * sf[2] / (W[20] * S), sf[5] * sf[2] / (W[21] * S), sf[6] * sf[2] / (W[22] * S), sf[7] * sf[2] / (W[23] * S), + sf[0] * sf[3] / (W[24] * S), sf[1] * sf[3] / (W[25] * S), sf[2] * sf[3] / (W[26] * S), sf[3] * sf[3] / (W[27] * S), sf[4] * sf[3] / (W[28] * S), sf[5] * sf[3] / (W[29] * S), sf[6] * sf[3] / (W[30] * S), sf[7] * sf[3] / (W[31] * S), + sf[0] * sf[4] / (W[32] * S), sf[1] * sf[4] / (W[33] * S), sf[2] * sf[4] / (W[34] * S), sf[3] * sf[4] / (W[35] * S), sf[4] * sf[4] / (W[36] * S), sf[5] * sf[4] / (W[37] * S), sf[6] * sf[4] / (W[38] * S), sf[7] * sf[4] / (W[39] * S), + sf[0] * sf[5] / (W[40] * S), sf[1] * sf[5] / (W[41] * S), sf[2] * sf[5] / (W[42] * S), sf[3] * sf[5] / (W[43] * S), sf[4] * sf[5] / (W[44] * S), sf[5] * sf[5] / (W[45] * S), sf[6] * sf[5] / (W[46] * S), sf[7] * sf[5] / (W[47] * S), + sf[0] * sf[6] / (W[48] * S), sf[1] * sf[6] / (W[49] * S), sf[2] * sf[6] / (W[50] * S), sf[3] * sf[6] / (W[51] * S), sf[4] * sf[6] / (W[52] * S), sf[5] * sf[6] / (W[53] * S), sf[6] * sf[6] / (W[54] * S), sf[7] * sf[6] / (W[55] * S), + sf[0] * sf[7] / (W[56] * S), sf[1] * sf[7] / (W[57] * S), sf[2] * sf[7] / (W[58] * S), sf[3] * sf[7] / (W[59] * S), sf[4] * sf[7] / (W[60] * S), sf[5] * sf[7] / (W[61] * S), sf[6] * sf[7] / (W[62] * S), sf[7] * sf[7] / (W[63] * S) }; // Clamp and pack a 9-bit and a 7-bit signed value into a 16-bit word. @@ -42,7 +48,7 @@ uint pack_9_7(int v9, int v7) return (uint(clamp(v9, -256, 255)) & 0x1ffu) | ((uint(clamp(v7, -64, 63)) & 0x7fu) << 9); } -// Scaled 1D DCT. y0 output is scaled by 8, everything else is scaled by 16. +// Scaled 1D DCT (AA&N). y0 is correctly scaled, all other y_k are scaled by sqrt(2) cos(k * Pi / 16). void dct_1d(inout float y0, inout float y1, inout float y2, inout float y3, inout float y4, inout float y5, inout float y6, inout float y7) { const float a1 = 0.7071067811865474; // sqrt(2)