X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=src%2Fnnue%2Flayers%2Faffine_transform.h;h=fc65c34339ff35bb44f69dee3a85285de69eb51b;hb=8a912951de6d4bff78d3ff5258213a0c7e6f494e;hp=b28712780b2684868bc2c2937628e4112b72b69c;hpb=18dcf1f09754284325157f2d270df10a09297958;p=stockfish diff --git a/src/nnue/layers/affine_transform.h b/src/nnue/layers/affine_transform.h index b2871278..fc65c343 100644 --- a/src/nnue/layers/affine_transform.h +++ b/src/nnue/layers/affine_transform.h @@ -1,6 +1,6 @@ /* Stockfish, a UCI chess playing engine derived from Glaurung 2.1 - Copyright (C) 2004-2021 The Stockfish developers (see AUTHORS file) + Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file) Stockfish is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -21,34 +21,18 @@ #ifndef NNUE_LAYERS_AFFINE_TRANSFORM_H_INCLUDED #define NNUE_LAYERS_AFFINE_TRANSFORM_H_INCLUDED +#include #include -#include -#include + #include "../nnue_common.h" -#include "../../simd.h" +#include "simd.h" /* This file contains the definition for a fully connected layer (aka affine transform). - Two approaches are employed, depending on the sizes of the transform. - - Approach 1: - - used when the PaddedInputDimensions >= 128 - - uses AVX512 if possible - - processes inputs in batches of 2*InputSimdWidth - - so in batches of 128 for AVX512 - - the weight blocks of size InputSimdWidth are transposed such that - access is sequential - - N columns of the weight matrix are processed a time, where N - depends on the architecture (the amount of registers) - - accumulate + hadd is used - - Approach 2: - - used when the PaddedInputDimensions < 128 - - does not use AVX512 + - expected use-case is for when PaddedInputDimensions == 32 and InputDimensions <= 32. - that's why AVX512 is hard to implement - expected use-case is small layers - - not optimized as well as the approach 1 - inputs are processed in chunks of 4, weights are respectively transposed - accumulation happens directly to int32s */ @@ -56,27 +40,24 @@ namespace Stockfish::Eval::NNUE::Layers { // Fallback implementation for older/other architectures. -// Identical for both approaches. Requires the input to be padded to at least 16 values. +// Requires the input to be padded to at least 16 values. #if !defined(USE_SSSE3) template static void affine_transform_non_ssse3(std::int32_t* output, const std::int8_t* weights, const std::int32_t* biases, const std::uint8_t* input) { +# if defined(USE_SSE2) || defined(USE_NEON_DOTPROD) || defined(USE_NEON) # if defined(USE_SSE2) // At least a multiple of 16, with SSE2. - static_assert(PaddedInputDimensions % 16 == 0); - constexpr IndexType NumChunks = PaddedInputDimensions / 16; + constexpr IndexType NumChunks = ceil_to_multiple(InputDimensions, 16) / 16; const __m128i Zeros = _mm_setzero_si128(); const auto inputVector = reinterpret_cast(input); -# elif defined(USE_MMX) - static_assert(InputDimensions % 8 == 0); - constexpr IndexType NumChunks = InputDimensions / 8; - const __m64 Zeros = _mm_setzero_si64(); - const auto inputVector = reinterpret_cast(input); +# elif defined(USE_NEON_DOTPROD) + constexpr IndexType NumChunks = ceil_to_multiple(InputDimensions, 16) / 16; + const auto inputVector = reinterpret_cast(input); # elif defined(USE_NEON) - static_assert(PaddedInputDimensions % 16 == 0); - constexpr IndexType NumChunks = PaddedInputDimensions / 16; + constexpr IndexType NumChunks = ceil_to_multiple(InputDimensions, 16) / 16; const auto inputVector = reinterpret_cast(input); # endif @@ -106,25 +87,13 @@ namespace Stockfish::Eval::NNUE::Layers { sum = _mm_add_epi32(sum, sum_second_32); output[i] = _mm_cvtsi128_si32(sum); -# elif defined(USE_MMX) - __m64 sumLo = _mm_cvtsi32_si64(biases[i]); - __m64 sumHi = Zeros; - const auto row = reinterpret_cast(&weights[offset]); +# elif defined(USE_NEON_DOTPROD) + int32x4_t sum = {biases[i]}; + const auto row = reinterpret_cast(&weights[offset]); for (IndexType j = 0; j < NumChunks; ++j) { - __m64 row_j = row[j]; - __m64 input_j = inputVector[j]; - __m64 extendedRowLo = _mm_srai_pi16(_mm_unpacklo_pi8(row_j, row_j), 8); - __m64 extendedRowHi = _mm_srai_pi16(_mm_unpackhi_pi8(row_j, row_j), 8); - __m64 extendedInputLo = _mm_unpacklo_pi8(input_j, Zeros); - __m64 extendedInputHi = _mm_unpackhi_pi8(input_j, Zeros); - __m64 productLo = _mm_madd_pi16(extendedRowLo, extendedInputLo); - __m64 productHi = _mm_madd_pi16(extendedRowHi, extendedInputHi); - sumLo = _mm_add_pi32(sumLo, productLo); - sumHi = _mm_add_pi32(sumHi, productHi); + sum = vdotq_s32(sum, inputVector[j], row[j]); } - __m64 sum = _mm_add_pi32(sumLo, sumHi); - sum = _mm_add_pi32(sum, _mm_unpackhi_pi32(sum, sum)); - output[i] = _mm_cvtsi64_si32(sum); + output[i] = vaddvq_s32(sum); # elif defined(USE_NEON) int32x4_t sum = {biases[i]}; @@ -136,115 +105,71 @@ namespace Stockfish::Eval::NNUE::Layers { } output[i] = sum[0] + sum[1] + sum[2] + sum[3]; -# else - std::int32_t sum = biases[i]; - for (IndexType j = 0; j < InputDimensions; ++j) { - sum += weights[offset + j] * input[j]; - } - output[i] = sum; # endif } - -# if defined(USE_MMX) - _mm_empty(); +# else + std::memcpy(output, biases, sizeof(std::int32_t) * OutputDimensions); + + // Traverse weights in transpose order to take advantage of input sparsity + for (IndexType i = 0; i < InputDimensions; ++i) + if (input[i]) { + const std::int8_t* w = &weights[i]; + const int in = input[i]; + for (IndexType j = 0; j < OutputDimensions; ++j) + output[j] += w[j * PaddedInputDimensions] * in; + } # endif } #endif - template - class AffineTransform; - - // A specialization for large inputs. - template - class AffineTransform= 2*64-1)>> { + template + class AffineTransform { public: // Input/output type - using InputType = typename PreviousLayer::OutputType; + using InputType = std::uint8_t; using OutputType = std::int32_t; - static_assert(std::is_same::value, ""); // Number of input/output dimensions - static constexpr IndexType InputDimensions = PreviousLayer::OutputDimensions; + static constexpr IndexType InputDimensions = InDims; static constexpr IndexType OutputDimensions = OutDims; static constexpr IndexType PaddedInputDimensions = ceil_to_multiple(InputDimensions, MaxSimdWidth); + static constexpr IndexType PaddedOutputDimensions = + ceil_to_multiple(OutputDimensions, MaxSimdWidth); - static_assert(PaddedInputDimensions >= 128, "Something went wrong. This specialization should not have been chosen."); - -#if defined (USE_AVX512) - static constexpr const IndexType InputSimdWidth = 64; - static constexpr const IndexType MaxNumOutputRegs = 16; -#elif defined (USE_AVX2) - static constexpr const IndexType InputSimdWidth = 32; - static constexpr const IndexType MaxNumOutputRegs = 8; -#elif defined (USE_SSSE3) - static constexpr const IndexType InputSimdWidth = 16; - static constexpr const IndexType MaxNumOutputRegs = 8; -#else - // The fallback implementation will not have permuted weights. - // We define these to avoid a lot of ifdefs later. - static constexpr const IndexType InputSimdWidth = 1; - static constexpr const IndexType MaxNumOutputRegs = 1; -#endif - - // A big block is a region in the weight matrix of the size [PaddedInputDimensions, NumOutputRegs]. - // A small block is a region of size [InputSimdWidth, 1] - - static constexpr const IndexType NumOutputRegs = std::min(MaxNumOutputRegs, OutputDimensions); - static constexpr const IndexType SmallBlockSize = InputSimdWidth; - static constexpr const IndexType BigBlockSize = NumOutputRegs * PaddedInputDimensions; - static constexpr const IndexType NumSmallBlocksInBigBlock = BigBlockSize / SmallBlockSize; - static constexpr const IndexType NumSmallBlocksPerOutput = PaddedInputDimensions / SmallBlockSize; - static constexpr const IndexType NumBigBlocks = OutputDimensions / NumOutputRegs; - - static_assert(OutputDimensions % NumOutputRegs == 0); - - // Size of forward propagation buffer used in this layer - static constexpr std::size_t SelfBufferSize = - ceil_to_multiple(OutputDimensions * sizeof(OutputType), CacheLineSize); - - // Size of the forward propagation buffer used from the input layer to this layer - static constexpr std::size_t BufferSize = - PreviousLayer::BufferSize + SelfBufferSize; + using OutputBuffer = OutputType[PaddedOutputDimensions]; // Hash value embedded in the evaluation file - static constexpr std::uint32_t get_hash_value() { + static constexpr std::uint32_t get_hash_value(std::uint32_t prevHash) { std::uint32_t hashValue = 0xCC03DAE4u; hashValue += OutputDimensions; - hashValue ^= PreviousLayer::get_hash_value() >> 1; - hashValue ^= PreviousLayer::get_hash_value() << 31; + hashValue ^= prevHash >> 1; + hashValue ^= prevHash << 31; return hashValue; } - /* - Transposes the small blocks within a block. - Effectively means that weights can be traversed sequentially during inference. - */ - static IndexType get_weight_index(IndexType i) + static constexpr IndexType get_weight_index_scrambled(IndexType i) + { + return + (i / 4) % (PaddedInputDimensions / 4) * OutputDimensions * 4 + + i / PaddedInputDimensions * 4 + + i % 4; + } + + static constexpr IndexType get_weight_index(IndexType i) { - const IndexType smallBlock = (i / SmallBlockSize) % NumSmallBlocksInBigBlock; - const IndexType smallBlockCol = smallBlock / NumSmallBlocksPerOutput; - const IndexType smallBlockRow = smallBlock % NumSmallBlocksPerOutput; - const IndexType bigBlock = i / BigBlockSize; - const IndexType rest = i % SmallBlockSize; - - const IndexType idx = - bigBlock * BigBlockSize - + smallBlockRow * SmallBlockSize * NumOutputRegs - + smallBlockCol * SmallBlockSize - + rest; - - return idx; +#if defined (USE_SSSE3) + return get_weight_index_scrambled(i); +#else + return i; +#endif } // Read network parameters bool read_parameters(std::istream& stream) { - if (!previousLayer.read_parameters(stream)) return false; - for (std::size_t i = 0; i < OutputDimensions; ++i) - biases[i] = read_little_endian(stream); - - for (std::size_t i = 0; i < OutputDimensions * PaddedInputDimensions; ++i) + read_little_endian(stream, biases, OutputDimensions); + for (IndexType i = 0; i < OutputDimensions * PaddedInputDimensions; ++i) weights[get_weight_index(i)] = read_little_endian(stream); return !stream.fail(); @@ -252,22 +177,21 @@ namespace Stockfish::Eval::NNUE::Layers { // Write network parameters bool write_parameters(std::ostream& stream) const { - if (!previousLayer.write_parameters(stream)) return false; - for (std::size_t i = 0; i < OutputDimensions; ++i) - write_little_endian(stream, biases[i]); + write_little_endian(stream, biases, OutputDimensions); - for (std::size_t i = 0; i < OutputDimensions * PaddedInputDimensions; ++i) + for (IndexType i = 0; i < OutputDimensions * PaddedInputDimensions; ++i) write_little_endian(stream, weights[get_weight_index(i)]); return !stream.fail(); } - // Forward propagation - const OutputType* propagate( - const TransformedFeatureType* transformedFeatures, char* buffer) const { - const auto input = previousLayer.propagate( - transformedFeatures, buffer + SelfBufferSize); - OutputType* output = reinterpret_cast(buffer); + void propagate( + const InputType* input, OutputType* output) const { + +#if defined (USE_SSSE3) + + if constexpr (OutputDimensions > 1) + { #if defined (USE_AVX512) using vec_t = __m512i; @@ -276,7 +200,6 @@ namespace Stockfish::Eval::NNUE::Layers { #define vec_add_dpbusd_32 Simd::m512_add_dpbusd_epi32 #define vec_add_dpbusd_32x2 Simd::m512_add_dpbusd_epi32x2 #define vec_hadd Simd::m512_hadd - #define vec_haddx4 Simd::m512_haddx4 #elif defined (USE_AVX2) using vec_t = __m256i; #define vec_setzero _mm256_setzero_si256 @@ -284,7 +207,6 @@ namespace Stockfish::Eval::NNUE::Layers { #define vec_add_dpbusd_32 Simd::m256_add_dpbusd_epi32 #define vec_add_dpbusd_32x2 Simd::m256_add_dpbusd_epi32x2 #define vec_hadd Simd::m256_hadd - #define vec_haddx4 Simd::m256_haddx4 #elif defined (USE_SSSE3) using vec_t = __m128i; #define vec_setzero _mm_setzero_si128 @@ -292,242 +214,86 @@ namespace Stockfish::Eval::NNUE::Layers { #define vec_add_dpbusd_32 Simd::m128_add_dpbusd_epi32 #define vec_add_dpbusd_32x2 Simd::m128_add_dpbusd_epi32x2 #define vec_hadd Simd::m128_hadd - #define vec_haddx4 Simd::m128_haddx4 #endif -#if defined (USE_SSSE3) - const vec_t* invec = reinterpret_cast(input); + static constexpr IndexType OutputSimdWidth = sizeof(vec_t) / sizeof(OutputType); + static_assert(OutputDimensions % OutputSimdWidth == 0); - // Perform accumulation to registers for each big block - for (IndexType bigBlock = 0; bigBlock < NumBigBlocks; ++bigBlock) - { - vec_t acc[NumOutputRegs] = { vec_setzero() }; - - // Each big block has NumOutputRegs small blocks in each "row", one per register. - // We process two small blocks at a time to save on one addition without VNNI. - for (IndexType smallBlock = 0; smallBlock < NumSmallBlocksPerOutput; smallBlock += 2) - { - const vec_t* weightvec = - reinterpret_cast( - weights - + bigBlock * BigBlockSize - + smallBlock * SmallBlockSize * NumOutputRegs); - - const vec_t in0 = invec[smallBlock + 0]; - const vec_t in1 = invec[smallBlock + 1]; + constexpr IndexType NumChunks = ceil_to_multiple(InputDimensions, 8) / 4; + constexpr IndexType NumRegs = OutputDimensions / OutputSimdWidth; - for (IndexType k = 0; k < NumOutputRegs; ++k) - vec_add_dpbusd_32x2(acc[k], in0, weightvec[k], in1, weightvec[k + NumOutputRegs]); - } + const auto input32 = reinterpret_cast(input); + const vec_t* biasvec = reinterpret_cast(biases); + vec_t acc[NumRegs]; + for (IndexType k = 0; k < NumRegs; ++k) + acc[k] = biasvec[k]; - // Horizontally add all accumulators. - if constexpr (NumOutputRegs % 4 == 0) - { - __m128i* outputvec = reinterpret_cast<__m128i*>(output); - const __m128i* biasvec = reinterpret_cast(biases); - - for (IndexType k = 0; k < NumOutputRegs; k += 4) - { - const IndexType idx = (bigBlock * NumOutputRegs + k) / 4; - outputvec[idx] = vec_haddx4(acc[k+0], acc[k+1], acc[k+2], acc[k+3], biasvec[idx]); - } - } - else + for (IndexType i = 0; i < NumChunks; i += 2) { - for (IndexType k = 0; k < NumOutputRegs; ++k) - { - const IndexType idx = (bigBlock * NumOutputRegs + k); - output[idx] = vec_hadd(acc[k], biases[idx]); - } + const vec_t in0 = vec_set_32(input32[i + 0]); + const vec_t in1 = vec_set_32(input32[i + 1]); + const auto col0 = reinterpret_cast(&weights[(i + 0) * OutputDimensions * 4]); + const auto col1 = reinterpret_cast(&weights[(i + 1) * OutputDimensions * 4]); + for (IndexType k = 0; k < NumRegs; ++k) + vec_add_dpbusd_32x2(acc[k], in0, col0[k], in1, col1[k]); } - } + + vec_t* outptr = reinterpret_cast(output); + for (IndexType k = 0; k < NumRegs; ++k) + outptr[k] = acc[k]; # undef vec_setzero # undef vec_set_32 # undef vec_add_dpbusd_32 # undef vec_add_dpbusd_32x2 # undef vec_hadd -# undef vec_haddx4 -#else - // Use old implementation for the other architectures. - affine_transform_non_ssse3< - InputDimensions, - PaddedInputDimensions, - OutputDimensions>(output, weights, biases, input); - -#endif - - return output; - } - - private: - using BiasType = OutputType; - using WeightType = std::int8_t; - - PreviousLayer previousLayer; - - alignas(CacheLineSize) BiasType biases[OutputDimensions]; - alignas(CacheLineSize) WeightType weights[OutputDimensions * PaddedInputDimensions]; - }; - template - class AffineTransform> { - public: - // Input/output type - using InputType = typename PreviousLayer::OutputType; - using OutputType = std::int32_t; - static_assert(std::is_same::value, ""); - - // Number of input/output dimensions - static constexpr IndexType InputDimensions = - PreviousLayer::OutputDimensions; - static constexpr IndexType OutputDimensions = OutDims; - static constexpr IndexType PaddedInputDimensions = - ceil_to_multiple(InputDimensions, MaxSimdWidth); - - static_assert(PaddedInputDimensions < 128, "Something went wrong. This specialization should not have been chosen."); - -#if defined (USE_SSSE3) - static constexpr const IndexType OutputSimdWidth = SimdWidth / 4; - static constexpr const IndexType InputSimdWidth = SimdWidth; -#endif - - // Size of forward propagation buffer used in this layer - static constexpr std::size_t SelfBufferSize = - ceil_to_multiple(OutputDimensions * sizeof(OutputType), CacheLineSize); - - // Size of the forward propagation buffer used from the input layer to this layer - static constexpr std::size_t BufferSize = - PreviousLayer::BufferSize + SelfBufferSize; - - // Hash value embedded in the evaluation file - static constexpr std::uint32_t get_hash_value() { - std::uint32_t hashValue = 0xCC03DAE4u; - hashValue += OutputDimensions; - hashValue ^= PreviousLayer::get_hash_value() >> 1; - hashValue ^= PreviousLayer::get_hash_value() << 31; - return hashValue; - } - - static IndexType get_weight_index_scrambled(IndexType i) - { - return - (i / 4) % (PaddedInputDimensions / 4) * OutputDimensions * 4 + - i / PaddedInputDimensions * 4 + - i % 4; - } - - static IndexType get_weight_index(IndexType i) - { -#if defined (USE_SSSE3) - return get_weight_index_scrambled(i); -#else - return i; -#endif - } - - // Read network parameters - bool read_parameters(std::istream& stream) { - if (!previousLayer.read_parameters(stream)) return false; - for (std::size_t i = 0; i < OutputDimensions; ++i) - biases[i] = read_little_endian(stream); - for (std::size_t i = 0; i < OutputDimensions * PaddedInputDimensions; ++i) - weights[get_weight_index(i)] = read_little_endian(stream); - - return !stream.fail(); - } - - // Write network parameters - bool write_parameters(std::ostream& stream) const { - if (!previousLayer.write_parameters(stream)) return false; - for (std::size_t i = 0; i < OutputDimensions; ++i) - write_little_endian(stream, biases[i]); - - for (std::size_t i = 0; i < OutputDimensions * PaddedInputDimensions; ++i) - write_little_endian(stream, weights[get_weight_index(i)]); - - return !stream.fail(); - } - // Forward propagation - const OutputType* propagate( - const TransformedFeatureType* transformedFeatures, char* buffer) const { - const auto input = previousLayer.propagate( - transformedFeatures, buffer + SelfBufferSize); - const auto output = reinterpret_cast(buffer); + } + else if constexpr (OutputDimensions == 1) + { +// We cannot use AVX512 for the last layer because there's only 32 inputs and the buffer is not padded to 64 elements. #if defined (USE_AVX2) using vec_t = __m256i; #define vec_setzero _mm256_setzero_si256 #define vec_set_32 _mm256_set1_epi32 #define vec_add_dpbusd_32 Simd::m256_add_dpbusd_epi32 #define vec_add_dpbusd_32x2 Simd::m256_add_dpbusd_epi32x2 - #define vec_add_dpbusd_32x4 Simd::m256_add_dpbusd_epi32x4 #define vec_hadd Simd::m256_hadd - #define vec_haddx4 Simd::m256_haddx4 #elif defined (USE_SSSE3) using vec_t = __m128i; #define vec_setzero _mm_setzero_si128 #define vec_set_32 _mm_set1_epi32 #define vec_add_dpbusd_32 Simd::m128_add_dpbusd_epi32 #define vec_add_dpbusd_32x2 Simd::m128_add_dpbusd_epi32x2 - #define vec_add_dpbusd_32x4 Simd::m128_add_dpbusd_epi32x4 #define vec_hadd Simd::m128_hadd - #define vec_haddx4 Simd::m128_haddx4 #endif -#if defined (USE_SSSE3) - const auto inputVector = reinterpret_cast(input); + const auto inputVector = reinterpret_cast(input); - static_assert(InputDimensions % 8 == 0); - static_assert(OutputDimensions % OutputSimdWidth == 0 || OutputDimensions == 1); + static constexpr IndexType InputSimdWidth = sizeof(vec_t) / sizeof(InputType); - if constexpr (OutputDimensions % OutputSimdWidth == 0) - { - constexpr IndexType NumChunks = InputDimensions / 4; - constexpr IndexType NumRegs = OutputDimensions / OutputSimdWidth; - - const auto input32 = reinterpret_cast(input); - const vec_t* biasvec = reinterpret_cast(biases); - vec_t acc[NumRegs]; - for (IndexType k = 0; k < NumRegs; ++k) - acc[k] = biasvec[k]; + static_assert(PaddedInputDimensions % InputSimdWidth == 0); - for (IndexType i = 0; i < NumChunks; i += 2) - { - const vec_t in0 = vec_set_32(input32[i + 0]); - const vec_t in1 = vec_set_32(input32[i + 1]); - const auto col0 = reinterpret_cast(&weights[(i + 0) * OutputDimensions * 4]); - const auto col1 = reinterpret_cast(&weights[(i + 1) * OutputDimensions * 4]); - for (IndexType k = 0; k < NumRegs; ++k) - vec_add_dpbusd_32x2(acc[k], in0, col0[k], in1, col1[k]); - } - - vec_t* outptr = reinterpret_cast(output); - for (IndexType k = 0; k < NumRegs; ++k) - outptr[k] = acc[k]; - } - else if constexpr (OutputDimensions == 1) - { - constexpr IndexType NumChunks = PaddedInputDimensions / SimdWidth; + constexpr IndexType NumChunks = PaddedInputDimensions / InputSimdWidth; vec_t sum0 = vec_setzero(); const auto row0 = reinterpret_cast(&weights[0]); - for (int j = 0; j < (int)NumChunks; ++j) + for (int j = 0; j < int(NumChunks); ++j) { const vec_t in = inputVector[j]; vec_add_dpbusd_32(sum0, in, row0[j]); } output[0] = vec_hadd(sum0, biases[0]); - } # undef vec_setzero # undef vec_set_32 # undef vec_add_dpbusd_32 # undef vec_add_dpbusd_32x2 -# undef vec_add_dpbusd_32x4 # undef vec_hadd -# undef vec_haddx4 + + } #else // Use old implementation for the other architectures. affine_transform_non_ssse3< @@ -535,16 +301,12 @@ namespace Stockfish::Eval::NNUE::Layers { PaddedInputDimensions, OutputDimensions>(output, weights, biases, input); #endif - - return output; } private: using BiasType = OutputType; using WeightType = std::int8_t; - PreviousLayer previousLayer; - alignas(CacheLineSize) BiasType biases[OutputDimensions]; alignas(CacheLineSize) WeightType weights[OutputDimensions * PaddedInputDimensions]; };