X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=src%2Fnnue%2Flayers%2Faffine_transform.h;h=44fa5d00a434f8285dea2357e8e6889cfdd8aed6;hb=HEAD;hp=d131836865f73800920fcb29ed635bd199c29b4f;hpb=d61d38586ee35fd4d93445eb547e4af27cc86e6b;p=stockfish diff --git a/src/nnue/layers/affine_transform.h b/src/nnue/layers/affine_transform.h index d1318368..44fa5d00 100644 --- a/src/nnue/layers/affine_transform.h +++ b/src/nnue/layers/affine_transform.h @@ -1,6 +1,6 @@ /* Stockfish, a UCI chess playing engine derived from Glaurung 2.1 - Copyright (C) 2004-2021 The Stockfish developers (see AUTHORS file) + Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file) Stockfish is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -21,613 +21,297 @@ #ifndef NNUE_LAYERS_AFFINE_TRANSFORM_H_INCLUDED #define NNUE_LAYERS_AFFINE_TRANSFORM_H_INCLUDED +#include #include + #include "../nnue_common.h" +#include "simd.h" + +/* + This file contains the definition for a fully connected layer (aka affine transform). + + - expected use-case is for when PaddedInputDimensions == 32 and InputDimensions <= 32. + - that's why AVX512 is hard to implement + - expected use-case is small layers + - inputs are processed in chunks of 4, weights are respectively transposed + - accumulation happens directly to int32s +*/ namespace Stockfish::Eval::NNUE::Layers { - // Affine transformation layer - template - class AffineTransform { +// Fallback implementation for older/other architectures. +// Requires the input to be padded to at least 16 values. +#if !defined(USE_SSSE3) +template +static void affine_transform_non_ssse3(std::int32_t* output, + const std::int8_t* weights, + const std::int32_t* biases, + const std::uint8_t* input) { + #if defined(USE_SSE2) || defined(USE_NEON_DOTPROD) || defined(USE_NEON) + #if defined(USE_SSE2) + // At least a multiple of 16, with SSE2. + constexpr IndexType NumChunks = ceil_to_multiple(InputDimensions, 16) / 16; + const __m128i Zeros = _mm_setzero_si128(); + const auto inputVector = reinterpret_cast(input); + + #elif defined(USE_NEON_DOTPROD) + constexpr IndexType NumChunks = ceil_to_multiple(InputDimensions, 16) / 16; + const auto inputVector = reinterpret_cast(input); + + #elif defined(USE_NEON) + constexpr IndexType NumChunks = ceil_to_multiple(InputDimensions, 16) / 16; + const auto inputVector = reinterpret_cast(input); + #endif + + for (IndexType i = 0; i < OutputDimensions; ++i) + { + const IndexType offset = i * PaddedInputDimensions; + + #if defined(USE_SSE2) + __m128i sumLo = _mm_cvtsi32_si128(biases[i]); + __m128i sumHi = Zeros; + const auto row = reinterpret_cast(&weights[offset]); + for (IndexType j = 0; j < NumChunks; ++j) + { + __m128i row_j = _mm_load_si128(&row[j]); + __m128i input_j = _mm_load_si128(&inputVector[j]); + __m128i extendedRowLo = _mm_srai_epi16(_mm_unpacklo_epi8(row_j, row_j), 8); + __m128i extendedRowHi = _mm_srai_epi16(_mm_unpackhi_epi8(row_j, row_j), 8); + __m128i extendedInputLo = _mm_unpacklo_epi8(input_j, Zeros); + __m128i extendedInputHi = _mm_unpackhi_epi8(input_j, Zeros); + __m128i productLo = _mm_madd_epi16(extendedRowLo, extendedInputLo); + __m128i productHi = _mm_madd_epi16(extendedRowHi, extendedInputHi); + sumLo = _mm_add_epi32(sumLo, productLo); + sumHi = _mm_add_epi32(sumHi, productHi); + } + __m128i sum = _mm_add_epi32(sumLo, sumHi); + __m128i sumHigh_64 = _mm_shuffle_epi32(sum, _MM_SHUFFLE(1, 0, 3, 2)); + sum = _mm_add_epi32(sum, sumHigh_64); + __m128i sum_second_32 = _mm_shufflelo_epi16(sum, _MM_SHUFFLE(1, 0, 3, 2)); + sum = _mm_add_epi32(sum, sum_second_32); + output[i] = _mm_cvtsi128_si32(sum); + + #elif defined(USE_NEON_DOTPROD) + int32x4_t sum = {biases[i]}; + const auto row = reinterpret_cast(&weights[offset]); + for (IndexType j = 0; j < NumChunks; ++j) + { + sum = vdotq_s32(sum, inputVector[j], row[j]); + } + output[i] = vaddvq_s32(sum); + + #elif defined(USE_NEON) + int32x4_t sum = {biases[i]}; + const auto row = reinterpret_cast(&weights[offset]); + for (IndexType j = 0; j < NumChunks; ++j) + { + int16x8_t product = vmull_s8(inputVector[j * 2], row[j * 2]); + product = vmlal_s8(product, inputVector[j * 2 + 1], row[j * 2 + 1]); + sum = vpadalq_s16(sum, product); + } + output[i] = sum[0] + sum[1] + sum[2] + sum[3]; + + #endif + } + #else + std::memcpy(output, biases, sizeof(std::int32_t) * OutputDimensions); + + // Traverse weights in transpose order to take advantage of input sparsity + for (IndexType i = 0; i < InputDimensions; ++i) + if (input[i]) + { + const std::int8_t* w = &weights[i]; + const int in = input[i]; + for (IndexType j = 0; j < OutputDimensions; ++j) + output[j] += w[j * PaddedInputDimensions] * in; + } + #endif +} +#endif + +template +class AffineTransform { public: // Input/output type - using InputType = typename PreviousLayer::OutputType; + using InputType = std::uint8_t; using OutputType = std::int32_t; - static_assert(std::is_same::value, ""); // Number of input/output dimensions - static constexpr IndexType InputDimensions = - PreviousLayer::OutputDimensions; + static constexpr IndexType InputDimensions = InDims; static constexpr IndexType OutputDimensions = OutDims; - static constexpr IndexType PaddedInputDimensions = - ceil_to_multiple(InputDimensions, MaxSimdWidth); -#if defined (USE_AVX512) - static constexpr const IndexType OutputSimdWidth = SimdWidth / 2; -#elif defined (USE_SSSE3) - static constexpr const IndexType OutputSimdWidth = SimdWidth / 4; -#endif -#if defined (USE_AVX512) - static constexpr const IndexType InputSimdWidth = SimdWidth * 2; -#elif defined (USE_SSSE3) - static constexpr const IndexType InputSimdWidth = SimdWidth; -#endif - // Size of forward propagation buffer used in this layer - static constexpr std::size_t SelfBufferSize = - ceil_to_multiple(OutputDimensions * sizeof(OutputType), CacheLineSize); + static constexpr IndexType PaddedInputDimensions = + ceil_to_multiple(InputDimensions, MaxSimdWidth); + static constexpr IndexType PaddedOutputDimensions = + ceil_to_multiple(OutputDimensions, MaxSimdWidth); - // Size of the forward propagation buffer used from the input layer to this layer - static constexpr std::size_t BufferSize = - PreviousLayer::BufferSize + SelfBufferSize; + using OutputBuffer = OutputType[PaddedOutputDimensions]; // Hash value embedded in the evaluation file - static constexpr std::uint32_t get_hash_value() { - std::uint32_t hashValue = 0xCC03DAE4u; - hashValue += OutputDimensions; - hashValue ^= PreviousLayer::get_hash_value() >> 1; - hashValue ^= PreviousLayer::get_hash_value() << 31; - return hashValue; + static constexpr std::uint32_t get_hash_value(std::uint32_t prevHash) { + std::uint32_t hashValue = 0xCC03DAE4u; + hashValue += OutputDimensions; + hashValue ^= prevHash >> 1; + hashValue ^= prevHash << 31; + return hashValue; } - // Read network parameters - bool read_parameters(std::istream& stream) { - if (!previousLayer.read_parameters(stream)) return false; - for (std::size_t i = 0; i < OutputDimensions; ++i) - biases[i] = read_little_endian(stream); - for (std::size_t i = 0; i < OutputDimensions * PaddedInputDimensions; ++i) -#if !defined (USE_SSSE3) - weights[i] = read_little_endian(stream); -#elif defined (USE_VNNI) || defined (USE_AVX512) - if constexpr (OutputDimensions <= 8 && OutputDimensions != 1) - weights[i] = read_little_endian(stream); - else - weights[ - (i / 4) % (PaddedInputDimensions / 4) * OutputDimensions * 4 + - i / PaddedInputDimensions * 4 + - i % 4 - ] = read_little_endian(stream); -#else - weights[ - (i / 4) % (PaddedInputDimensions / 4) * OutputDimensions * 4 + - i / PaddedInputDimensions * 4 + - i % 4 - ] = read_little_endian(stream); -#endif - - return !stream.fail(); + static constexpr IndexType get_weight_index_scrambled(IndexType i) { + return (i / 4) % (PaddedInputDimensions / 4) * OutputDimensions * 4 + + i / PaddedInputDimensions * 4 + i % 4; } - // Write network parameters - bool write_parameters(std::ostream& stream) const { - if (!previousLayer.write_parameters(stream)) return false; - for (std::size_t i = 0; i < OutputDimensions; ++i) - write_little_endian(stream, biases[i]); -#if !defined (USE_SSSE3) - for (std::size_t i = 0; i < OutputDimensions * PaddedInputDimensions; ++i) - write_little_endian(stream, weights[i]); + static constexpr IndexType get_weight_index(IndexType i) { +#if defined(USE_SSSE3) + return get_weight_index_scrambled(i); #else - std::unique_ptr unscrambledWeights = std::make_unique(OutputDimensions * PaddedInputDimensions); - for (std::size_t i = 0; i < OutputDimensions * PaddedInputDimensions; ++i) { - unscrambledWeights[i] = - weights[ - (i / 4) % (PaddedInputDimensions / 4) * OutputDimensions * 4 + - i / PaddedInputDimensions * 4 + - i % 4 - ]; - } - - for (std::size_t i = 0; i < OutputDimensions * PaddedInputDimensions; ++i) - write_little_endian(stream, unscrambledWeights[i]); + return i; #endif - - return !stream.fail(); } - // Forward propagation - const OutputType* propagate( - const TransformedFeatureType* transformedFeatures, char* buffer) const { - const auto input = previousLayer.propagate( - transformedFeatures, buffer + SelfBufferSize); - -#if defined (USE_AVX512) - - [[maybe_unused]] const __m512i Ones512 = _mm512_set1_epi16(1); - - [[maybe_unused]] auto m512_hadd = [](__m512i sum, int bias) -> int { - return _mm512_reduce_add_epi32(sum) + bias; - }; - - [[maybe_unused]] auto m512_hadd128x16_interleave = []( - __m512i sum0, __m512i sum1, __m512i sum2, __m512i sum3) -> __m512i { - - __m512i sum01a = _mm512_unpacklo_epi32(sum0, sum1); - __m512i sum01b = _mm512_unpackhi_epi32(sum0, sum1); - - __m512i sum23a = _mm512_unpacklo_epi32(sum2, sum3); - __m512i sum23b = _mm512_unpackhi_epi32(sum2, sum3); - - __m512i sum01 = _mm512_add_epi32(sum01a, sum01b); - __m512i sum23 = _mm512_add_epi32(sum23a, sum23b); - - __m512i sum0123a = _mm512_unpacklo_epi64(sum01, sum23); - __m512i sum0123b = _mm512_unpackhi_epi64(sum01, sum23); - - return _mm512_add_epi32(sum0123a, sum0123b); - }; - - [[maybe_unused]] auto m512_haddx4 = [m512_hadd128x16_interleave]( - __m512i sum0, __m512i sum1, __m512i sum2, __m512i sum3, __m128i bias) -> __m128i { - - __m512i sum = m512_hadd128x16_interleave(sum0, sum1, sum2, sum3); - - __m256i sum256lo = _mm512_castsi512_si256(sum); - __m256i sum256hi = _mm512_extracti64x4_epi64(sum, 1); - - sum256lo = _mm256_add_epi32(sum256lo, sum256hi); - - __m128i sum128lo = _mm256_castsi256_si128(sum256lo); - __m128i sum128hi = _mm256_extracti128_si256(sum256lo, 1); - - return _mm_add_epi32(_mm_add_epi32(sum128lo, sum128hi), bias); - }; - - [[maybe_unused]] auto m512_add_dpbusd_epi32 = [=](__m512i& acc, __m512i a, __m512i b) { -#if defined (USE_VNNI) - acc = _mm512_dpbusd_epi32(acc, a, b); -#else - __m512i product0 = _mm512_maddubs_epi16(a, b); - product0 = _mm512_madd_epi16(product0, Ones512); - acc = _mm512_add_epi32(acc, product0); -#endif - }; - - [[maybe_unused]] auto m512_add_dpbusd_epi32x2 = [=](__m512i& acc, __m512i a0, __m512i b0, __m512i a1, __m512i b1) { -#if defined (USE_VNNI) - acc = _mm512_dpbusd_epi32(acc, a0, b0); - acc = _mm512_dpbusd_epi32(acc, a1, b1); -#else - __m512i product0 = _mm512_maddubs_epi16(a0, b0); - __m512i product1 = _mm512_maddubs_epi16(a1, b1); - product0 = _mm512_adds_epi16(product0, product1); - product0 = _mm512_madd_epi16(product0, Ones512); - acc = _mm512_add_epi32(acc, product0); -#endif - }; - - [[maybe_unused]] auto m512_add_dpbusd_epi32x4 = [=](__m512i& acc, __m512i a0, __m512i b0, __m512i a1, __m512i b1, - __m512i a2, __m512i b2, __m512i a3, __m512i b3) { -#if defined (USE_VNNI) - acc = _mm512_dpbusd_epi32(acc, a0, b0); - acc = _mm512_dpbusd_epi32(acc, a1, b1); - acc = _mm512_dpbusd_epi32(acc, a2, b2); - acc = _mm512_dpbusd_epi32(acc, a3, b3); -#else - __m512i product0 = _mm512_maddubs_epi16(a0, b0); - __m512i product1 = _mm512_maddubs_epi16(a1, b1); - __m512i product2 = _mm512_maddubs_epi16(a2, b2); - __m512i product3 = _mm512_maddubs_epi16(a3, b3); - product0 = _mm512_adds_epi16(product0, product1); - product0 = _mm512_madd_epi16(product0, Ones512); - product2 = _mm512_adds_epi16(product2, product3); - product2 = _mm512_madd_epi16(product2, Ones512); - acc = _mm512_add_epi32(acc, _mm512_add_epi32(product0, product2)); -#endif - }; - -#endif -#if defined (USE_AVX2) - [[maybe_unused]] const __m256i Ones256 = _mm256_set1_epi16(1); - - [[maybe_unused]] auto m256_hadd = [](__m256i sum, int bias) -> int { - __m128i sum128 = _mm_add_epi32(_mm256_castsi256_si128(sum), _mm256_extracti128_si256(sum, 1)); - sum128 = _mm_add_epi32(sum128, _mm_shuffle_epi32(sum128, _MM_PERM_BADC)); - sum128 = _mm_add_epi32(sum128, _mm_shuffle_epi32(sum128, _MM_PERM_CDAB)); - return _mm_cvtsi128_si32(sum128) + bias; - }; - - [[maybe_unused]] auto m256_haddx4 = [](__m256i sum0, __m256i sum1, __m256i sum2, __m256i sum3, __m128i bias) -> __m128i { - sum0 = _mm256_hadd_epi32(sum0, sum1); - sum2 = _mm256_hadd_epi32(sum2, sum3); - - sum0 = _mm256_hadd_epi32(sum0, sum2); - - __m128i sum128lo = _mm256_castsi256_si128(sum0); - __m128i sum128hi = _mm256_extracti128_si256(sum0, 1); - - return _mm_add_epi32(_mm_add_epi32(sum128lo, sum128hi), bias); - }; - - [[maybe_unused]] auto m256_add_dpbusd_epi32 = [=](__m256i& acc, __m256i a, __m256i b) { -#if defined (USE_VNNI) - acc = _mm256_dpbusd_epi32(acc, a, b); -#else - __m256i product0 = _mm256_maddubs_epi16(a, b); - product0 = _mm256_madd_epi16(product0, Ones256); - acc = _mm256_add_epi32(acc, product0); -#endif - }; - - [[maybe_unused]] auto m256_add_dpbusd_epi32x2 = [=](__m256i& acc, __m256i a0, __m256i b0, __m256i a1, __m256i b1) { -#if defined (USE_VNNI) - acc = _mm256_dpbusd_epi32(acc, a0, b0); - acc = _mm256_dpbusd_epi32(acc, a1, b1); -#else - __m256i product0 = _mm256_maddubs_epi16(a0, b0); - __m256i product1 = _mm256_maddubs_epi16(a1, b1); - product0 = _mm256_adds_epi16(product0, product1); - product0 = _mm256_madd_epi16(product0, Ones256); - acc = _mm256_add_epi32(acc, product0); -#endif - }; - - [[maybe_unused]] auto m256_add_dpbusd_epi32x4 = [=](__m256i& acc, __m256i a0, __m256i b0, __m256i a1, __m256i b1, - __m256i a2, __m256i b2, __m256i a3, __m256i b3) { -#if defined (USE_VNNI) - acc = _mm256_dpbusd_epi32(acc, a0, b0); - acc = _mm256_dpbusd_epi32(acc, a1, b1); - acc = _mm256_dpbusd_epi32(acc, a2, b2); - acc = _mm256_dpbusd_epi32(acc, a3, b3); -#else - __m256i product0 = _mm256_maddubs_epi16(a0, b0); - __m256i product1 = _mm256_maddubs_epi16(a1, b1); - __m256i product2 = _mm256_maddubs_epi16(a2, b2); - __m256i product3 = _mm256_maddubs_epi16(a3, b3); - product0 = _mm256_adds_epi16(product0, product1); - product0 = _mm256_madd_epi16(product0, Ones256); - product2 = _mm256_adds_epi16(product2, product3); - product2 = _mm256_madd_epi16(product2, Ones256); - acc = _mm256_add_epi32(acc, _mm256_add_epi32(product0, product2)); -#endif - }; - -#endif -#if defined (USE_SSSE3) - - [[maybe_unused]] const __m128i Ones128 = _mm_set1_epi16(1); - - [[maybe_unused]] auto m128_hadd = [](__m128i sum, int bias) -> int { - sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0x4E)); //_MM_PERM_BADC - sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0xB1)); //_MM_PERM_CDAB - return _mm_cvtsi128_si32(sum) + bias; - }; - - [[maybe_unused]] auto m128_haddx4 = [](__m128i sum0, __m128i sum1, __m128i sum2, __m128i sum3, __m128i bias) -> __m128i { - sum0 = _mm_hadd_epi32(sum0, sum1); - sum2 = _mm_hadd_epi32(sum2, sum3); - sum0 = _mm_hadd_epi32(sum0, sum2); - return _mm_add_epi32(sum0, bias); - }; - - [[maybe_unused]] auto m128_add_dpbusd_epi32 = [=](__m128i& acc, __m128i a, __m128i b) { - __m128i product0 = _mm_maddubs_epi16(a, b); - product0 = _mm_madd_epi16(product0, Ones128); - acc = _mm_add_epi32(acc, product0); - }; - - [[maybe_unused]] auto m128_add_dpbusd_epi32x2 = [=](__m128i& acc, __m128i a0, __m128i b0, __m128i a1, __m128i b1) { - __m128i product0 = _mm_maddubs_epi16(a0, b0); - __m128i product1 = _mm_maddubs_epi16(a1, b1); - product0 = _mm_adds_epi16(product0, product1); - product0 = _mm_madd_epi16(product0, Ones128); - acc = _mm_add_epi32(acc, product0); - }; - - [[maybe_unused]] auto m128_add_dpbusd_epi32x4 = [=](__m128i& acc, __m128i a0, __m128i b0, __m128i a1, __m128i b1, - __m128i a2, __m128i b2, __m128i a3, __m128i b3) { - __m128i product0 = _mm_maddubs_epi16(a0, b0); - __m128i product1 = _mm_maddubs_epi16(a1, b1); - __m128i product2 = _mm_maddubs_epi16(a2, b2); - __m128i product3 = _mm_maddubs_epi16(a3, b3); - product0 = _mm_adds_epi16(product0, product1); - product0 = _mm_madd_epi16(product0, Ones128); - product2 = _mm_adds_epi16(product2, product3); - product2 = _mm_madd_epi16(product2, Ones128); - acc = _mm_add_epi32(acc, _mm_add_epi32(product0, product2)); - }; - -#endif - -#if defined (USE_AVX512) - using vec_t = __m512i; - #define vec_setzero _mm512_setzero_si512 - #define vec_set_32 _mm512_set1_epi32 - [[maybe_unused]] auto& vec_add_dpbusd_32 = m512_add_dpbusd_epi32; - [[maybe_unused]] auto& vec_add_dpbusd_32x2 = m512_add_dpbusd_epi32x2; - [[maybe_unused]] auto& vec_add_dpbusd_32x4 = m512_add_dpbusd_epi32x4; - [[maybe_unused]] auto& vec_hadd = m512_hadd; - [[maybe_unused]] auto& vec_haddx4 = m512_haddx4; -#elif defined (USE_AVX2) - using vec_t = __m256i; - #define vec_setzero _mm256_setzero_si256 - #define vec_set_32 _mm256_set1_epi32 - [[maybe_unused]] auto& vec_add_dpbusd_32 = m256_add_dpbusd_epi32; - [[maybe_unused]] auto& vec_add_dpbusd_32x2 = m256_add_dpbusd_epi32x2; - [[maybe_unused]] auto& vec_add_dpbusd_32x4 = m256_add_dpbusd_epi32x4; - [[maybe_unused]] auto& vec_hadd = m256_hadd; - [[maybe_unused]] auto& vec_haddx4 = m256_haddx4; -#elif defined (USE_SSSE3) - using vec_t = __m128i; - #define vec_setzero _mm_setzero_si128 - #define vec_set_32 _mm_set1_epi32 - [[maybe_unused]] auto& vec_add_dpbusd_32 = m128_add_dpbusd_epi32; - [[maybe_unused]] auto& vec_add_dpbusd_32x2 = m128_add_dpbusd_epi32x2; - [[maybe_unused]] auto& vec_add_dpbusd_32x4 = m128_add_dpbusd_epi32x4; - [[maybe_unused]] auto& vec_hadd = m128_hadd; - [[maybe_unused]] auto& vec_haddx4 = m128_haddx4; -#endif - -#if defined (USE_SSSE3) - const auto output = reinterpret_cast(buffer); - const auto inputVector = reinterpret_cast(input); -#endif - -#if defined (USE_VNNI) || defined (USE_AVX512) - - static_assert(OutputDimensions == 1 || OutputDimensions % 4 == 0); - - // OutputDimensions is either 1 or a multiple of SimdWidth - // because then it is also an input dimension. - if constexpr (OutputDimensions <= 8 && OutputDimensions != 1) - { - constexpr IndexType NumChunks = PaddedInputDimensions / InputSimdWidth; - - static_assert(NumChunks % 2 == 0); - - const auto input_vec = reinterpret_cast(input); - const auto bias_vec = reinterpret_cast(biases); - auto out_vec = reinterpret_cast<__m128i*>(output); - - vec_t regs[OutputDimensions]; - for (IndexType k = 0; k < OutputDimensions; ++k) - regs[k] = vec_setzero(); - - for (IndexType i = 0; i < NumChunks / 2; ++i) - { - const vec_t in0 = input_vec[i * 2 + 0]; - const vec_t in1 = input_vec[i * 2 + 1]; - for (IndexType k = 0; k < OutputDimensions; ++k) - { - const vec_t w0 = reinterpret_cast(&weights[k * PaddedInputDimensions])[i * 2 + 0]; - const vec_t w1 = reinterpret_cast(&weights[k * PaddedInputDimensions])[i * 2 + 1]; - vec_add_dpbusd_32(regs[k], in0, w0); - vec_add_dpbusd_32(regs[k], in1, w1); - } - } - - for (IndexType k = 0; k < OutputDimensions / 4; ++k) - { - out_vec[k] = vec_haddx4( - regs[k * 4 + 0], - regs[k * 4 + 1], - regs[k * 4 + 2], - regs[k * 4 + 3], - bias_vec[k] - ); - } - } - else if constexpr (InputDimensions == 8) - { - const auto input32 = reinterpret_cast(input); - __m256i* outptr = reinterpret_cast<__m256i*>(output); - std::memcpy(output, biases, OutputDimensions * sizeof(OutputType)); - - const __m256i in0 = _mm256_set1_epi32(input32[0]); - const __m256i in1 = _mm256_set1_epi32(input32[1]); - const auto col0 = reinterpret_cast(&weights[0]); - const auto col1 = reinterpret_cast(&weights[OutputDimensions * 4]); - for (IndexType j = 0; j * 8 < OutputDimensions; ++j) - m256_add_dpbusd_epi32x2(outptr[j], in0, col0[j], in1, col1[j]); - } - else - -#elif defined (USE_SSSE3) - - if constexpr (OutputDimensions % OutputSimdWidth == 0 && InputDimensions == 8) - { - const auto input32 = reinterpret_cast(input); - vec_t* outptr = reinterpret_cast(output); - std::memcpy(output, biases, OutputDimensions * sizeof(OutputType)); - - const vec_t in0 = vec_set_32(input32[0]); - const vec_t in1 = vec_set_32(input32[1]); - const auto col0 = reinterpret_cast(&weights[0]); - const auto col1 = reinterpret_cast(&weights[OutputDimensions * 4]); - for (IndexType j = 0; j * OutputSimdWidth < OutputDimensions; ++j) - vec_add_dpbusd_32x2(outptr[j], in0, col0[j], in1, col1[j]); - } - else - -#endif - -#if defined (USE_SSSE3) - - if constexpr (OutputDimensions % OutputSimdWidth == 0) - { - static_assert(InputDimensions % 16 == 0); - - constexpr IndexType NumChunks = InputDimensions / 4; - constexpr IndexType NumRegs = OutputDimensions / OutputSimdWidth; - - const auto input32 = reinterpret_cast(input); - const vec_t* biasvec = reinterpret_cast(biases); - vec_t outs[NumRegs]; - for (IndexType k = 0; k < NumRegs; ++k) - outs[k] = biasvec[k]; - - for (IndexType i = 0; i < NumChunks; i += 4) - { - const vec_t in0 = vec_set_32(input32[i + 0]); - const vec_t in1 = vec_set_32(input32[i + 1]); - const vec_t in2 = vec_set_32(input32[i + 2]); - const vec_t in3 = vec_set_32(input32[i + 3]); - const auto col0 = reinterpret_cast(&weights[(i + 0) * OutputDimensions * 4]); - const auto col1 = reinterpret_cast(&weights[(i + 1) * OutputDimensions * 4]); - const auto col2 = reinterpret_cast(&weights[(i + 2) * OutputDimensions * 4]); - const auto col3 = reinterpret_cast(&weights[(i + 3) * OutputDimensions * 4]); - for (IndexType k = 0; k < NumRegs; ++k) - vec_add_dpbusd_32x4(outs[k], in0, col0[k], in1, col1[k], in2, col2[k], in3, col3[k]); - } - - vec_t* outptr = reinterpret_cast(output); - for (IndexType k = 0; k < NumRegs; ++k) - outptr[k] = outs[k]; - } - else if constexpr (OutputDimensions == 1) - { - static_assert(InputDimensions % 4 == 0); - -#if defined (USE_AVX512) - if constexpr (PaddedInputDimensions % (SimdWidth * 2) != 0) - { - constexpr IndexType NumChunks = PaddedInputDimensions / SimdWidth; - const auto inputVector256 = reinterpret_cast(input); - - __m256i sum0 = _mm256_setzero_si256(); - const auto row0 = reinterpret_cast(&weights[0]); - - for (int j = 0; j < (int)NumChunks; ++j) - { - const __m256i in = inputVector256[j]; - m256_add_dpbusd_epi32(sum0, in, row0[j]); - } - output[0] = m256_hadd(sum0, biases[0]); - } - else -#endif - { -#if defined (USE_AVX512) - constexpr IndexType NumChunks = PaddedInputDimensions / (SimdWidth * 2); -#else - constexpr IndexType NumChunks = PaddedInputDimensions / SimdWidth; -#endif - vec_t sum0 = vec_setzero(); - const auto row0 = reinterpret_cast(&weights[0]); - - for (int j = 0; j < (int)NumChunks; ++j) - { - const vec_t in = inputVector[j]; - vec_add_dpbusd_32(sum0, in, row0[j]); - } - output[0] = vec_hadd(sum0, biases[0]); - } - } - -#else - -// Use old implementation for the other architectures. - - auto output = reinterpret_cast(buffer); - -#if defined(USE_SSE2) - // At least a multiple of 16, with SSE2. - static_assert(PaddedInputDimensions % SimdWidth == 0); - constexpr IndexType NumChunks = PaddedInputDimensions / SimdWidth; - const __m128i Zeros = _mm_setzero_si128(); - const auto inputVector = reinterpret_cast(input); + // Read network parameters + bool read_parameters(std::istream& stream) { + read_little_endian(stream, biases, OutputDimensions); + for (IndexType i = 0; i < OutputDimensions * PaddedInputDimensions; ++i) + weights[get_weight_index(i)] = read_little_endian(stream); -#elif defined(USE_MMX) - static_assert(InputDimensions % SimdWidth == 0); - constexpr IndexType NumChunks = InputDimensions / SimdWidth; - const __m64 Zeros = _mm_setzero_si64(); - const auto inputVector = reinterpret_cast(input); + return !stream.fail(); + } -#elif defined(USE_NEON) - static_assert(PaddedInputDimensions % SimdWidth == 0); - constexpr IndexType NumChunks = PaddedInputDimensions / SimdWidth; - const auto inputVector = reinterpret_cast(input); -#endif + // Write network parameters + bool write_parameters(std::ostream& stream) const { + write_little_endian(stream, biases, OutputDimensions); - for (IndexType i = 0; i < OutputDimensions; ++i) { - const IndexType offset = i * PaddedInputDimensions; + for (IndexType i = 0; i < OutputDimensions * PaddedInputDimensions; ++i) + write_little_endian(stream, weights[get_weight_index(i)]); -#if defined(USE_SSE2) - __m128i sumLo = _mm_cvtsi32_si128(biases[i]); - __m128i sumHi = Zeros; - const auto row = reinterpret_cast(&weights[offset]); - for (IndexType j = 0; j < NumChunks; ++j) { - __m128i row_j = _mm_load_si128(&row[j]); - __m128i input_j = _mm_load_si128(&inputVector[j]); - __m128i extendedRowLo = _mm_srai_epi16(_mm_unpacklo_epi8(row_j, row_j), 8); - __m128i extendedRowHi = _mm_srai_epi16(_mm_unpackhi_epi8(row_j, row_j), 8); - __m128i extendedInputLo = _mm_unpacklo_epi8(input_j, Zeros); - __m128i extendedInputHi = _mm_unpackhi_epi8(input_j, Zeros); - __m128i productLo = _mm_madd_epi16(extendedRowLo, extendedInputLo); - __m128i productHi = _mm_madd_epi16(extendedRowHi, extendedInputHi); - sumLo = _mm_add_epi32(sumLo, productLo); - sumHi = _mm_add_epi32(sumHi, productHi); - } - __m128i sum = _mm_add_epi32(sumLo, sumHi); - __m128i sumHigh_64 = _mm_shuffle_epi32(sum, _MM_SHUFFLE(1, 0, 3, 2)); - sum = _mm_add_epi32(sum, sumHigh_64); - __m128i sum_second_32 = _mm_shufflelo_epi16(sum, _MM_SHUFFLE(1, 0, 3, 2)); - sum = _mm_add_epi32(sum, sum_second_32); - output[i] = _mm_cvtsi128_si32(sum); - -#elif defined(USE_MMX) - __m64 sumLo = _mm_cvtsi32_si64(biases[i]); - __m64 sumHi = Zeros; - const auto row = reinterpret_cast(&weights[offset]); - for (IndexType j = 0; j < NumChunks; ++j) { - __m64 row_j = row[j]; - __m64 input_j = inputVector[j]; - __m64 extendedRowLo = _mm_srai_pi16(_mm_unpacklo_pi8(row_j, row_j), 8); - __m64 extendedRowHi = _mm_srai_pi16(_mm_unpackhi_pi8(row_j, row_j), 8); - __m64 extendedInputLo = _mm_unpacklo_pi8(input_j, Zeros); - __m64 extendedInputHi = _mm_unpackhi_pi8(input_j, Zeros); - __m64 productLo = _mm_madd_pi16(extendedRowLo, extendedInputLo); - __m64 productHi = _mm_madd_pi16(extendedRowHi, extendedInputHi); - sumLo = _mm_add_pi32(sumLo, productLo); - sumHi = _mm_add_pi32(sumHi, productHi); + return !stream.fail(); + } + // Forward propagation + void propagate(const InputType* input, OutputType* output) const { + +#if defined(USE_SSSE3) + + if constexpr (OutputDimensions > 1) + { + + #if defined(USE_AVX512) + using vec_t = __m512i; + #define vec_setzero _mm512_setzero_si512 + #define vec_set_32 _mm512_set1_epi32 + #define vec_add_dpbusd_32 Simd::m512_add_dpbusd_epi32 + #define vec_add_dpbusd_32x2 Simd::m512_add_dpbusd_epi32x2 + #define vec_hadd Simd::m512_hadd + #elif defined(USE_AVX2) + using vec_t = __m256i; + #define vec_setzero _mm256_setzero_si256 + #define vec_set_32 _mm256_set1_epi32 + #define vec_add_dpbusd_32 Simd::m256_add_dpbusd_epi32 + #define vec_add_dpbusd_32x2 Simd::m256_add_dpbusd_epi32x2 + #define vec_hadd Simd::m256_hadd + #elif defined(USE_SSSE3) + using vec_t = __m128i; + #define vec_setzero _mm_setzero_si128 + #define vec_set_32 _mm_set1_epi32 + #define vec_add_dpbusd_32 Simd::m128_add_dpbusd_epi32 + #define vec_add_dpbusd_32x2 Simd::m128_add_dpbusd_epi32x2 + #define vec_hadd Simd::m128_hadd + #endif + + static constexpr IndexType OutputSimdWidth = sizeof(vec_t) / sizeof(OutputType); + + static_assert(OutputDimensions % OutputSimdWidth == 0); + + constexpr IndexType NumChunks = ceil_to_multiple(InputDimensions, 8) / 4; + constexpr IndexType NumRegs = OutputDimensions / OutputSimdWidth; + + const auto input32 = reinterpret_cast(input); + const vec_t* biasvec = reinterpret_cast(biases); + vec_t acc[NumRegs]; + for (IndexType k = 0; k < NumRegs; ++k) + acc[k] = biasvec[k]; + + for (IndexType i = 0; i < NumChunks; i += 2) + { + const vec_t in0 = vec_set_32(input32[i + 0]); + const vec_t in1 = vec_set_32(input32[i + 1]); + const auto col0 = + reinterpret_cast(&weights[(i + 0) * OutputDimensions * 4]); + const auto col1 = + reinterpret_cast(&weights[(i + 1) * OutputDimensions * 4]); + for (IndexType k = 0; k < NumRegs; ++k) + vec_add_dpbusd_32x2(acc[k], in0, col0[k], in1, col1[k]); + } + + vec_t* outptr = reinterpret_cast(output); + for (IndexType k = 0; k < NumRegs; ++k) + outptr[k] = acc[k]; + + #undef vec_setzero + #undef vec_set_32 + #undef vec_add_dpbusd_32 + #undef vec_add_dpbusd_32x2 + #undef vec_hadd } - __m64 sum = _mm_add_pi32(sumLo, sumHi); - sum = _mm_add_pi32(sum, _mm_unpackhi_pi32(sum, sum)); - output[i] = _mm_cvtsi64_si32(sum); - -#elif defined(USE_NEON) - int32x4_t sum = {biases[i]}; - const auto row = reinterpret_cast(&weights[offset]); - for (IndexType j = 0; j < NumChunks; ++j) { - int16x8_t product = vmull_s8(inputVector[j * 2], row[j * 2]); - product = vmlal_s8(product, inputVector[j * 2 + 1], row[j * 2 + 1]); - sum = vpadalq_s16(sum, product); + else if constexpr (OutputDimensions == 1) + { + + // We cannot use AVX512 for the last layer because there are only 32 inputs + // and the buffer is not padded to 64 elements. + #if defined(USE_AVX2) + using vec_t = __m256i; + #define vec_setzero _mm256_setzero_si256 + #define vec_set_32 _mm256_set1_epi32 + #define vec_add_dpbusd_32 Simd::m256_add_dpbusd_epi32 + #define vec_add_dpbusd_32x2 Simd::m256_add_dpbusd_epi32x2 + #define vec_hadd Simd::m256_hadd + #elif defined(USE_SSSE3) + using vec_t = __m128i; + #define vec_setzero _mm_setzero_si128 + #define vec_set_32 _mm_set1_epi32 + #define vec_add_dpbusd_32 Simd::m128_add_dpbusd_epi32 + #define vec_add_dpbusd_32x2 Simd::m128_add_dpbusd_epi32x2 + #define vec_hadd Simd::m128_hadd + #endif + + const auto inputVector = reinterpret_cast(input); + + static constexpr IndexType InputSimdWidth = sizeof(vec_t) / sizeof(InputType); + + static_assert(PaddedInputDimensions % InputSimdWidth == 0); + + constexpr IndexType NumChunks = PaddedInputDimensions / InputSimdWidth; + vec_t sum0 = vec_setzero(); + const auto row0 = reinterpret_cast(&weights[0]); + + for (int j = 0; j < int(NumChunks); ++j) + { + const vec_t in = inputVector[j]; + vec_add_dpbusd_32(sum0, in, row0[j]); + } + output[0] = vec_hadd(sum0, biases[0]); + + #undef vec_setzero + #undef vec_set_32 + #undef vec_add_dpbusd_32 + #undef vec_add_dpbusd_32x2 + #undef vec_hadd } - output[i] = sum[0] + sum[1] + sum[2] + sum[3]; - #else - OutputType sum = biases[i]; - for (IndexType j = 0; j < InputDimensions; ++j) { - sum += weights[offset + j] * input[j]; - } - output[i] = sum; -#endif - - } -#if defined(USE_MMX) - _mm_empty(); -#endif - + // Use old implementation for the other architectures. + affine_transform_non_ssse3( + output, weights, biases, input); #endif - -#if (!defined (USE_SSSE3) && defined (USE_SSE2)) || defined (USE_NEON) - static_assert(SimdWidth <= 16, "Otherwise we run outside of the padding for the output."); - if constexpr (SimdWidth > OutputDimensions && OutputDimensions != 1) - for (IndexType i = OutputDimensions; i < SimdWidth; ++i) - output[i] = 0; -#endif - - return output; } private: - using BiasType = OutputType; + using BiasType = OutputType; using WeightType = std::int8_t; - PreviousLayer previousLayer; - alignas(CacheLineSize) BiasType biases[OutputDimensions]; alignas(CacheLineSize) WeightType weights[OutputDimensions * PaddedInputDimensions]; - }; +}; } // namespace Stockfish::Eval::NNUE::Layers -#endif // #ifndef NNUE_LAYERS_AFFINE_TRANSFORM_H_INCLUDED +#endif // #ifndef NNUE_LAYERS_AFFINE_TRANSFORM_H_INCLUDED