X-Git-Url: https://git.sesse.net/?p=stockfish;a=blobdiff_plain;f=src%2Fnnue%2Flayers%2Faffine_transform.h;h=adf152eea5b8894fcdf26cdfebffcdbd602b5d7f;hp=0e0515f932a0773cc82f72c1620bc0a1afe5e5eb;hb=303713b560e356a902c1830bce205716cef54a44;hpb=c7f0a768cb9d5972861baae0f215d69f9e86a626 diff --git a/src/nnue/layers/affine_transform.h b/src/nnue/layers/affine_transform.h index 0e0515f9..adf152ee 100644 --- a/src/nnue/layers/affine_transform.h +++ b/src/nnue/layers/affine_transform.h @@ -1,6 +1,6 @@ /* Stockfish, a UCI chess playing engine derived from Glaurung 2.1 - Copyright (C) 2004-2020 The Stockfish developers (see AUTHORS file) + Copyright (C) 2004-2021 The Stockfish developers (see AUTHORS file) Stockfish is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -41,6 +41,11 @@ namespace Eval::NNUE::Layers { static constexpr IndexType kOutputDimensions = OutputDimensions; static constexpr IndexType kPaddedInputDimensions = CeilToMultiple(kInputDimensions, kMaxSimdWidth); +#if defined (USE_AVX512) + static constexpr const IndexType kOutputSimdWidth = kSimdWidth / 2; +#elif defined (USE_SSSE3) + static constexpr const IndexType kOutputSimdWidth = kSimdWidth / 4; +#endif // Size of forward propagation buffer used in this layer static constexpr std::size_t kSelfBufferSize = @@ -65,7 +70,58 @@ namespace Eval::NNUE::Layers { for (std::size_t i = 0; i < kOutputDimensions; ++i) biases_[i] = read_little_endian(stream); for (std::size_t i = 0; i < kOutputDimensions * kPaddedInputDimensions; ++i) +#if !defined (USE_SSSE3) weights_[i] = read_little_endian(stream); +#else + weights_[ + (i / 4) % (kPaddedInputDimensions / 4) * kOutputDimensions * 4 + + i / kPaddedInputDimensions * 4 + + i % 4 + ] = read_little_endian(stream); + + // Determine if eights of weight and input products can be summed using 16bits + // without saturation. We assume worst case combinations of 0 and 127 for all inputs. + if (kOutputDimensions > 1 && !stream.fail()) + { + canSaturate16.count = 0; +#if !defined(USE_VNNI) + for (IndexType i = 0; i < kPaddedInputDimensions; i += 16) + for (IndexType j = 0; j < kOutputDimensions; ++j) + for (int x = 0; x < 2; ++x) + { + WeightType* w = &weights_[i * kOutputDimensions + j * 4 + x * 2]; + int sum[2] = {0, 0}; + for (int k = 0; k < 8; ++k) + { + IndexType idx = k / 2 * kOutputDimensions * 4 + k % 2; + sum[w[idx] < 0] += w[idx]; + } + for (int sign : {-1, 1}) + while (sign * sum[sign == -1] > 258) + { + int maxK = 0, maxW = 0; + for (int k = 0; k < 8; ++k) + { + IndexType idx = k / 2 * kOutputDimensions * 4 + k % 2; + if (maxW < sign * w[idx]) + maxK = k, maxW = sign * w[idx]; + } + + IndexType idx = maxK / 2 * kOutputDimensions * 4 + maxK % 2; + sum[sign == -1] -= w[idx]; + canSaturate16.add(j, i + maxK / 2 * 4 + maxK % 2 + x * 2, w[idx]); + w[idx] = 0; + } + } + + // Non functional optimization for faster more linear access + std::sort(canSaturate16.ids, canSaturate16.ids + canSaturate16.count, + [](const typename CanSaturate::Entry& e1, const typename CanSaturate::Entry& e2) + { return e1.in == e2.in ? e1.out < e2.out : e1.in < e2.in; }); +#endif + } +#endif + return !stream.fail(); } @@ -83,111 +139,33 @@ namespace Eval::NNUE::Layers { return _mm512_reduce_add_epi32(sum) + bias; }; - // This function takes - // sum0 = [xmm0a, xmm0b, xmm0c, xmm0d] - // sum1 = [xmm1a, xmm1b, xmm1c, xmm1d] - // sum2 = [xmm2a, xmm2b, xmm2c, xmm2d] - // sum3 = [xmm3a, xmm3b, xmm3c, xmm3d] - // and returns - // ret = [ - // reduce_add_epi32(xmm0a), reduce_add_epi32(xmm1a), reduce_add_epi32(xmm2a), reduce_add_epi32(xmm3a), - // reduce_add_epi32(xmm0b), reduce_add_epi32(xmm1b), reduce_add_epi32(xmm2b), reduce_add_epi32(xmm3b), - // reduce_add_epi32(xmm0c), reduce_add_epi32(xmm1c), reduce_add_epi32(xmm2c), reduce_add_epi32(xmm3c), - // reduce_add_epi32(xmm0d), reduce_add_epi32(xmm1d), reduce_add_epi32(xmm2d), reduce_add_epi32(xmm3d) - // ] - [[maybe_unused]] auto m512_hadd128x16_interleave = []( - __m512i sum0, __m512i sum1, __m512i sum2, __m512i sum3) -> __m512i { - - __m512i sum01a = _mm512_unpacklo_epi32(sum0, sum1); - __m512i sum01b = _mm512_unpackhi_epi32(sum0, sum1); - - __m512i sum23a = _mm512_unpacklo_epi32(sum2, sum3); - __m512i sum23b = _mm512_unpackhi_epi32(sum2, sum3); - - __m512i sum01 = _mm512_add_epi32(sum01a, sum01b); - __m512i sum23 = _mm512_add_epi32(sum23a, sum23b); - - __m512i sum0123a = _mm512_unpacklo_epi64(sum01, sum23); - __m512i sum0123b = _mm512_unpackhi_epi64(sum01, sum23); - - return _mm512_add_epi32(sum0123a, sum0123b); - }; - - [[maybe_unused]] auto m512_haddx4 = [m512_hadd128x16_interleave]( - __m512i sum0, __m512i sum1, __m512i sum2, __m512i sum3, __m128i bias) -> __m128i { - - __m512i sum = m512_hadd128x16_interleave(sum0, sum1, sum2, sum3); - - __m256i sum256lo = _mm512_castsi512_si256(sum); - __m256i sum256hi = _mm512_extracti64x4_epi64(sum, 1); - - sum256lo = _mm256_add_epi32(sum256lo, sum256hi); - - __m128i sum128lo = _mm256_castsi256_si128(sum256lo); - __m128i sum128hi = _mm256_extracti128_si256(sum256lo, 1); - - return _mm_add_epi32(_mm_add_epi32(sum128lo, sum128hi), bias); - }; - - [[maybe_unused]] auto m512_haddx8 = [m512_hadd128x16_interleave]( - __m512i sum0, __m512i sum1, __m512i sum2, __m512i sum3, - __m512i sum4, __m512i sum5, __m512i sum6, __m512i sum7, __m256i bias) -> __m256i { - - __m512i suma = m512_hadd128x16_interleave(sum0, sum1, sum2, sum3); - __m512i sumb = m512_hadd128x16_interleave(sum4, sum5, sum6, sum7); - - __m512i indices0 = _mm512_setr_epi64(0, 1, 8, 9, 4, 5, 12, 13); - __m512i indices1 = _mm512_setr_epi64(2, 3, 10, 11, 6, 7, 14, 15); - __m512i x = _mm512_add_epi32( - _mm512_permutex2var_epi64(suma, indices0, sumb), - _mm512_permutex2var_epi64(suma, indices1, sumb)); - - __m256i sum256lo = _mm512_castsi512_si256(x); - __m256i sum256hi = _mm512_extracti64x4_epi64(x, 1); - - return _mm256_add_epi32(_mm256_add_epi32(sum256lo, sum256hi), bias); - }; - - [[maybe_unused]] auto m512_hadd256x8 =[m512_hadd128x16_interleave]( - __m512i sum0, __m512i sum1, __m512i sum2, __m512i sum3, __m256i bias) -> __m256i { - - __m512i sum = m512_hadd128x16_interleave(sum0, sum1, sum2, sum3); - - __m512i indices = _mm512_setr_epi32( - 0, 4, 8, 12, 2, 6, 10, 14, - 1, 5, 9, 13, 3, 7, 11, 15); - sum = _mm512_permutexvar_epi32(indices, sum); - - __m256i sum256lo = _mm512_castsi512_si256(sum); - __m256i sum256hi = _mm512_extracti64x4_epi64(sum, 1); - - return _mm256_add_epi32(_mm256_hadd_epi32(sum256lo, sum256hi), bias); - }; - - [[maybe_unused]] auto m512_hadd256x16 = [m512_hadd128x16_interleave]( - __m512i sum0, __m512i sum1, __m512i sum2, __m512i sum3, - __m512i sum4, __m512i sum5, __m512i sum6, __m512i sum7, __m512i bias) -> __m512i { - - __m512i suma = m512_hadd128x16_interleave(sum0, sum1, sum2, sum3); - __m512i sumb = m512_hadd128x16_interleave(sum4, sum5, sum6, sum7); - - __m512i indices0 = _mm512_setr_epi64(0, 1, 8, 9, 4, 5, 12, 13); - __m512i indices1 = _mm512_setr_epi64(2, 3, 10, 11, 6, 7, 14, 15); - __m512i x = _mm512_add_epi32( - _mm512_permutex2var_epi64(suma, indices0, sumb), - _mm512_permutex2var_epi64(suma, indices1, sumb)); - - __m512i indices = _mm512_setr_epi32(0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15); - return _mm512_add_epi32(_mm512_permutexvar_epi32(indices, x), bias); - }; - -#if defined (USE_VNNI) [[maybe_unused]] auto m512_add_dpbusd_epi32 = [=](__m512i& acc, __m512i a, __m512i b) { +#if defined (USE_VNNI) acc = _mm512_dpbusd_epi32(acc, a, b); #else - [[maybe_unused]] auto m512_dpbusd_epi32 = [=](__m512i a, __m512i b) -> __m512i { __m512i product0 = _mm512_maddubs_epi16(a, b); - return _mm512_madd_epi16(product0, kOnes512); + product0 = _mm512_madd_epi16(product0, kOnes512); + acc = _mm512_add_epi32(acc, product0); +#endif + }; + + [[maybe_unused]] auto m512_add_dpbusd_epi32x4 = [=](__m512i& acc, __m512i a0, __m512i b0, __m512i a1, __m512i b1, + __m512i a2, __m512i b2, __m512i a3, __m512i b3) { +#if defined (USE_VNNI) + acc = _mm512_dpbusd_epi32(acc, a0, b0); + acc = _mm512_dpbusd_epi32(acc, a1, b1); + acc = _mm512_dpbusd_epi32(acc, a2, b2); + acc = _mm512_dpbusd_epi32(acc, a3, b3); +#else + __m512i product0 = _mm512_maddubs_epi16(a0, b0); + __m512i product1 = _mm512_maddubs_epi16(a1, b1); + __m512i product2 = _mm512_maddubs_epi16(a2, b2); + __m512i product3 = _mm512_maddubs_epi16(a3, b3); + product0 = _mm512_add_epi16(product0, product1); + product2 = _mm512_add_epi16(product2, product3); + product0 = _mm512_add_epi16(product0, product2); + product0 = _mm512_madd_epi16(product0, kOnes512); + acc = _mm512_add_epi32(acc, product0); #endif }; @@ -203,29 +181,37 @@ namespace Eval::NNUE::Layers { return _mm_cvtsi128_si32(sum128) + bias; }; - [[maybe_unused]] auto m256_haddx4 = [](__m256i sum0, __m256i sum1, __m256i sum2, __m256i sum3, __m128i bias) -> __m128i { - sum0 = _mm256_hadd_epi32(sum0, sum1); - sum2 = _mm256_hadd_epi32(sum2, sum3); - - sum0 = _mm256_hadd_epi32(sum0, sum2); - - __m128i sum128lo = _mm256_castsi256_si128(sum0); - __m128i sum128hi = _mm256_extracti128_si256(sum0, 1); - - return _mm_add_epi32(_mm_add_epi32(sum128lo, sum128hi), bias); - }; -#if defined (USE_VNNI) [[maybe_unused]] auto m256_add_dpbusd_epi32 = [=](__m256i& acc, __m256i a, __m256i b) { +#if defined (USE_VNNI) acc = _mm256_dpbusd_epi32(acc, a, b); #else - [[maybe_unused]] auto m256_dpbusd_epi32 = [=](__m256i a, __m256i b) -> __m256i { __m256i product0 = _mm256_maddubs_epi16(a, b); - return _mm256_madd_epi16(product0, kOnes256); + product0 = _mm256_madd_epi16(product0, kOnes256); + acc = _mm256_add_epi32(acc, product0); #endif }; + [[maybe_unused]] auto m256_add_dpbusd_epi32x4 = [=](__m256i& acc, __m256i a0, __m256i b0, __m256i a1, __m256i b1, + __m256i a2, __m256i b2, __m256i a3, __m256i b3) { +#if defined (USE_VNNI) + acc = _mm256_dpbusd_epi32(acc, a0, b0); + acc = _mm256_dpbusd_epi32(acc, a1, b1); + acc = _mm256_dpbusd_epi32(acc, a2, b2); + acc = _mm256_dpbusd_epi32(acc, a3, b3); +#else + __m256i product0 = _mm256_maddubs_epi16(a0, b0); + __m256i product1 = _mm256_maddubs_epi16(a1, b1); + __m256i product2 = _mm256_maddubs_epi16(a2, b2); + __m256i product3 = _mm256_maddubs_epi16(a3, b3); + product0 = _mm256_add_epi16(product0, product1); + product2 = _mm256_add_epi16(product2, product3); + product0 = _mm256_add_epi16(product0, product2); + product0 = _mm256_madd_epi16(product0, kOnes256); + acc = _mm256_add_epi32(acc, product0); #endif + }; +#endif #if defined (USE_SSSE3) [[maybe_unused]] const __m128i kOnes128 = _mm_set1_epi16(1); @@ -236,413 +222,119 @@ namespace Eval::NNUE::Layers { return _mm_cvtsi128_si32(sum) + bias; }; - [[maybe_unused]] auto m128_haddx4 = [](__m128i sum0, __m128i sum1, __m128i sum2, __m128i sum3, __m128i bias) -> __m128i { - sum0 = _mm_hadd_epi32(sum0, sum1); - sum2 = _mm_hadd_epi32(sum2, sum3); - - sum0 = _mm_hadd_epi32(sum0, sum2); - - return _mm_add_epi32(sum0, bias); + [[maybe_unused]] auto m128_add_dpbusd_epi32 = [=](__m128i& acc, __m128i a, __m128i b) { + __m128i product0 = _mm_maddubs_epi16(a, b); + product0 = _mm_madd_epi16(product0, kOnes128); + acc = _mm_add_epi32(acc, product0); }; - [[maybe_unused]] auto m128_dpbusd_epi32 = [=](__m128i a, __m128i b) -> __m128i { - __m128i product0 = _mm_maddubs_epi16(a, b); - return _mm_madd_epi16(product0, kOnes128); + [[maybe_unused]] auto m128_add_dpbusd_epi32x4 = [=](__m128i& acc, __m128i a0, __m128i b0, __m128i a1, __m128i b1, + __m128i a2, __m128i b2, __m128i a3, __m128i b3) { + __m128i product0 = _mm_maddubs_epi16(a0, b0); + __m128i product1 = _mm_maddubs_epi16(a1, b1); + __m128i product2 = _mm_maddubs_epi16(a2, b2); + __m128i product3 = _mm_maddubs_epi16(a3, b3); + product0 = _mm_adds_epi16(product0, product1); + product2 = _mm_adds_epi16(product2, product3); + product0 = _mm_adds_epi16(product0, product2); + product0 = _mm_madd_epi16(product0, kOnes128); + acc = _mm_add_epi32(acc, product0); }; #endif #if defined (USE_AVX512) + using vec_t = __m512i; + #define vec_setzero _mm512_setzero_si512 + #define vec_set_32 _mm512_set1_epi32 + auto& vec_add_dpbusd_32 = m512_add_dpbusd_epi32; + auto& vec_add_dpbusd_32x4 = m512_add_dpbusd_epi32x4; + auto& vec_hadd = m512_hadd; +#elif defined (USE_AVX2) + using vec_t = __m256i; + #define vec_setzero _mm256_setzero_si256 + #define vec_set_32 _mm256_set1_epi32 + auto& vec_add_dpbusd_32 = m256_add_dpbusd_epi32; + auto& vec_add_dpbusd_32x4 = m256_add_dpbusd_epi32x4; + auto& vec_hadd = m256_hadd; +#elif defined (USE_SSSE3) + using vec_t = __m128i; + #define vec_setzero _mm_setzero_si128 + #define vec_set_32 _mm_set1_epi32 + auto& vec_add_dpbusd_32 = m128_add_dpbusd_epi32; + auto& vec_add_dpbusd_32x4 = m128_add_dpbusd_epi32x4; + auto& vec_hadd = m128_hadd; +#endif - constexpr IndexType kNumChunks512 = kPaddedInputDimensions / (kSimdWidth * 2); - constexpr IndexType kNumChunks256 = kPaddedInputDimensions / kSimdWidth; +#if defined (USE_SSSE3) const auto output = reinterpret_cast(buffer); + const auto input_vector = reinterpret_cast(input); - // Since to saturate a zmm register it takes 64 bytes we - // cannot use AVX512 for the smaller affine transforms. - // Instead we fallback to a AVX2 implementation if the - // kInputDimensions isn't a multiple of 64. - // Note that this means that for example for - // kInputDimensions of 96 we fallback to AVX2 even though - // the first 64 elements could be processed with AVX512. - // This is caused by mixing the __m256 and __m512 variables - // required to better handle that case and it would - // require handling more cases statically not to lose performance. - // This should be revisited if such input dimensions are to be considered. - [[maybe_unused]] const auto input_vector512 = reinterpret_cast(input); - [[maybe_unused]] const auto input_vector256 = reinterpret_cast(input); + static_assert(kOutputDimensions % kOutputSimdWidth == 0 || kOutputDimensions == 1); // kOutputDimensions is either 1 or a multiple of kSimdWidth // because then it is also an input dimension. - if constexpr (kOutputDimensions % 16 == 0 && kNumChunks256 == 1) - { - for (IndexType i = 0; i < kOutputDimensions; i += 16) - { - const IndexType offset01a = (i + 0) * kPaddedInputDimensions; - const IndexType offset23a = (i + 2) * kPaddedInputDimensions; - const IndexType offset45a = (i + 4) * kPaddedInputDimensions; - const IndexType offset67a = (i + 6) * kPaddedInputDimensions; - const IndexType offset01b = (i + 8) * kPaddedInputDimensions; - const IndexType offset23b = (i + 10) * kPaddedInputDimensions; - const IndexType offset45b = (i + 12) * kPaddedInputDimensions; - const IndexType offset67b = (i + 14) * kPaddedInputDimensions; - - const __m512i bias = *reinterpret_cast(&biases_[i]); - __m512i* outptr = reinterpret_cast<__m512i*>(&output[i]); - - const auto row01a = *reinterpret_cast(&weights_[offset01a]); - const auto row23a = *reinterpret_cast(&weights_[offset23a]); - const auto row45a = *reinterpret_cast(&weights_[offset45a]); - const auto row67a = *reinterpret_cast(&weights_[offset67a]); - const auto row01b = *reinterpret_cast(&weights_[offset01b]); - const auto row23b = *reinterpret_cast(&weights_[offset23b]); - const auto row45b = *reinterpret_cast(&weights_[offset45b]); - const auto row67b = *reinterpret_cast(&weights_[offset67b]); - - const __m256i in256 = input_vector256[0]; - const __m512i in = _mm512_inserti64x4(_mm512_castsi256_si512(in256), in256, 1); - -#if defined (USE_VNNI) - __m512i sum01a = _mm512_setzero_si512(); - __m512i sum23a = _mm512_setzero_si512(); - __m512i sum45a = _mm512_setzero_si512(); - __m512i sum67a = _mm512_setzero_si512(); - __m512i sum01b = _mm512_setzero_si512(); - __m512i sum23b = _mm512_setzero_si512(); - __m512i sum45b = _mm512_setzero_si512(); - __m512i sum67b = _mm512_setzero_si512(); - - m512_add_dpbusd_epi32(sum01a, in, row01a); - m512_add_dpbusd_epi32(sum23a, in, row23a); - m512_add_dpbusd_epi32(sum45a, in, row45a); - m512_add_dpbusd_epi32(sum67a, in, row67a); - m512_add_dpbusd_epi32(sum01b, in, row01b); - m512_add_dpbusd_epi32(sum23b, in, row23b); - m512_add_dpbusd_epi32(sum45b, in, row45b); - m512_add_dpbusd_epi32(sum67b, in, row67b); -#else - __m512i sum01a = m512_dpbusd_epi32(in, row01a); - __m512i sum23a = m512_dpbusd_epi32(in, row23a); - __m512i sum45a = m512_dpbusd_epi32(in, row45a); - __m512i sum67a = m512_dpbusd_epi32(in, row67a); - __m512i sum01b = m512_dpbusd_epi32(in, row01b); - __m512i sum23b = m512_dpbusd_epi32(in, row23b); - __m512i sum45b = m512_dpbusd_epi32(in, row45b); - __m512i sum67b = m512_dpbusd_epi32(in, row67b); -#endif - - *outptr = m512_hadd256x16( - sum01a, sum23a, sum45a, sum67a, - sum01b, sum23b, sum45b, sum67b, bias); - } - } - else if constexpr (kOutputDimensions % 4 == 0) + if constexpr (kOutputDimensions % kOutputSimdWidth == 0) { - for (IndexType i = 0; i < kOutputDimensions; i += 4) - { - const IndexType offset0 = (i + 0) * kPaddedInputDimensions; - const IndexType offset1 = (i + 1) * kPaddedInputDimensions; - const IndexType offset2 = (i + 2) * kPaddedInputDimensions; - const IndexType offset3 = (i + 3) * kPaddedInputDimensions; + constexpr IndexType kNumChunks = kPaddedInputDimensions / 4; - const __m128i bias = *reinterpret_cast(&biases_[i]); - __m128i* outptr = reinterpret_cast<__m128i*>(&output[i]); + const auto input32 = reinterpret_cast(input); + vec_t* outptr = reinterpret_cast(output); + std::memcpy(output, biases_, kOutputDimensions * sizeof(OutputType)); - if constexpr (kPaddedInputDimensions % (kSimdWidth * 2) == 0) + for (int i = 0; i < (int)kNumChunks - 3; i += 4) { - const auto row0 = reinterpret_cast(&weights_[offset0]); - const auto row1 = reinterpret_cast(&weights_[offset1]); - const auto row2 = reinterpret_cast(&weights_[offset2]); - const auto row3 = reinterpret_cast(&weights_[offset3]); - -#if defined (USE_VNNI) - __m512i sum0 = _mm512_setzero_si512(); - __m512i sum1 = _mm512_setzero_si512(); - __m512i sum2 = _mm512_setzero_si512(); - __m512i sum3 = _mm512_setzero_si512(); - const IndexType kStart = 0; -#else - __m512i sum0 = m512_dpbusd_epi32(input_vector512[0], row0[0]); - __m512i sum1 = m512_dpbusd_epi32(input_vector512[0], row1[0]); - __m512i sum2 = m512_dpbusd_epi32(input_vector512[0], row2[0]); - __m512i sum3 = m512_dpbusd_epi32(input_vector512[0], row3[0]); - const IndexType kStart = 1; -#endif - - for (IndexType j = kStart; j < kNumChunks512; ++j) - { - const __m512i in = input_vector512[j]; - -#if defined (USE_VNNI) - m512_add_dpbusd_epi32(sum0, in, row0[j]); - m512_add_dpbusd_epi32(sum1, in, row1[j]); - m512_add_dpbusd_epi32(sum2, in, row2[j]); - m512_add_dpbusd_epi32(sum3, in, row3[j]); -#else - sum0 = _mm512_add_epi32(sum0, m512_dpbusd_epi32(in, row0[j])); - sum1 = _mm512_add_epi32(sum1, m512_dpbusd_epi32(in, row1[j])); - sum2 = _mm512_add_epi32(sum2, m512_dpbusd_epi32(in, row2[j])); - sum3 = _mm512_add_epi32(sum3, m512_dpbusd_epi32(in, row3[j])); -#endif - } - - *outptr = m512_haddx4(sum0, sum1, sum2, sum3, bias); + const vec_t in0 = vec_set_32(input32[i + 0]); + const vec_t in1 = vec_set_32(input32[i + 1]); + const vec_t in2 = vec_set_32(input32[i + 2]); + const vec_t in3 = vec_set_32(input32[i + 3]); + const auto col0 = reinterpret_cast(&weights_[(i + 0) * kOutputDimensions * 4]); + const auto col1 = reinterpret_cast(&weights_[(i + 1) * kOutputDimensions * 4]); + const auto col2 = reinterpret_cast(&weights_[(i + 2) * kOutputDimensions * 4]); + const auto col3 = reinterpret_cast(&weights_[(i + 3) * kOutputDimensions * 4]); + for (int j = 0; j * kOutputSimdWidth < kOutputDimensions; ++j) + vec_add_dpbusd_32x4(outptr[j], in0, col0[j], in1, col1[j], in2, col2[j], in3, col3[j]); } - else - { - const auto row0 = reinterpret_cast(&weights_[offset0]); - const auto row1 = reinterpret_cast(&weights_[offset1]); - const auto row2 = reinterpret_cast(&weights_[offset2]); - const auto row3 = reinterpret_cast(&weights_[offset3]); - -#if defined (USE_VNNI) - __m256i sum0 = _mm256_setzero_si256(); - __m256i sum1 = _mm256_setzero_si256(); - __m256i sum2 = _mm256_setzero_si256(); - __m256i sum3 = _mm256_setzero_si256(); - const IndexType kStart = 0; -#else - __m256i sum0 = m256_dpbusd_epi32(input_vector256[0], row0[0]); - __m256i sum1 = m256_dpbusd_epi32(input_vector256[0], row1[0]); - __m256i sum2 = m256_dpbusd_epi32(input_vector256[0], row2[0]); - __m256i sum3 = m256_dpbusd_epi32(input_vector256[0], row3[0]); - const IndexType kStart = 1; -#endif - - for (IndexType j = kStart; j < kNumChunks256; ++j) - { - const __m256i in = input_vector256[j]; - -#if defined (USE_VNNI) - m256_add_dpbusd_epi32(sum0, in, row0[j]); - m256_add_dpbusd_epi32(sum1, in, row1[j]); - m256_add_dpbusd_epi32(sum2, in, row2[j]); - m256_add_dpbusd_epi32(sum3, in, row3[j]); -#else - sum0 = _mm256_add_epi32(sum0, m256_dpbusd_epi32(in, row0[j])); - sum1 = _mm256_add_epi32(sum1, m256_dpbusd_epi32(in, row1[j])); - sum2 = _mm256_add_epi32(sum2, m256_dpbusd_epi32(in, row2[j])); - sum3 = _mm256_add_epi32(sum3, m256_dpbusd_epi32(in, row3[j])); -#endif - } - - *outptr = m256_haddx4(sum0, sum1, sum2, sum3, bias); - } - } + for (int i = 0; i < canSaturate16.count; ++i) + output[canSaturate16.ids[i].out] += input[canSaturate16.ids[i].in] * canSaturate16.ids[i].w; } else if constexpr (kOutputDimensions == 1) { - if constexpr (kPaddedInputDimensions % (kSimdWidth * 2) == 0) - { - const auto row0 = reinterpret_cast(&weights_[0]); - -#if defined (USE_VNNI) - __m512i sum0 = _mm512_setzero_si512(); - const IndexType kStart = 0; -#else - __m512i sum0 = m512_dpbusd_epi32(input_vector512[0], row0[0]); - const IndexType kStart = 1; -#endif - - for (IndexType j = kStart; j < kNumChunks512; ++j) - { - const __m512i in = input_vector512[j]; - -#if defined (USE_VNNI) - m512_add_dpbusd_epi32(sum0, in, row0[j]); -#else - sum0 = _mm512_add_epi32(sum0, m512_dpbusd_epi32(in, row0[j])); -#endif - } - - output[0] = m512_hadd(sum0, biases_[0]); - } - else - { - const auto row0 = reinterpret_cast(&weights_[0]); - -#if defined (USE_VNNI) - __m256i sum0 = _mm256_setzero_si256(); - const IndexType kStart = 0; -#else - __m256i sum0 = m256_dpbusd_epi32(input_vector256[0], row0[0]); - const IndexType kStart = 1; -#endif - - for (IndexType j = kStart; j < kNumChunks256; ++j) +#if defined (USE_AVX512) + if constexpr (kPaddedInputDimensions % (kSimdWidth * 2) != 0) { - const __m256i in = input_vector256[j]; - -#if defined (USE_VNNI) - m256_add_dpbusd_epi32(sum0, in, row0[j]); -#else - sum0 = _mm256_add_epi32(sum0, m256_dpbusd_epi32(in, row0[j])); -#endif + constexpr IndexType kNumChunks = kPaddedInputDimensions / kSimdWidth; + const auto input_vector256 = reinterpret_cast(input); + + __m256i sum0 = _mm256_setzero_si256(); + const auto row0 = reinterpret_cast(&weights_[0]); + + for (int j = 0; j < (int)kNumChunks; ++j) + { + const __m256i in = input_vector256[j]; + m256_add_dpbusd_epi32(sum0, in, row0[j]); + } + output[0] = m256_hadd(sum0, biases_[0]); } - - output[0] = m256_hadd(sum0, biases_[0]); - } - } - else - { - // This case can never happen because kOutputDimensions - // is always 1 or a multiple of kSimdWidth. - assert(false); - } - -#elif defined (USE_AVX2) - - constexpr IndexType kNumChunks = kPaddedInputDimensions / kSimdWidth; - - const auto output = reinterpret_cast(buffer); - const auto input_vector = reinterpret_cast(input); - - // kOutputDimensions is either 1 or a multiple of kSimdWidth - // because then it is also an input dimension. - if constexpr (kOutputDimensions % 4 == 0) - { - for (IndexType i = 0; i < kOutputDimensions; i += 4) - { - const IndexType offset0 = (i + 0) * kPaddedInputDimensions; - const IndexType offset1 = (i + 1) * kPaddedInputDimensions; - const IndexType offset2 = (i + 2) * kPaddedInputDimensions; - const IndexType offset3 = (i + 3) * kPaddedInputDimensions; - - const __m128i bias = *reinterpret_cast(&biases_[i]); - __m128i* outptr = reinterpret_cast<__m128i*>(&output[i]); - - const auto row0 = reinterpret_cast(&weights_[offset0]); - const auto row1 = reinterpret_cast(&weights_[offset1]); - const auto row2 = reinterpret_cast(&weights_[offset2]); - const auto row3 = reinterpret_cast(&weights_[offset3]); - -#if defined (USE_VNNI) - __m256i sum0 = _mm256_setzero_si256(); - __m256i sum1 = _mm256_setzero_si256(); - __m256i sum2 = _mm256_setzero_si256(); - __m256i sum3 = _mm256_setzero_si256(); - const IndexType kStart = 0; -#else - __m256i sum0 = m256_dpbusd_epi32(input_vector[0], row0[0]); - __m256i sum1 = m256_dpbusd_epi32(input_vector[0], row1[0]); - __m256i sum2 = m256_dpbusd_epi32(input_vector[0], row2[0]); - __m256i sum3 = m256_dpbusd_epi32(input_vector[0], row3[0]); - const IndexType kStart = 1; + else #endif - - for (IndexType j = kStart; j < kNumChunks; ++j) { - const __m256i in = input_vector[j]; - -#if defined (USE_VNNI) - m256_add_dpbusd_epi32(sum0, in, row0[j]); - m256_add_dpbusd_epi32(sum1, in, row1[j]); - m256_add_dpbusd_epi32(sum2, in, row2[j]); - m256_add_dpbusd_epi32(sum3, in, row3[j]); -#else - sum0 = _mm256_add_epi32(sum0, m256_dpbusd_epi32(in, row0[j])); - sum1 = _mm256_add_epi32(sum1, m256_dpbusd_epi32(in, row1[j])); - sum2 = _mm256_add_epi32(sum2, m256_dpbusd_epi32(in, row2[j])); - sum3 = _mm256_add_epi32(sum3, m256_dpbusd_epi32(in, row3[j])); -#endif - } - - *outptr = m256_haddx4(sum0, sum1, sum2, sum3, bias); - } - } - else if constexpr (kOutputDimensions == 1) - { - const auto row0 = reinterpret_cast(&weights_[0]); - -#if defined (USE_VNNI) - __m256i sum0 = _mm256_setzero_si256(); - const IndexType kStart = 0; -#else - __m256i sum0 = m256_dpbusd_epi32(input_vector[0], row0[0]); - const IndexType kStart = 1; -#endif - - for (IndexType j = kStart; j < kNumChunks; ++j) - { - const __m256i in = input_vector[j]; - -#if defined (USE_VNNI) - m256_add_dpbusd_epi32(sum0, in, row0[j]); +#if defined (USE_AVX512) + constexpr IndexType kNumChunks = kPaddedInputDimensions / (kSimdWidth * 2); #else - sum0 = _mm256_add_epi32(sum0, m256_dpbusd_epi32(in, row0[j])); + constexpr IndexType kNumChunks = kPaddedInputDimensions / kSimdWidth; #endif - } - - output[0] = m256_hadd(sum0, biases_[0]); - } - else - { - // This case can never happen because kOutputDimensions - // is always 1 or a multiple of kSimdWidth. - assert(false); - } - -#elif defined (USE_SSSE3) - - constexpr IndexType kNumChunks = kPaddedInputDimensions / kSimdWidth; - - auto output = reinterpret_cast(buffer); - const auto input_vector = reinterpret_cast(input); - - // kOutputDimensions is either 1 or a multiple of kSimdWidth - // because then it is also an input dimension. - if constexpr (kOutputDimensions % 4 == 0) - { - for (IndexType i = 0; i < kOutputDimensions; i += 4) - { - const IndexType offset0 = (i + 0) * kPaddedInputDimensions; - const IndexType offset1 = (i + 1) * kPaddedInputDimensions; - const IndexType offset2 = (i + 2) * kPaddedInputDimensions; - const IndexType offset3 = (i + 3) * kPaddedInputDimensions; - - const __m128i bias = *reinterpret_cast(&biases_[i]); - __m128i* outptr = reinterpret_cast<__m128i*>(&output[i]); - - const auto row0 = reinterpret_cast(&weights_[offset0]); - const auto row1 = reinterpret_cast(&weights_[offset1]); - const auto row2 = reinterpret_cast(&weights_[offset2]); - const auto row3 = reinterpret_cast(&weights_[offset3]); - - __m128i sum0 = m128_dpbusd_epi32(input_vector[0], row0[0]); - __m128i sum1 = m128_dpbusd_epi32(input_vector[0], row1[0]); - __m128i sum2 = m128_dpbusd_epi32(input_vector[0], row2[0]); - __m128i sum3 = m128_dpbusd_epi32(input_vector[0], row3[0]); - - for (int j = 1; j < (int)kNumChunks; ++j) - { - const __m128i in = input_vector[j]; - - sum0 = _mm_add_epi32(sum0, m128_dpbusd_epi32(in, row0[j])); - sum1 = _mm_add_epi32(sum1, m128_dpbusd_epi32(in, row1[j])); - sum2 = _mm_add_epi32(sum2, m128_dpbusd_epi32(in, row2[j])); - sum3 = _mm_add_epi32(sum3, m128_dpbusd_epi32(in, row3[j])); + vec_t sum0 = vec_setzero(); + const auto row0 = reinterpret_cast(&weights_[0]); + + for (int j = 0; j < (int)kNumChunks; ++j) + { + const vec_t in = input_vector[j]; + vec_add_dpbusd_32(sum0, in, row0[j]); + } + output[0] = vec_hadd(sum0, biases_[0]); } - - *outptr = m128_haddx4(sum0, sum1, sum2, sum3, bias); - } - } - else if constexpr (kOutputDimensions == 1) - { - const auto row0 = reinterpret_cast(&weights_[0]); - - __m128i sum0 = m128_dpbusd_epi32(input_vector[0], row0[0]); - - for (int j = 1; j < (int)kNumChunks; ++j) - sum0 = _mm_add_epi32(sum0, m128_dpbusd_epi32(input_vector[j], row0[j])); - - output[0] = m128_hadd(sum0, biases_[0]); - } - else - { - // This case can never happen because kOutputDimensions - // is always 1 or a multiple of kSimdWidth. - assert(false); } #else @@ -653,11 +345,7 @@ namespace Eval::NNUE::Layers { #if defined(USE_SSE2) constexpr IndexType kNumChunks = kPaddedInputDimensions / kSimdWidth; -#ifndef USE_SSSE3 const __m128i kZeros = _mm_setzero_si128(); -#else - const __m128i kOnes = _mm_set1_epi16(1); -#endif const auto input_vector = reinterpret_cast(input); #elif defined(USE_MMX) @@ -751,8 +439,24 @@ namespace Eval::NNUE::Layers { PreviousLayer previous_layer_; alignas(kCacheLineSize) BiasType biases_[kOutputDimensions]; - alignas(kCacheLineSize) - WeightType weights_[kOutputDimensions * kPaddedInputDimensions]; + alignas(kCacheLineSize) WeightType weights_[kOutputDimensions * kPaddedInputDimensions]; +#if defined (USE_SSSE3) + struct CanSaturate { + int count; + struct Entry { + uint16_t out; + uint16_t in; + int8_t w; + } ids[kPaddedInputDimensions * kOutputDimensions * 3 / 4]; + + void add(int i, int j, int8_t w) { + ids[count].out = i; + ids[count].in = j; + ids[count].w = w; + ++count; + } + } canSaturate16; +#endif }; } // namespace Eval::NNUE::Layers