X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=src%2Fnnue%2Flayers%2Fsimd.h;h=f478cd7819f9aee7d9e9d6c77d2cf2af879ae841;hb=8192945870967fb9c8801247d0b040b2bc657443;hp=231f78912168a5c210c96d097ba7c7176d1a55db;hpb=da5bcec481dc22ad2de2e2c8e5c6e8304a373445;p=stockfish

diff --git a/src/nnue/layers/simd.h b/src/nnue/layers/simd.h
index 231f7891..f478cd78 100644
--- a/src/nnue/layers/simd.h
+++ b/src/nnue/layers/simd.h
@@ -38,21 +38,6 @@
 # include <arm_neon.h>
 #endif
 
-// The inline asm is only safe for GCC, where it is necessary to get good codegen.
-// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101693
-// Clang does fine without it.
-// Play around here: https://godbolt.org/z/7EWqrYq51
-#if (defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER))
-#define USE_INLINE_ASM
-#endif
-
-// Use either the AVX512 or AVX-VNNI version of the VNNI instructions.
-#if defined(USE_AVXVNNI)
-#define VNNI_PREFIX "%{vex%} "
-#else
-#define VNNI_PREFIX ""
-#endif
-
 namespace Stockfish::Simd {
 
 #if defined (USE_AVX512)
@@ -94,52 +79,17 @@ namespace Stockfish::Simd {
       return _mm512_add_epi32(sum0123a, sum0123b);
     }
 
-    [[maybe_unused]] static __m128i m512_haddx4(
-        __m512i sum0, __m512i sum1, __m512i sum2, __m512i sum3,
-        __m128i bias) {
-
-      __m512i sum = m512_hadd128x16_interleave(sum0, sum1, sum2, sum3);
-
-      __m256i sum256lo = _mm512_castsi512_si256(sum);
-      __m256i sum256hi = _mm512_extracti64x4_epi64(sum, 1);
-
-      sum256lo = _mm256_add_epi32(sum256lo, sum256hi);
-
-      __m128i sum128lo = _mm256_castsi256_si128(sum256lo);
-      __m128i sum128hi = _mm256_extracti128_si256(sum256lo, 1);
-
-      return _mm_add_epi32(_mm_add_epi32(sum128lo, sum128hi), bias);
-    }
-
     [[maybe_unused]] static void m512_add_dpbusd_epi32(
         __m512i& acc,
         __m512i a,
         __m512i b) {
 
 # if defined (USE_VNNI)
-#   if defined (USE_INLINE_ASM)
-      asm(
-        "vpdpbusd %[b], %[a], %[acc]\n\t"
-        : [acc]"+v"(acc)
-        : [a]"v"(a), [b]"vm"(b)
-      );
-#   else
       acc = _mm512_dpbusd_epi32(acc, a, b);
-#   endif
 # else
-#   if defined (USE_INLINE_ASM)
-      __m512i tmp = _mm512_maddubs_epi16(a, b);
-      asm(
-          "vpmaddwd    %[tmp], %[ones], %[tmp]\n\t"
-          "vpaddd      %[acc], %[tmp], %[acc]\n\t"
-          : [acc]"+v"(acc), [tmp]"+&v"(tmp)
-          : [ones]"v"(_mm512_set1_epi16(1))
-      );
-#   else
       __m512i product0 = _mm512_maddubs_epi16(a, b);
       product0 = _mm512_madd_epi16(product0, _mm512_set1_epi16(1));
       acc = _mm512_add_epi32(acc, product0);
-#   endif
 # endif
     }
 
@@ -149,35 +99,14 @@ namespace Stockfish::Simd {
         __m512i a1, __m512i b1) {
 
 # if defined (USE_VNNI)
-#   if defined (USE_INLINE_ASM)
-      asm(
-        "vpdpbusd %[b0], %[a0], %[acc]\n\t"
-        "vpdpbusd %[b1], %[a1], %[acc]\n\t"
-        : [acc]"+&v"(acc)
-        : [a0]"v"(a0), [b0]"vm"(b0), [a1]"v"(a1), [b1]"vm"(b1)
-      );
-#   else
       acc = _mm512_dpbusd_epi32(acc, a0, b0);
       acc = _mm512_dpbusd_epi32(acc, a1, b1);
-#   endif
 # else
-#   if defined (USE_INLINE_ASM)
-      __m512i tmp0 = _mm512_maddubs_epi16(a0, b0);
-      __m512i tmp1 = _mm512_maddubs_epi16(a1, b1);
-      asm(
-          "vpaddsw     %[tmp0], %[tmp1], %[tmp0]\n\t"
-          "vpmaddwd    %[tmp0], %[ones], %[tmp0]\n\t"
-          "vpaddd      %[acc], %[tmp0], %[acc]\n\t"
-          : [acc]"+v"(acc), [tmp0]"+&v"(tmp0)
-          : [tmp1]"v"(tmp1), [ones]"v"(_mm512_set1_epi16(1))
-      );
-#   else
       __m512i product0 = _mm512_maddubs_epi16(a0, b0);
       __m512i product1 = _mm512_maddubs_epi16(a1, b1);
-      product0 = _mm512_adds_epi16(product0, product1);
       product0 = _mm512_madd_epi16(product0, _mm512_set1_epi16(1));
-      acc = _mm512_add_epi32(acc, product0);
-#   endif
+      product1 = _mm512_madd_epi16(product1, _mm512_set1_epi16(1));
+      acc = _mm512_add_epi32(acc, _mm512_add_epi32(product0, product1));
 # endif
     }
 
@@ -192,50 +121,17 @@ namespace Stockfish::Simd {
       return _mm_cvtsi128_si32(sum128) + bias;
     }
 
-    [[maybe_unused]] static __m128i m256_haddx4(
-        __m256i sum0, __m256i sum1, __m256i sum2, __m256i sum3,
-        __m128i bias) {
-
-      sum0 = _mm256_hadd_epi32(sum0, sum1);
-      sum2 = _mm256_hadd_epi32(sum2, sum3);
-
-      sum0 = _mm256_hadd_epi32(sum0, sum2);
-
-      __m128i sum128lo = _mm256_castsi256_si128(sum0);
-      __m128i sum128hi = _mm256_extracti128_si256(sum0, 1);
-
-      return _mm_add_epi32(_mm_add_epi32(sum128lo, sum128hi), bias);
-    }
-
     [[maybe_unused]] static void m256_add_dpbusd_epi32(
         __m256i& acc,
         __m256i a,
         __m256i b) {
 
 # if defined (USE_VNNI)
-#   if defined (USE_INLINE_ASM)
-      asm(
-        VNNI_PREFIX "vpdpbusd %[b], %[a], %[acc]\n\t"
-        : [acc]"+v"(acc)
-        : [a]"v"(a), [b]"vm"(b)
-      );
-#   else
       acc = _mm256_dpbusd_epi32(acc, a, b);
-#   endif
 # else
-#   if defined (USE_INLINE_ASM)
-      __m256i tmp = _mm256_maddubs_epi16(a, b);
-      asm(
-          "vpmaddwd    %[tmp], %[ones], %[tmp]\n\t"
-          "vpaddd      %[acc], %[tmp], %[acc]\n\t"
-          : [acc]"+v"(acc), [tmp]"+&v"(tmp)
-          : [ones]"v"(_mm256_set1_epi16(1))
-      );
-#   else
       __m256i product0 = _mm256_maddubs_epi16(a, b);
       product0 = _mm256_madd_epi16(product0, _mm256_set1_epi16(1));
       acc = _mm256_add_epi32(acc, product0);
-#   endif
 # endif
     }
 
@@ -245,35 +141,14 @@ namespace Stockfish::Simd {
         __m256i a1, __m256i b1) {
 
 # if defined (USE_VNNI)
-#   if defined (USE_INLINE_ASM)
-      asm(
-        VNNI_PREFIX "vpdpbusd %[b0], %[a0], %[acc]\n\t"
-        VNNI_PREFIX "vpdpbusd %[b1], %[a1], %[acc]\n\t"
-        : [acc]"+&v"(acc)
-        : [a0]"v"(a0), [b0]"vm"(b0), [a1]"v"(a1), [b1]"vm"(b1)
-      );
-#   else
       acc = _mm256_dpbusd_epi32(acc, a0, b0);
       acc = _mm256_dpbusd_epi32(acc, a1, b1);
-#   endif
 # else
-#   if defined (USE_INLINE_ASM)
-      __m256i tmp0 = _mm256_maddubs_epi16(a0, b0);
-      __m256i tmp1 = _mm256_maddubs_epi16(a1, b1);
-      asm(
-          "vpaddsw     %[tmp0], %[tmp1], %[tmp0]\n\t"
-          "vpmaddwd    %[tmp0], %[ones], %[tmp0]\n\t"
-          "vpaddd      %[acc], %[tmp0], %[acc]\n\t"
-          : [acc]"+v"(acc), [tmp0]"+&v"(tmp0)
-          : [tmp1]"v"(tmp1), [ones]"v"(_mm256_set1_epi16(1))
-      );
-#   else
       __m256i product0 = _mm256_maddubs_epi16(a0, b0);
       __m256i product1 = _mm256_maddubs_epi16(a1, b1);
-      product0 = _mm256_adds_epi16(product0, product1);
       product0 = _mm256_madd_epi16(product0, _mm256_set1_epi16(1));
-      acc = _mm256_add_epi32(acc, product0);
-#   endif
+      product1 = _mm256_madd_epi16(product1, _mm256_set1_epi16(1));
+      acc = _mm256_add_epi32(acc, _mm256_add_epi32(product0, product1));
 # endif
     }
 
@@ -287,34 +162,14 @@ namespace Stockfish::Simd {
       return _mm_cvtsi128_si32(sum) + bias;
     }
 
-    [[maybe_unused]] static __m128i m128_haddx4(
-        __m128i sum0, __m128i sum1, __m128i sum2, __m128i sum3,
-        __m128i bias) {
-
-      sum0 = _mm_hadd_epi32(sum0, sum1);
-      sum2 = _mm_hadd_epi32(sum2, sum3);
-      sum0 = _mm_hadd_epi32(sum0, sum2);
-      return _mm_add_epi32(sum0, bias);
-    }
-
     [[maybe_unused]] static void m128_add_dpbusd_epi32(
         __m128i& acc,
         __m128i a,
         __m128i b) {
 
-#   if defined (USE_INLINE_ASM)
-      __m128i tmp = _mm_maddubs_epi16(a, b);
-      asm(
-          "pmaddwd    %[ones], %[tmp]\n\t"
-          "paddd      %[tmp], %[acc]\n\t"
-          : [acc]"+v"(acc), [tmp]"+&v"(tmp)
-          : [ones]"v"(_mm_set1_epi16(1))
-      );
-#   else
       __m128i product0 = _mm_maddubs_epi16(a, b);
       product0 = _mm_madd_epi16(product0, _mm_set1_epi16(1));
       acc = _mm_add_epi32(acc, product0);
-#   endif
     }
 
     [[maybe_unused]] static void m128_add_dpbusd_epi32x2(
@@ -322,25 +177,32 @@ namespace Stockfish::Simd {
         __m128i a0, __m128i b0,
         __m128i a1, __m128i b1) {
 
-#   if defined (USE_INLINE_ASM)
-      __m128i tmp0 = _mm_maddubs_epi16(a0, b0);
-      __m128i tmp1 = _mm_maddubs_epi16(a1, b1);
-      asm(
-          "paddsw     %[tmp1], %[tmp0]\n\t"
-          "pmaddwd    %[ones], %[tmp0]\n\t"
-          "paddd      %[tmp0], %[acc]\n\t"
-          : [acc]"+v"(acc), [tmp0]"+&v"(tmp0)
-          : [tmp1]"v"(tmp1), [ones]"v"(_mm_set1_epi16(1))
-      );
-#   else
       __m128i product0 = _mm_maddubs_epi16(a0, b0);
       __m128i product1 = _mm_maddubs_epi16(a1, b1);
-      product0 = _mm_adds_epi16(product0, product1);
       product0 = _mm_madd_epi16(product0, _mm_set1_epi16(1));
-      acc = _mm_add_epi32(acc, product0);
-#   endif
+      product1 = _mm_madd_epi16(product1, _mm_set1_epi16(1));
+      acc = _mm_add_epi32(acc, _mm_add_epi32(product0, product1));
+    }
+
+#endif
+
+#if defined (USE_NEON_DOTPROD)
+
+    [[maybe_unused]] static void dotprod_m128_add_dpbusd_epi32x2(
+        int32x4_t& acc,
+        int8x16_t a0, int8x16_t b0,
+        int8x16_t a1, int8x16_t b1) {
+
+        acc = vdotq_s32(acc, a0, b0);
+        acc = vdotq_s32(acc, a1, b1);
     }
 
+    [[maybe_unused]] static void dotprod_m128_add_dpbusd_epi32(
+        int32x4_t& acc,
+        int8x16_t a, int8x16_t b) {
+
+        acc = vdotq_s32(acc, a, b);
+    }
 #endif
 
 #if defined (USE_NEON)
@@ -357,19 +219,6 @@ namespace Stockfish::Simd {
       return neon_m128_reduce_add_epi32(sum) + bias;
     }
 
-    [[maybe_unused]] static int32x4_t neon_m128_haddx4(
-        int32x4_t sum0, int32x4_t sum1, int32x4_t sum2, int32x4_t sum3,
-        int32x4_t bias) {
-
-      int32x4_t hsums {
-        neon_m128_reduce_add_epi32(sum0),
-        neon_m128_reduce_add_epi32(sum1),
-        neon_m128_reduce_add_epi32(sum2),
-        neon_m128_reduce_add_epi32(sum3)
-      };
-      return vaddq_s32(hsums, bias);
-    }
-
     [[maybe_unused]] static void neon_m128_add_dpbusd_epi32x2(
         int32x4_t& acc,
         int8x8_t a0, int8x8_t b0,
@@ -379,9 +228,19 @@ namespace Stockfish::Simd {
       product = vmlal_s8(product, a1, b1);
       acc = vpadalq_s16(acc, product);
     }
-
 #endif
 
+#if USE_NEON >= 8
+    [[maybe_unused]] static void neon_m128_add_dpbusd_epi32(
+        int32x4_t& acc,
+        int8x16_t a, int8x16_t b) {
+
+      int16x8_t product0 = vmull_s8(vget_low_s8(a), vget_low_s8(b));
+      int16x8_t product1 = vmull_high_s8(a, b);
+      int16x8_t sum = vpaddq_s16(product0, product1);
+      acc = vpadalq_s16(acc, sum);
+    }
+#endif
 }
 
 #endif // STOCKFISH_SIMD_H_INCLUDED