- [[maybe_unused]] static __m512i m512_hadd128x16_interleave(
- __m512i sum0, __m512i sum1, __m512i sum2, __m512i sum3) {
-
- __m512i sum01a = _mm512_unpacklo_epi32(sum0, sum1);
- __m512i sum01b = _mm512_unpackhi_epi32(sum0, sum1);
-
- __m512i sum23a = _mm512_unpacklo_epi32(sum2, sum3);
- __m512i sum23b = _mm512_unpackhi_epi32(sum2, sum3);
-
- __m512i sum01 = _mm512_add_epi32(sum01a, sum01b);
- __m512i sum23 = _mm512_add_epi32(sum23a, sum23b);
-
- __m512i sum0123a = _mm512_unpacklo_epi64(sum01, sum23);
- __m512i sum0123b = _mm512_unpackhi_epi64(sum01, sum23);
-
- return _mm512_add_epi32(sum0123a, sum0123b);
- }
-
- [[maybe_unused]] static __m128i m512_haddx4(
- __m512i sum0, __m512i sum1, __m512i sum2, __m512i sum3,
- __m128i bias) {
-
- __m512i sum = m512_hadd128x16_interleave(sum0, sum1, sum2, sum3);
-
- __m256i sum256lo = _mm512_castsi512_si256(sum);
- __m256i sum256hi = _mm512_extracti64x4_epi64(sum, 1);
-
- sum256lo = _mm256_add_epi32(sum256lo, sum256hi);
-
- __m128i sum128lo = _mm256_castsi256_si128(sum256lo);
- __m128i sum128hi = _mm256_extracti128_si256(sum256lo, 1);
-
- return _mm_add_epi32(_mm_add_epi32(sum128lo, sum128hi), bias);
- }
-
- [[maybe_unused]] static void m512_add_dpbusd_epi32(
- __m512i& acc,
- __m512i a,
- __m512i b) {
-
-# if defined (USE_VNNI)
-# if defined (USE_INLINE_ASM)
- asm(
- "vpdpbusd %[b], %[a], %[acc]\n\t"
- : [acc]"+v"(acc)
- : [a]"v"(a), [b]"vm"(b)
- );
-# else
- acc = _mm512_dpbusd_epi32(acc, a, b);
-# endif
-# else
-# if defined (USE_INLINE_ASM)
- __m512i tmp = _mm512_maddubs_epi16(a, b);
- asm(
- "vpmaddwd %[tmp], %[ones], %[tmp]\n\t"
- "vpaddd %[acc], %[tmp], %[acc]\n\t"
- : [acc]"+v"(acc), [tmp]"+&v"(tmp)
- : [ones]"v"(_mm512_set1_epi16(1))
- );
-# else
- __m512i product0 = _mm512_maddubs_epi16(a, b);
- product0 = _mm512_madd_epi16(product0, _mm512_set1_epi16(1));
- acc = _mm512_add_epi32(acc, product0);
-# endif
-# endif
- }
-
- [[maybe_unused]] static void m512_add_dpbusd_epi32x2(
- __m512i& acc,
- __m512i a0, __m512i b0,
- __m512i a1, __m512i b1) {
-
-# if defined (USE_VNNI)
-# if defined (USE_INLINE_ASM)
- asm(
- "vpdpbusd %[b0], %[a0], %[acc]\n\t"
- "vpdpbusd %[b1], %[a1], %[acc]\n\t"
- : [acc]"+&v"(acc)
- : [a0]"v"(a0), [b0]"vm"(b0), [a1]"v"(a1), [b1]"vm"(b1)
- );
-# else
- acc = _mm512_dpbusd_epi32(acc, a0, b0);
- acc = _mm512_dpbusd_epi32(acc, a1, b1);
-# endif
-# else
-# if defined (USE_INLINE_ASM)
- __m512i tmp0 = _mm512_maddubs_epi16(a0, b0);
- __m512i tmp1 = _mm512_maddubs_epi16(a1, b1);
- asm(
- "vpmaddwd %[tmp0], %[ones], %[tmp0]\n\t"
- "vpmaddwd %[tmp1], %[ones], %[tmp1]\n\t"
- "vpaddd %[tmp0], %[tmp1], %[tmp0]\n\t"
- "vpaddd %[acc], %[tmp0], %[acc]\n\t"
- : [acc]"+v"(acc), [tmp0]"+&v"(tmp0), [tmp1]"+&v"(tmp1)
- : [ones]"v"(_mm512_set1_epi16(1))
- );
-# else
- __m512i product0 = _mm512_maddubs_epi16(a0, b0);
- __m512i product1 = _mm512_maddubs_epi16(a1, b1);
- product0 = _mm512_madd_epi16(product0, _mm512_set1_epi16(1));
- product1 = _mm512_madd_epi16(product1, _mm512_set1_epi16(1));
- acc = _mm512_add_epi32(acc, _mm512_add_epi32(product0, product1));
-# endif
-# endif
- }
+[[maybe_unused]] static __m512i
+m512_hadd128x16_interleave(__m512i sum0, __m512i sum1, __m512i sum2, __m512i sum3) {
+
+ __m512i sum01a = _mm512_unpacklo_epi32(sum0, sum1);
+ __m512i sum01b = _mm512_unpackhi_epi32(sum0, sum1);
+
+ __m512i sum23a = _mm512_unpacklo_epi32(sum2, sum3);
+ __m512i sum23b = _mm512_unpackhi_epi32(sum2, sum3);
+
+ __m512i sum01 = _mm512_add_epi32(sum01a, sum01b);
+ __m512i sum23 = _mm512_add_epi32(sum23a, sum23b);
+
+ __m512i sum0123a = _mm512_unpacklo_epi64(sum01, sum23);
+ __m512i sum0123b = _mm512_unpackhi_epi64(sum01, sum23);
+
+ return _mm512_add_epi32(sum0123a, sum0123b);
+}
+
+[[maybe_unused]] static void m512_add_dpbusd_epi32(__m512i& acc, __m512i a, __m512i b) {
+
+ #if defined(USE_VNNI)
+ acc = _mm512_dpbusd_epi32(acc, a, b);
+ #else
+ __m512i product0 = _mm512_maddubs_epi16(a, b);
+ product0 = _mm512_madd_epi16(product0, _mm512_set1_epi16(1));
+ acc = _mm512_add_epi32(acc, product0);
+ #endif
+}
+
+[[maybe_unused]] static void
+m512_add_dpbusd_epi32x2(__m512i& acc, __m512i a0, __m512i b0, __m512i a1, __m512i b1) {
+
+ #if defined(USE_VNNI)
+ acc = _mm512_dpbusd_epi32(acc, a0, b0);
+ acc = _mm512_dpbusd_epi32(acc, a1, b1);
+ #else
+ __m512i product0 = _mm512_maddubs_epi16(a0, b0);
+ __m512i product1 = _mm512_maddubs_epi16(a1, b1);
+ product0 = _mm512_madd_epi16(product0, _mm512_set1_epi16(1));
+ product1 = _mm512_madd_epi16(product1, _mm512_set1_epi16(1));
+ acc = _mm512_add_epi32(acc, _mm512_add_epi32(product0, product1));
+ #endif
+}