- transformedFeatures, buffer + SelfBufferSize);
-
-#if defined (USE_AVX512)
-
- [[maybe_unused]] const __m512i Ones512 = _mm512_set1_epi16(1);
-
- [[maybe_unused]] auto m512_hadd = [](__m512i sum, int bias) -> int {
- return _mm512_reduce_add_epi32(sum) + bias;
- };
-
- [[maybe_unused]] auto m512_add_dpbusd_epi32 = [=](__m512i& acc, __m512i a, __m512i b) {
-#if defined (USE_VNNI)
- acc = _mm512_dpbusd_epi32(acc, a, b);
-#else
- __m512i product0 = _mm512_maddubs_epi16(a, b);
- product0 = _mm512_madd_epi16(product0, Ones512);
- acc = _mm512_add_epi32(acc, product0);
-#endif
- };
-
- [[maybe_unused]] auto m512_add_dpbusd_epi32x4 = [=](__m512i& acc, __m512i a0, __m512i b0, __m512i a1, __m512i b1,
- __m512i a2, __m512i b2, __m512i a3, __m512i b3) {
-#if defined (USE_VNNI)
- acc = _mm512_dpbusd_epi32(acc, a0, b0);
- acc = _mm512_dpbusd_epi32(acc, a1, b1);
- acc = _mm512_dpbusd_epi32(acc, a2, b2);
- acc = _mm512_dpbusd_epi32(acc, a3, b3);
-#else
- __m512i product0 = _mm512_maddubs_epi16(a0, b0);
- __m512i product1 = _mm512_maddubs_epi16(a1, b1);
- __m512i product2 = _mm512_maddubs_epi16(a2, b2);
- __m512i product3 = _mm512_maddubs_epi16(a3, b3);
- product0 = _mm512_add_epi16(product0, product1);
- product2 = _mm512_add_epi16(product2, product3);
- product0 = _mm512_add_epi16(product0, product2);
- product0 = _mm512_madd_epi16(product0, Ones512);
- acc = _mm512_add_epi32(acc, product0);
-#endif
- };
-
-#endif
-#if defined (USE_AVX2)
-
- [[maybe_unused]] const __m256i Ones256 = _mm256_set1_epi16(1);
-
- [[maybe_unused]] auto m256_hadd = [](__m256i sum, int bias) -> int {
- __m128i sum128 = _mm_add_epi32(_mm256_castsi256_si128(sum), _mm256_extracti128_si256(sum, 1));
- sum128 = _mm_add_epi32(sum128, _mm_shuffle_epi32(sum128, _MM_PERM_BADC));
- sum128 = _mm_add_epi32(sum128, _mm_shuffle_epi32(sum128, _MM_PERM_CDAB));
- return _mm_cvtsi128_si32(sum128) + bias;
- };
-
- [[maybe_unused]] auto m256_add_dpbusd_epi32 = [=](__m256i& acc, __m256i a, __m256i b) {
-#if defined (USE_VNNI)
- acc = _mm256_dpbusd_epi32(acc, a, b);
-#else
- __m256i product0 = _mm256_maddubs_epi16(a, b);
- product0 = _mm256_madd_epi16(product0, Ones256);
- acc = _mm256_add_epi32(acc, product0);
-#endif
- };
-
- [[maybe_unused]] auto m256_add_dpbusd_epi32x4 = [=](__m256i& acc, __m256i a0, __m256i b0, __m256i a1, __m256i b1,
- __m256i a2, __m256i b2, __m256i a3, __m256i b3) {
-#if defined (USE_VNNI)
- acc = _mm256_dpbusd_epi32(acc, a0, b0);
- acc = _mm256_dpbusd_epi32(acc, a1, b1);
- acc = _mm256_dpbusd_epi32(acc, a2, b2);
- acc = _mm256_dpbusd_epi32(acc, a3, b3);
-#else
- __m256i product0 = _mm256_maddubs_epi16(a0, b0);
- __m256i product1 = _mm256_maddubs_epi16(a1, b1);
- __m256i product2 = _mm256_maddubs_epi16(a2, b2);
- __m256i product3 = _mm256_maddubs_epi16(a3, b3);
- product0 = _mm256_add_epi16(product0, product1);
- product2 = _mm256_add_epi16(product2, product3);
- product0 = _mm256_add_epi16(product0, product2);
- product0 = _mm256_madd_epi16(product0, Ones256);
- acc = _mm256_add_epi32(acc, product0);
-#endif
- };
-
-#endif
-#if defined (USE_SSSE3)
-
- [[maybe_unused]] const __m128i Ones128 = _mm_set1_epi16(1);
-
- [[maybe_unused]] auto m128_hadd = [](__m128i sum, int bias) -> int {
- sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0x4E)); //_MM_PERM_BADC
- sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0xB1)); //_MM_PERM_CDAB
- return _mm_cvtsi128_si32(sum) + bias;
- };
-
- [[maybe_unused]] auto m128_add_dpbusd_epi32 = [=](__m128i& acc, __m128i a, __m128i b) {
- __m128i product0 = _mm_maddubs_epi16(a, b);
- product0 = _mm_madd_epi16(product0, Ones128);
- acc = _mm_add_epi32(acc, product0);
- };
-
- [[maybe_unused]] auto m128_add_dpbusd_epi32x4 = [=](__m128i& acc, __m128i a0, __m128i b0, __m128i a1, __m128i b1,
- __m128i a2, __m128i b2, __m128i a3, __m128i b3) {
- __m128i product0 = _mm_maddubs_epi16(a0, b0);
- __m128i product1 = _mm_maddubs_epi16(a1, b1);
- __m128i product2 = _mm_maddubs_epi16(a2, b2);
- __m128i product3 = _mm_maddubs_epi16(a3, b3);
- product0 = _mm_add_epi16(product0, product1);
- product2 = _mm_add_epi16(product2, product3);
- product0 = _mm_add_epi16(product0, product2);
- product0 = _mm_madd_epi16(product0, Ones128);
- acc = _mm_add_epi32(acc, product0);
- };
-
-#endif