2 Stockfish, a UCI chess playing engine derived from Glaurung 2.1
3 Copyright (C) 2004-2023 The Stockfish developers (see AUTHORS file)
5 Stockfish is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation, either version 3 of the License, or
8 (at your option) any later version.
10 Stockfish is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
19 #ifndef STOCKFISH_SIMD_H_INCLUDED
20 #define STOCKFISH_SIMD_H_INCLUDED
23 #include <immintrin.h>
25 #elif defined(USE_SSE41)
26 #include <smmintrin.h>
28 #elif defined(USE_SSSE3)
29 #include <tmmintrin.h>
31 #elif defined(USE_SSE2)
32 #include <emmintrin.h>
34 #elif defined(USE_NEON)
38 namespace Stockfish::Simd {
40 #if defined(USE_AVX512)
42 [[maybe_unused]] static int m512_hadd(__m512i sum, int bias) {
43 return _mm512_reduce_add_epi32(sum) + bias;
48 sum0 = [zmm0.i128[0], zmm0.i128[1], zmm0.i128[2], zmm0.i128[3]]
49 sum1 = [zmm1.i128[0], zmm1.i128[1], zmm1.i128[2], zmm1.i128[3]]
50 sum2 = [zmm2.i128[0], zmm2.i128[1], zmm2.i128[2], zmm2.i128[3]]
51 sum3 = [zmm3.i128[0], zmm3.i128[1], zmm3.i128[2], zmm3.i128[3]]
55 reduce_add_epi32(zmm0.i128[0]), reduce_add_epi32(zmm1.i128[0]), reduce_add_epi32(zmm2.i128[0]), reduce_add_epi32(zmm3.i128[0]),
56 reduce_add_epi32(zmm0.i128[1]), reduce_add_epi32(zmm1.i128[1]), reduce_add_epi32(zmm2.i128[1]), reduce_add_epi32(zmm3.i128[1]),
57 reduce_add_epi32(zmm0.i128[2]), reduce_add_epi32(zmm1.i128[2]), reduce_add_epi32(zmm2.i128[2]), reduce_add_epi32(zmm3.i128[2]),
58 reduce_add_epi32(zmm0.i128[3]), reduce_add_epi32(zmm1.i128[3]), reduce_add_epi32(zmm2.i128[3]), reduce_add_epi32(zmm3.i128[3])
61 [[maybe_unused]] static __m512i
62 m512_hadd128x16_interleave(__m512i sum0, __m512i sum1, __m512i sum2, __m512i sum3) {
64 __m512i sum01a = _mm512_unpacklo_epi32(sum0, sum1);
65 __m512i sum01b = _mm512_unpackhi_epi32(sum0, sum1);
67 __m512i sum23a = _mm512_unpacklo_epi32(sum2, sum3);
68 __m512i sum23b = _mm512_unpackhi_epi32(sum2, sum3);
70 __m512i sum01 = _mm512_add_epi32(sum01a, sum01b);
71 __m512i sum23 = _mm512_add_epi32(sum23a, sum23b);
73 __m512i sum0123a = _mm512_unpacklo_epi64(sum01, sum23);
74 __m512i sum0123b = _mm512_unpackhi_epi64(sum01, sum23);
76 return _mm512_add_epi32(sum0123a, sum0123b);
79 [[maybe_unused]] static void m512_add_dpbusd_epi32(__m512i& acc, __m512i a, __m512i b) {
82 acc = _mm512_dpbusd_epi32(acc, a, b);
84 __m512i product0 = _mm512_maddubs_epi16(a, b);
85 product0 = _mm512_madd_epi16(product0, _mm512_set1_epi16(1));
86 acc = _mm512_add_epi32(acc, product0);
90 [[maybe_unused]] static void
91 m512_add_dpbusd_epi32x2(__m512i& acc, __m512i a0, __m512i b0, __m512i a1, __m512i b1) {
94 acc = _mm512_dpbusd_epi32(acc, a0, b0);
95 acc = _mm512_dpbusd_epi32(acc, a1, b1);
97 __m512i product0 = _mm512_maddubs_epi16(a0, b0);
98 __m512i product1 = _mm512_maddubs_epi16(a1, b1);
99 product0 = _mm512_madd_epi16(product0, _mm512_set1_epi16(1));
100 product1 = _mm512_madd_epi16(product1, _mm512_set1_epi16(1));
101 acc = _mm512_add_epi32(acc, _mm512_add_epi32(product0, product1));
107 #if defined(USE_AVX2)
109 [[maybe_unused]] static int m256_hadd(__m256i sum, int bias) {
110 __m128i sum128 = _mm_add_epi32(_mm256_castsi256_si128(sum), _mm256_extracti128_si256(sum, 1));
111 sum128 = _mm_add_epi32(sum128, _mm_shuffle_epi32(sum128, _MM_PERM_BADC));
112 sum128 = _mm_add_epi32(sum128, _mm_shuffle_epi32(sum128, _MM_PERM_CDAB));
113 return _mm_cvtsi128_si32(sum128) + bias;
116 [[maybe_unused]] static void m256_add_dpbusd_epi32(__m256i& acc, __m256i a, __m256i b) {
118 #if defined(USE_VNNI)
119 acc = _mm256_dpbusd_epi32(acc, a, b);
121 __m256i product0 = _mm256_maddubs_epi16(a, b);
122 product0 = _mm256_madd_epi16(product0, _mm256_set1_epi16(1));
123 acc = _mm256_add_epi32(acc, product0);
127 [[maybe_unused]] static void
128 m256_add_dpbusd_epi32x2(__m256i& acc, __m256i a0, __m256i b0, __m256i a1, __m256i b1) {
130 #if defined(USE_VNNI)
131 acc = _mm256_dpbusd_epi32(acc, a0, b0);
132 acc = _mm256_dpbusd_epi32(acc, a1, b1);
134 __m256i product0 = _mm256_maddubs_epi16(a0, b0);
135 __m256i product1 = _mm256_maddubs_epi16(a1, b1);
136 product0 = _mm256_madd_epi16(product0, _mm256_set1_epi16(1));
137 product1 = _mm256_madd_epi16(product1, _mm256_set1_epi16(1));
138 acc = _mm256_add_epi32(acc, _mm256_add_epi32(product0, product1));
144 #if defined(USE_SSSE3)
146 [[maybe_unused]] static int m128_hadd(__m128i sum, int bias) {
147 sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0x4E)); //_MM_PERM_BADC
148 sum = _mm_add_epi32(sum, _mm_shuffle_epi32(sum, 0xB1)); //_MM_PERM_CDAB
149 return _mm_cvtsi128_si32(sum) + bias;
152 [[maybe_unused]] static void m128_add_dpbusd_epi32(__m128i& acc, __m128i a, __m128i b) {
154 __m128i product0 = _mm_maddubs_epi16(a, b);
155 product0 = _mm_madd_epi16(product0, _mm_set1_epi16(1));
156 acc = _mm_add_epi32(acc, product0);
159 [[maybe_unused]] static void
160 m128_add_dpbusd_epi32x2(__m128i& acc, __m128i a0, __m128i b0, __m128i a1, __m128i b1) {
162 __m128i product0 = _mm_maddubs_epi16(a0, b0);
163 __m128i product1 = _mm_maddubs_epi16(a1, b1);
164 product0 = _mm_madd_epi16(product0, _mm_set1_epi16(1));
165 product1 = _mm_madd_epi16(product1, _mm_set1_epi16(1));
166 acc = _mm_add_epi32(acc, _mm_add_epi32(product0, product1));
171 #if defined(USE_NEON_DOTPROD)
173 [[maybe_unused]] static void dotprod_m128_add_dpbusd_epi32x2(
174 int32x4_t& acc, int8x16_t a0, int8x16_t b0, int8x16_t a1, int8x16_t b1) {
176 acc = vdotq_s32(acc, a0, b0);
177 acc = vdotq_s32(acc, a1, b1);
180 [[maybe_unused]] static void
181 dotprod_m128_add_dpbusd_epi32(int32x4_t& acc, int8x16_t a, int8x16_t b) {
183 acc = vdotq_s32(acc, a, b);
187 #if defined(USE_NEON)
189 [[maybe_unused]] static int neon_m128_reduce_add_epi32(int32x4_t s) {
191 return vaddvq_s32(s);
193 return s[0] + s[1] + s[2] + s[3];
197 [[maybe_unused]] static int neon_m128_hadd(int32x4_t sum, int bias) {
198 return neon_m128_reduce_add_epi32(sum) + bias;
201 [[maybe_unused]] static void
202 neon_m128_add_dpbusd_epi32x2(int32x4_t& acc, int8x8_t a0, int8x8_t b0, int8x8_t a1, int8x8_t b1) {
204 int16x8_t product = vmull_s8(a0, b0);
205 product = vmlal_s8(product, a1, b1);
206 acc = vpadalq_s16(acc, product);
211 [[maybe_unused]] static void neon_m128_add_dpbusd_epi32(int32x4_t& acc, int8x16_t a, int8x16_t b) {
213 int16x8_t product0 = vmull_s8(vget_low_s8(a), vget_low_s8(b));
214 int16x8_t product1 = vmull_high_s8(a, b);
215 int16x8_t sum = vpaddq_s16(product0, product1);
216 acc = vpadalq_s16(acc, sum);
221 #endif // STOCKFISH_SIMD_H_INCLUDED