2 * Copyright (C) 2013 Andrea Mazzoleni
4 * This program is free software: you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation, either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
15 #ifndef __RAID_INTERNAL_H
16 #define __RAID_INTERNAL_H
19 * Supported instruction sets.
21 * It may happen that the assembler is too old to support
22 * all instructions, even if the architecture supports them.
23 * These defines allow to exclude from the build the not supported ones.
25 * If in your project you use a predefined assembler, you can define them
26 * using fixed values, instead of using the HAVE_* defines.
30 /* Includes the project configuration for HAVE_* defines */
33 /* If the compiler supports assembly */
35 /* Autodetect from the compiler */
38 #define CONFIG_X86_32 1
40 #if defined(__x86_64__)
42 #define CONFIG_X86_64 1
46 /* Enables SSE2, SSSE3, AVX2 only if the assembler supports it */
51 #define CONFIG_SSSE3 1
57 #else /* if HAVE_CONFIG_H is not defined */
59 /* Assume that assembly is always supported */
62 #define CONFIG_X86_32 1
65 #if defined(__x86_64__)
67 #define CONFIG_X86_64 1
70 /* Assumes that the assembler supports everything */
73 #define CONFIG_SSSE3 1
79 * Includes anything required for compatibility.
89 #define BUG_ON(a) assert(!(a))
94 #ifndef __always_inline
95 #define __always_inline inline __attribute__((always_inline))
102 #define __aligned(a) __attribute__((aligned(a)))
106 * Align a pointer at the specified size.
108 static __always_inline void *__align_ptr(void *ptr, uintptr_t size)
110 uintptr_t offset = (uintptr_t)ptr;
112 offset = (offset + size - 1U) & ~(size - 1U);
114 return (void *)offset;
118 * Includes the main interface headers.
124 * Internal functions.
126 * These are intended to provide access for testing.
128 int raid_selftest(void);
129 void raid_gen_ref(int nd, int np, size_t size, void **vv);
130 void raid_invert(uint8_t *M, uint8_t *V, int n);
131 void raid_delta_gen(int nr, int *id, int *ip, int nd, size_t size, void **v);
132 void raid_rec1of1(int *id, int nd, size_t size, void **v);
133 void raid_rec2of2_int8(int *id, int *ip, int nd, size_t size, void **vv);
134 void raid_gen1_int32(int nd, size_t size, void **vv);
135 void raid_gen1_int64(int nd, size_t size, void **vv);
136 void raid_gen1_sse2(int nd, size_t size, void **vv);
137 void raid_gen1_avx2(int nd, size_t size, void **vv);
138 void raid_gen2_int32(int nd, size_t size, void **vv);
139 void raid_gen2_int64(int nd, size_t size, void **vv);
140 void raid_gen2_sse2(int nd, size_t size, void **vv);
141 void raid_gen2_avx2(int nd, size_t size, void **vv);
142 void raid_gen2_sse2ext(int nd, size_t size, void **vv);
143 void raid_genz_int32(int nd, size_t size, void **vv);
144 void raid_genz_int64(int nd, size_t size, void **vv);
145 void raid_genz_sse2(int nd, size_t size, void **vv);
146 void raid_genz_sse2ext(int nd, size_t size, void **vv);
147 void raid_genz_avx2ext(int nd, size_t size, void **vv);
148 void raid_gen3_int8(int nd, size_t size, void **vv);
149 void raid_gen3_ssse3(int nd, size_t size, void **vv);
150 void raid_gen3_ssse3ext(int nd, size_t size, void **vv);
151 void raid_gen3_avx2ext(int nd, size_t size, void **vv);
152 void raid_gen4_int8(int nd, size_t size, void **vv);
153 void raid_gen4_ssse3(int nd, size_t size, void **vv);
154 void raid_gen4_ssse3ext(int nd, size_t size, void **vv);
155 void raid_gen4_avx2ext(int nd, size_t size, void **vv);
156 void raid_gen5_int8(int nd, size_t size, void **vv);
157 void raid_gen5_ssse3(int nd, size_t size, void **vv);
158 void raid_gen5_ssse3ext(int nd, size_t size, void **vv);
159 void raid_gen5_avx2ext(int nd, size_t size, void **vv);
160 void raid_gen6_int8(int nd, size_t size, void **vv);
161 void raid_gen6_ssse3(int nd, size_t size, void **vv);
162 void raid_gen6_ssse3ext(int nd, size_t size, void **vv);
163 void raid_gen6_avx2ext(int nd, size_t size, void **vv);
164 void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
165 void raid_rec2_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
166 void raid_recX_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
167 void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
168 void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
169 void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
170 void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
171 void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
172 void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
177 * These are intented to provide access for testing.
179 const char *raid_gen1_tag(void);
180 const char *raid_gen2_tag(void);
181 const char *raid_genz_tag(void);
182 const char *raid_gen3_tag(void);
183 const char *raid_gen4_tag(void);
184 const char *raid_gen5_tag(void);
185 const char *raid_gen6_tag(void);
186 const char *raid_rec1_tag(void);
187 const char *raid_rec2_tag(void);
188 const char *raid_recX_tag(void);
191 * Internal forwarders.
193 extern void (*raid_gen3_ptr)(int nd, size_t size, void **vv);
194 extern void (*raid_genz_ptr)(int nd, size_t size, void **vv);
195 extern void (*raid_gen_ptr[RAID_PARITY_MAX])(
196 int nd, size_t size, void **vv);
197 extern void (*raid_rec_ptr[RAID_PARITY_MAX])(
198 int nr, int *id, int *ip, int nd, size_t size, void **vv);
203 extern const uint8_t raid_gfmul[256][256] __aligned(256);
204 extern const uint8_t raid_gfexp[256] __aligned(256);
205 extern const uint8_t raid_gfinv[256] __aligned(256);
206 extern const uint8_t raid_gfvandermonde[3][256] __aligned(256);
207 extern const uint8_t raid_gfcauchy[6][256] __aligned(256);
208 extern const uint8_t raid_gfcauchypshufb[251][4][2][16] __aligned(256);
209 extern const uint8_t raid_gfmulpshufb[256][2][16] __aligned(256);
210 extern const uint8_t (*raid_gfgen)[256];
211 #define gfmul raid_gfmul
212 #define gfexp raid_gfexp
213 #define gfinv raid_gfinv
214 #define gfvandermonde raid_gfvandermonde
215 #define gfcauchy raid_gfcauchy
216 #define gfgenpshufb raid_gfcauchypshufb
217 #define gfmulpshufb raid_gfmulpshufb
218 #define gfgen raid_gfgen
225 static __always_inline void raid_sse_begin(void)
229 static __always_inline void raid_sse_end(void)
231 /* SSE and AVX code uses non-temporal writes, like MOVNTDQ, */
232 /* that use a weak memory model. To ensure that other processors */
233 /* see correctly the data written, we use a store-store memory */
234 /* barrier at the end of the asm code */
235 asm volatile ("sfence" : : : "memory");
237 /* clobbers registers used in the asm code */
238 /* this is required because in the Windows ABI, */
239 /* registers xmm6-xmm15 should be kept by the callee. */
240 /* this clobber list force the compiler to save any */
241 /* register that needs to be saved */
242 /* we check for __SSE2_ because we require that the */
243 /* compiler supports SSE2 registers in the clobber list */
245 asm volatile ("" : : : "%xmm0", "%xmm1", "%xmm2", "%xmm3");
246 asm volatile ("" : : : "%xmm4", "%xmm5", "%xmm6", "%xmm7");
248 asm volatile ("" : : : "%xmm8", "%xmm9", "%xmm10", "%xmm11");
249 asm volatile ("" : : : "%xmm12", "%xmm13", "%xmm14", "%xmm15");
256 static __always_inline void raid_avx_begin(void)
261 static __always_inline void raid_avx_end(void)
265 /* reset the upper part of the ymm registers */
266 /* to avoid the 70 clocks penality on the next */
267 /* xmm register use */
268 asm volatile ("vzeroupper" : : : "memory");
271 #endif /* CONFIG_X86 */