1 #ifndef __TOOLS_LINUX_ATOMIC_H
2 #define __TOOLS_LINUX_ATOMIC_H
4 #include <linux/compiler.h>
5 #include <linux/types.h>
21 #include <urcu/uatomic.h>
23 #if (CAA_BITS_PER_LONG != 64)
24 #define ATOMIC64_SPINLOCK
27 #define __ATOMIC_READ(p) uatomic_read(p)
28 #define __ATOMIC_SET(p, v) uatomic_set(p, v)
29 #define __ATOMIC_ADD_RETURN(v, p) uatomic_add_return(p, v)
30 #define __ATOMIC_SUB_RETURN(v, p) uatomic_sub_return(p, v)
31 #define __ATOMIC_ADD(v, p) uatomic_add(p, v)
32 #define __ATOMIC_SUB(v, p) uatomic_sub(p, v)
33 #define __ATOMIC_INC(p) uatomic_inc(p)
34 #define __ATOMIC_DEC(p) uatomic_dec(p)
35 #define __ATOMIC_AND(v, p) uatomic_and(p, v)
36 #define __ATOMIC_OR(v, p) uatomic_or(p, v)
38 #define xchg(p, v) uatomic_xchg(p, v)
39 #define xchg_acquire(p, v) uatomic_xchg(p, v)
40 #define cmpxchg(p, old, new) uatomic_cmpxchg(p, old, new)
41 #define cmpxchg_acquire(p, old, new) uatomic_cmpxchg(p, old, new)
42 #define cmpxchg_release(p, old, new) uatomic_cmpxchg(p, old, new)
44 #define smp_mb__before_atomic() cmm_smp_mb__before_uatomic_add()
45 #define smp_mb__after_atomic() cmm_smp_mb__after_uatomic_add()
46 #define smp_wmb() cmm_smp_wmb()
47 #define smp_rmb() cmm_smp_rmb()
48 #define smp_mb() cmm_smp_mb()
49 #define smp_read_barrier_depends() cmm_smp_read_barrier_depends()
50 #define smp_acquire__after_ctrl_dep() cmm_smp_mb()
52 #else /* C11_ATOMICS */
54 #define __ATOMIC_READ(p) __atomic_load_n(p, __ATOMIC_RELAXED)
55 #define __ATOMIC_SET(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED)
56 #define __ATOMIC_ADD_RETURN(v, p) __atomic_add_fetch(p, v, __ATOMIC_RELAXED)
57 #define __ATOMIC_ADD_RETURN_RELEASE(v, p) \
58 __atomic_add_fetch(p, v, __ATOMIC_RELEASE)
59 #define __ATOMIC_SUB_RETURN(v, p) __atomic_sub_fetch(p, v, __ATOMIC_RELAXED)
60 #define __ATOMIC_SUB_RETURN_RELEASE(v, p) \
61 __atomic_sub_fetch(p, v, __ATOMIC_RELEASE)
62 #define __ATOMIC_AND(p) __atomic_and_fetch(p, v, __ATOMIC_RELAXED)
63 #define __ATOMIC_OR(p) __atomic_or_fetch(p, v, __ATOMIC_RELAXED)
65 #define xchg(p, v) __atomic_exchange_n(p, v, __ATOMIC_SEQ_CST)
66 #define xchg_acquire(p, v) __atomic_exchange_n(p, v, __ATOMIC_ACQUIRE)
68 #define cmpxchg(p, old, new) \
70 typeof(*(p)) __old = (old); \
72 __atomic_compare_exchange_n((p), &__old, new, false, \
78 #define cmpxchg_acquire(p, old, new) \
80 typeof(*(p)) __old = (old); \
82 __atomic_compare_exchange_n((p), &__old, new, false, \
88 #define cmpxchg_release(p, old, new) \
90 typeof(*(p)) __old = (old); \
92 __atomic_compare_exchange_n((p), &__old, new, false, \
98 #define smp_mb__before_atomic() __atomic_thread_fence(__ATOMIC_SEQ_CST)
99 #define smp_mb__after_atomic() __atomic_thread_fence(__ATOMIC_SEQ_CST)
100 #define smp_wmb() __atomic_thread_fence(__ATOMIC_SEQ_CST)
101 #define smp_rmb() __atomic_thread_fence(__ATOMIC_SEQ_CST)
102 #define smp_mb() __atomic_thread_fence(__ATOMIC_SEQ_CST)
103 #define smp_read_barrier_depends()
107 #define smp_store_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)
109 #define smp_load_acquire(p) \
111 typeof(*p) ___p1 = READ_ONCE(*p); \
116 #define smp_store_release(p, v) \
122 /* atomic interface: */
125 #define __ATOMIC_ADD(i, v) __ATOMIC_ADD_RETURN(i, v)
128 #ifndef __ATOMIC_ADD_RETURN_RELEASE
129 #define __ATOMIC_ADD_RETURN_RELEASE(i, v) \
130 ({ smp_mb__before_atomic(); __ATOMIC_ADD_RETURN(i, v); })
133 #ifndef __ATOMIC_SUB_RETURN_RELEASE
134 #define __ATOMIC_SUB_RETURN_RELEASE(i, v) \
135 ({ smp_mb__before_atomic(); __ATOMIC_SUB_RETURN(i, v); })
139 #define __ATOMIC_SUB(i, v) __ATOMIC_SUB_RETURN(i, v)
142 #ifndef __ATOMIC_INC_RETURN
143 #define __ATOMIC_INC_RETURN(v) __ATOMIC_ADD_RETURN(1, v)
146 #ifndef __ATOMIC_DEC_RETURN
147 #define __ATOMIC_DEC_RETURN(v) __ATOMIC_SUB_RETURN(1, v)
151 #define __ATOMIC_INC(v) __ATOMIC_ADD(1, v)
155 #define __ATOMIC_DEC(v) __ATOMIC_SUB(1, v)
158 #define DEF_ATOMIC_OPS(a_type, i_type) \
159 static inline i_type a_type##_read(const a_type##_t *v) \
161 return __ATOMIC_READ(&v->counter); \
164 static inline i_type a_type##_read_acquire(const a_type##_t *v) \
166 i_type ret = __ATOMIC_READ(&v->counter); \
167 smp_mb__after_atomic(); \
171 static inline void a_type##_set(a_type##_t *v, i_type i) \
173 return __ATOMIC_SET(&v->counter, i); \
176 static inline i_type a_type##_add_return(i_type i, a_type##_t *v) \
178 return __ATOMIC_ADD_RETURN(i, &v->counter); \
181 static inline i_type a_type##_add_return_release(i_type i, a_type##_t *v)\
183 return __ATOMIC_ADD_RETURN_RELEASE(i, &v->counter); \
186 static inline i_type a_type##_sub_return_release(i_type i, a_type##_t *v)\
188 return __ATOMIC_SUB_RETURN_RELEASE(i, &v->counter); \
191 static inline i_type a_type##_sub_return(i_type i, a_type##_t *v) \
193 return __ATOMIC_SUB_RETURN(i, &v->counter); \
196 static inline void a_type##_add(i_type i, a_type##_t *v) \
198 __ATOMIC_ADD(i, &v->counter); \
201 static inline void a_type##_sub(i_type i, a_type##_t *v) \
203 __ATOMIC_SUB(i, &v->counter); \
206 static inline i_type a_type##_inc_return(a_type##_t *v) \
208 return __ATOMIC_INC_RETURN(&v->counter); \
211 static inline i_type a_type##_dec_return(a_type##_t *v) \
213 return __ATOMIC_DEC_RETURN(&v->counter); \
216 static inline i_type a_type##_dec_return_release(a_type##_t *v) \
218 return __ATOMIC_SUB_RETURN_RELEASE(1, &v->counter); \
221 static inline void a_type##_inc(a_type##_t *v) \
223 __ATOMIC_INC(&v->counter); \
226 static inline void a_type##_dec(a_type##_t *v) \
228 __ATOMIC_DEC(&v->counter); \
231 static inline bool a_type##_add_negative(i_type i, a_type##_t *v) \
233 return __ATOMIC_ADD_RETURN(i, &v->counter) < 0; \
236 static inline bool a_type##_sub_and_test(i_type i, a_type##_t *v) \
238 return __ATOMIC_SUB_RETURN(i, &v->counter) == 0; \
241 static inline bool a_type##_inc_and_test(a_type##_t *v) \
243 return __ATOMIC_INC_RETURN(&v->counter) == 0; \
246 static inline bool a_type##_dec_and_test(a_type##_t *v) \
248 return __ATOMIC_DEC_RETURN(&v->counter) == 0; \
251 static inline i_type a_type##_add_unless(a_type##_t *v, i_type a, i_type u)\
253 i_type old, c = __ATOMIC_READ(&v->counter); \
254 while (c != u && (old = cmpxchg(&v->counter, c, c + a)) != c) \
259 static inline bool a_type##_inc_not_zero(a_type##_t *v) \
261 return a_type##_add_unless(v, 1, 0); \
264 static inline void a_type##_and(i_type a, a_type##_t *v) \
266 __ATOMIC_AND(a, v); \
269 static inline void a_type##_or(i_type a, a_type##_t *v) \
274 static inline i_type a_type##_xchg(a_type##_t *v, i_type i) \
276 return xchg(&v->counter, i); \
279 static inline i_type a_type##_cmpxchg(a_type##_t *v, i_type old, i_type new)\
281 return cmpxchg(&v->counter, old, new); \
284 static inline i_type a_type##_cmpxchg_acquire(a_type##_t *v, i_type old, i_type new)\
286 return cmpxchg_acquire(&v->counter, old, new); \
289 static inline bool a_type##_try_cmpxchg_acquire(a_type##_t *v, i_type *old, i_type new)\
291 i_type prev = *old; \
292 *old = cmpxchg_acquire(&v->counter, *old, new); \
293 return prev == *old; \
296 DEF_ATOMIC_OPS(atomic, int)
297 DEF_ATOMIC_OPS(atomic_long, long)
299 #ifndef ATOMIC64_SPINLOCK
300 DEF_ATOMIC_OPS(atomic64, s64)
302 s64 atomic64_read(const atomic64_t *v);
303 static inline s64 atomic64_read_acquire(const atomic64_t *v)
305 s64 ret = atomic64_read(v);
306 smp_mb__after_atomic();
310 void atomic64_set(atomic64_t *v, s64);
312 s64 atomic64_add_return(s64, atomic64_t *);
313 s64 atomic64_sub_return(s64, atomic64_t *);
314 void atomic64_add(s64, atomic64_t *);
315 void atomic64_sub(s64, atomic64_t *);
317 s64 atomic64_xchg(atomic64_t *, s64);
318 s64 atomic64_cmpxchg(atomic64_t *, s64, s64);
320 #define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0)
321 #define atomic64_inc(v) atomic64_add(1LL, (v))
322 #define atomic64_inc_return(v) atomic64_add_return(1LL, (v))
323 #define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
324 #define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0)
325 #define atomic64_dec(v) atomic64_sub(1LL, (v))
326 #define atomic64_dec_return(v) atomic64_sub_return(1LL, (v))
327 #define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0)
328 #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1LL, 0LL)
330 static inline s64 atomic64_add_return_release(s64 i, atomic64_t *v)
332 smp_mb__before_atomic();
333 return atomic64_add_return(i, v);
336 static inline s64 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
338 return atomic64_cmpxchg(v, old, new);
341 static inline s64 atomic64_sub_return_release(s64 i, atomic64_t *v)
343 smp_mb__before_atomic();
344 return atomic64_sub_return(i, v);
349 #endif /* __TOOLS_LINUX_ATOMIC_H */