git.sesse.net Git - bcachefs-tools-debian/blob - linux/atomic64.c

   1 /*
   2  * Generic implementation of 64-bit atomics using spinlocks,
   3  * useful on processors that don't have 64-bit atomic instructions.
   4  *
   5  * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License
   9  * as published by the Free Software Foundation; either version
  10  * 2 of the License, or (at your option) any later version.
  11  */
  12 #include <linux/types.h>
  13 #include <linux/cache.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/atomic.h>
  16
  17 #ifdef ATOMIC64_SPINLOCK
  18
  19 /*
  20  * We use a hashed array of spinlocks to provide exclusive access
  21  * to each atomic64_t variable.  Since this is expected to used on
  22  * systems with small numbers of CPUs (<= 4 or so), we use a
  23  * relatively small array of 16 spinlocks to avoid wasting too much
  24  * memory on the spinlock array.
  25  */
  26 #define NR_LOCKS        16
  27
  28 /*
  29  * Ensure each lock is in a separate cacheline.
  30  */
  31 static union {
  32         raw_spinlock_t lock;
  33         char pad[L1_CACHE_BYTES];
  34 } atomic64_lock[NR_LOCKS] ____cacheline_aligned_in_smp = {
  35         [0 ... (NR_LOCKS - 1)] = {
  36                 .lock =  __RAW_SPIN_LOCK_UNLOCKED(atomic64_lock.lock),
  37         },
  38 };
  39
  40 static inline raw_spinlock_t *lock_addr(const atomic64_t *v)
  41 {
  42         unsigned long addr = (unsigned long) v;
  43
  44         addr >>= L1_CACHE_SHIFT;
  45         addr ^= (addr >> 8) ^ (addr >> 16);
  46         return &atomic64_lock[addr & (NR_LOCKS - 1)].lock;
  47 }
  48
  49 long long atomic64_read(const atomic64_t *v)
  50 {
  51         unsigned long flags;
  52         raw_spinlock_t *lock = lock_addr(v);
  53         long long val;
  54
  55         raw_spin_lock_irqsave(lock, flags);
  56         val = v->counter;
  57         raw_spin_unlock_irqrestore(lock, flags);
  58         return val;
  59 }
  60
  61 void atomic64_set(atomic64_t *v, long long i)
  62 {
  63         unsigned long flags;
  64         raw_spinlock_t *lock = lock_addr(v);
  65
  66         raw_spin_lock_irqsave(lock, flags);
  67         v->counter = i;
  68         raw_spin_unlock_irqrestore(lock, flags);
  69 }
  70
  71 #define ATOMIC64_OP(op, c_op)                                           \
  72 void atomic64_##op(long long a, atomic64_t *v)                          \
  73 {                                                                       \
  74         unsigned long flags;                                            \
  75         raw_spinlock_t *lock = lock_addr(v);                            \
  76                                                                         \
  77         raw_spin_lock_irqsave(lock, flags);                             \
  78         v->counter c_op a;                                              \
  79         raw_spin_unlock_irqrestore(lock, flags);                        \
  80 }
  81
  82 #define ATOMIC64_OP_RETURN(op, c_op)                                    \
  83 long long atomic64_##op##_return(long long a, atomic64_t *v)            \
  84 {                                                                       \
  85         unsigned long flags;                                            \
  86         raw_spinlock_t *lock = lock_addr(v);                            \
  87         long long val;                                                  \
  88                                                                         \
  89         raw_spin_lock_irqsave(lock, flags);                             \
  90         val = (v->counter c_op a);                                      \
  91         raw_spin_unlock_irqrestore(lock, flags);                        \
  92         return val;                                                     \
  93 }
  94
  95 #define ATOMIC64_FETCH_OP(op, c_op)                                     \
  96 long long atomic64_fetch_##op(long long a, atomic64_t *v)               \
  97 {                                                                       \
  98         unsigned long flags;                                            \
  99         raw_spinlock_t *lock = lock_addr(v);                            \
 100         long long val;                                                  \
 101                                                                         \
 102         raw_spin_lock_irqsave(lock, flags);                             \
 103         val = v->counter;                                               \
 104         v->counter c_op a;                                              \
 105         raw_spin_unlock_irqrestore(lock, flags);                        \
 106         return val;                                                     \
 107 }
 108
 109 #define ATOMIC64_OPS(op, c_op)                                          \
 110         ATOMIC64_OP(op, c_op)                                           \
 111         ATOMIC64_OP_RETURN(op, c_op)                                    \
 112         ATOMIC64_FETCH_OP(op, c_op)
 113
 114 ATOMIC64_OPS(add, +=)
 115 ATOMIC64_OPS(sub, -=)
 116
 117 #undef ATOMIC64_OPS
 118 #define ATOMIC64_OPS(op, c_op)                                          \
 119         ATOMIC64_OP(op, c_op)                                           \
 120         ATOMIC64_OP_RETURN(op, c_op)                                    \
 121         ATOMIC64_FETCH_OP(op, c_op)
 122
 123 ATOMIC64_OPS(and, &=)
 124 ATOMIC64_OPS(or, |=)
 125 ATOMIC64_OPS(xor, ^=)
 126
 127 #undef ATOMIC64_OPS
 128 #undef ATOMIC64_FETCH_OP
 129 #undef ATOMIC64_OP_RETURN
 130 #undef ATOMIC64_OP
 131
 132 long long atomic64_dec_if_positive(atomic64_t *v)
 133 {
 134         unsigned long flags;
 135         raw_spinlock_t *lock = lock_addr(v);
 136         long long val;
 137
 138         raw_spin_lock_irqsave(lock, flags);
 139         val = v->counter - 1;
 140         if (val >= 0)
 141                 v->counter = val;
 142         raw_spin_unlock_irqrestore(lock, flags);
 143         return val;
 144 }
 145
 146 long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n)
 147 {
 148         unsigned long flags;
 149         raw_spinlock_t *lock = lock_addr(v);
 150         long long val;
 151
 152         raw_spin_lock_irqsave(lock, flags);
 153         val = v->counter;
 154         if (val == o)
 155                 v->counter = n;
 156         raw_spin_unlock_irqrestore(lock, flags);
 157         return val;
 158 }
 159
 160 long long atomic64_xchg(atomic64_t *v, long long new)
 161 {
 162         unsigned long flags;
 163         raw_spinlock_t *lock = lock_addr(v);
 164         long long val;
 165
 166         raw_spin_lock_irqsave(lock, flags);
 167         val = v->counter;
 168         v->counter = new;
 169         raw_spin_unlock_irqrestore(lock, flags);
 170         return val;
 171 }
 172
 173 int atomic64_add_unless(atomic64_t *v, long long a, long long u)
 174 {
 175         unsigned long flags;
 176         raw_spinlock_t *lock = lock_addr(v);
 177         int ret = 0;
 178
 179         raw_spin_lock_irqsave(lock, flags);
 180         if (v->counter != u) {
 181                 v->counter += a;
 182                 ret = 1;
 183         }
 184         raw_spin_unlock_irqrestore(lock, flags);
 185         return ret;
 186 }
 187
 188 #endif