git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/six.h

   1 #ifndef _BCACHEFS_SIX_H
   2 #define _BCACHEFS_SIX_H
   3
   4 #include <linux/lockdep.h>
   5 #include <linux/osq_lock.h>
   6 #include <linux/sched.h>
   7 #include <linux/types.h>
   8
   9 #include "util.h"
  10
  11 #define SIX_LOCK_SEPARATE_LOCKFNS
  12
  13 /*
  14  * LOCK STATES:
  15  *
  16  * read, intent, write (i.e. shared/intent/exclusive, hence the name)
  17  *
  18  * read and write work as with normal read/write locks - a lock can have
  19  * multiple readers, but write excludes reads and other write locks.
  20  *
  21  * Intent does not block read, but it does block other intent locks. The idea is
  22  * by taking an intent lock, you can then later upgrade to a write lock without
  23  * dropping your read lock and without deadlocking - because no other thread has
  24  * the intent lock and thus no other thread could be trying to take the write
  25  * lock.
  26  */
  27
  28 union six_lock_state {
  29         struct {
  30                 atomic64_t      counter;
  31         };
  32
  33         struct {
  34                 u64             v;
  35         };
  36
  37         struct {
  38                 /* for waitlist_bitnr() */
  39                 unsigned long   l;
  40         };
  41
  42         struct {
  43                 unsigned        read_lock:26;
  44                 unsigned        intent_lock:3;
  45                 unsigned        waiters:3;
  46                 /*
  47                  * seq works much like in seqlocks: it's incremented every time
  48                  * we lock and unlock for write.
  49                  *
  50                  * If it's odd write lock is held, even unlocked.
  51                  *
  52                  * Thus readers can unlock, and then lock again later iff it
  53                  * hasn't been modified in the meantime.
  54                  */
  55                 u32             seq;
  56         };
  57 };
  58
  59 #define SIX_LOCK_MAX_RECURSE    ((1 << 3) - 1)
  60
  61 enum six_lock_type {
  62         SIX_LOCK_read,
  63         SIX_LOCK_intent,
  64         SIX_LOCK_write,
  65 };
  66
  67 struct six_lock {
  68         union six_lock_state    state;
  69         struct task_struct      *owner;
  70         struct optimistic_spin_queue osq;
  71
  72         raw_spinlock_t          wait_lock;
  73         struct list_head        wait_list[2];
  74 #ifdef CONFIG_DEBUG_LOCK_ALLOC
  75         struct lockdep_map      dep_map;
  76 #endif
  77 };
  78
  79 static __always_inline void __six_lock_init(struct six_lock *lock,
  80                                             const char *name,
  81                                             struct lock_class_key *key)
  82 {
  83         atomic64_set(&lock->state.counter, 0);
  84         raw_spin_lock_init(&lock->wait_lock);
  85         INIT_LIST_HEAD(&lock->wait_list[SIX_LOCK_read]);
  86         INIT_LIST_HEAD(&lock->wait_list[SIX_LOCK_intent]);
  87 #ifdef CONFIG_DEBUG_LOCK_ALLOC
  88         debug_check_no_locks_freed((void *) lock, sizeof(*lock));
  89         lockdep_init_map(&lock->dep_map, name, key, 0);
  90 #endif
  91 }
  92
  93 #define six_lock_init(lock)                                             \
  94 do {                                                                    \
  95         static struct lock_class_key __key;                             \
  96                                                                         \
  97         __six_lock_init((lock), #lock, &__key);                         \
  98 } while (0)
  99
 100 #define __SIX_VAL(field, _v)    (((union six_lock_state) { .field = _v }).v)
 101
 102 #ifdef SIX_LOCK_SEPARATE_LOCKFNS
 103
 104 #define __SIX_LOCK(type)                                                \
 105 bool six_trylock_##type(struct six_lock *);                             \
 106 bool six_relock_##type(struct six_lock *, u32);                         \
 107 void six_lock_##type(struct six_lock *);                                \
 108 void six_unlock_##type(struct six_lock *);
 109
 110 __SIX_LOCK(read)
 111 __SIX_LOCK(intent)
 112 __SIX_LOCK(write)
 113 #undef __SIX_LOCK
 114
 115 #define SIX_LOCK_DISPATCH(type, fn, ...)                        \
 116         switch (type) {                                         \
 117         case SIX_LOCK_read:                                     \
 118                 return fn##_read(__VA_ARGS__);                  \
 119         case SIX_LOCK_intent:                                   \
 120                 return fn##_intent(__VA_ARGS__);                \
 121         case SIX_LOCK_write:                                    \
 122                 return fn##_write(__VA_ARGS__);                 \
 123         default:                                                \
 124                 BUG();                                          \
 125         }
 126
 127 static inline bool six_trylock_type(struct six_lock *lock, enum six_lock_type type)
 128 {
 129         SIX_LOCK_DISPATCH(type, six_trylock, lock);
 130 }
 131
 132 static inline bool six_relock_type(struct six_lock *lock, enum six_lock_type type,
 133                      unsigned seq)
 134 {
 135         SIX_LOCK_DISPATCH(type, six_relock, lock, seq);
 136 }
 137
 138 static inline void six_lock_type(struct six_lock *lock, enum six_lock_type type)
 139 {
 140         SIX_LOCK_DISPATCH(type, six_lock, lock);
 141 }
 142
 143 static inline void six_unlock_type(struct six_lock *lock, enum six_lock_type type)
 144 {
 145         SIX_LOCK_DISPATCH(type, six_unlock, lock);
 146 }
 147
 148 #else
 149
 150 bool six_trylock_type(struct six_lock *, enum six_lock_type);
 151 bool six_relock_type(struct six_lock *, enum six_lock_type, unsigned);
 152 void six_lock_type(struct six_lock *, enum six_lock_type);
 153 void six_unlock_type(struct six_lock *, enum six_lock_type);
 154
 155 #define __SIX_LOCK(type)                                                \
 156 static __always_inline bool six_trylock_##type(struct six_lock *lock)   \
 157 {                                                                       \
 158         return six_trylock_type(lock, SIX_LOCK_##type);                 \
 159 }                                                                       \
 160                                                                         \
 161 static __always_inline bool six_relock_##type(struct six_lock *lock, u32 seq)\
 162 {                                                                       \
 163         return six_relock_type(lock, SIX_LOCK_##type, seq);             \
 164 }                                                                       \
 165                                                                         \
 166 static __always_inline void six_lock_##type(struct six_lock *lock)      \
 167 {                                                                       \
 168         six_lock_type(lock, SIX_LOCK_##type);                           \
 169 }                                                                       \
 170                                                                         \
 171 static __always_inline void six_unlock_##type(struct six_lock *lock)    \
 172 {                                                                       \
 173         six_unlock_type(lock, SIX_LOCK_##type);                         \
 174 }
 175
 176 __SIX_LOCK(read)
 177 __SIX_LOCK(intent)
 178 __SIX_LOCK(write)
 179 #undef __SIX_LOCK
 180
 181 #endif
 182
 183 void six_lock_downgrade(struct six_lock *);
 184 bool six_lock_tryupgrade(struct six_lock *);
 185 bool six_trylock_convert(struct six_lock *, enum six_lock_type,
 186                          enum six_lock_type);
 187
 188 void six_lock_increment(struct six_lock *, enum six_lock_type);
 189
 190 #endif /* _BCACHEFS_SIX_H */