git.sesse.net Git - bcachefs-tools-debian/blob - linux/wait.c

   1 /*
   2  * Generic waiting primitives.
   3  *
   4  * (C) 2004 Nadia Yvette Chambers, Oracle
   5  */
   6
   7 #include <linux/completion.h>
   8 #include <linux/sched.h>
   9 #include <linux/wait.h>
  10
  11 static inline int waitqueue_active(wait_queue_head_t *q)
  12 {
  13         return !list_empty(&q->task_list);
  14 }
  15
  16 static inline void __add_wait_queue(wait_queue_head_t *head, wait_queue_t *new)
  17 {
  18         list_add(&new->task_list, &head->task_list);
  19 }
  20
  21 static inline void __add_wait_queue_tail(wait_queue_head_t *head,
  22                                          wait_queue_t *new)
  23 {
  24         list_add_tail(&new->task_list, &head->task_list);
  25 }
  26
  27 static inline void
  28 __add_wait_queue_tail_exclusive(wait_queue_head_t *q, wait_queue_t *wait)
  29 {
  30         wait->flags |= WQ_FLAG_EXCLUSIVE;
  31         __add_wait_queue_tail(q, wait);
  32 }
  33
  34 static inline void
  35 __remove_wait_queue(wait_queue_head_t *head, wait_queue_t *old)
  36 {
  37         list_del(&old->task_list);
  38 }
  39
  40 static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
  41                              int nr_exclusive, int wake_flags, void *key)
  42 {
  43         wait_queue_t *curr, *next;
  44
  45         list_for_each_entry_safe(curr, next, &q->task_list, task_list) {
  46                 unsigned flags = curr->flags;
  47
  48                 if (curr->func(curr, mode, wake_flags, key) &&
  49                                 (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
  50                         break;
  51         }
  52 }
  53
  54 static void __wake_up(wait_queue_head_t *q, unsigned int mode,
  55                       int nr_exclusive, void *key)
  56 {
  57         unsigned long flags;
  58
  59         spin_lock_irqsave(&q->lock, flags);
  60         __wake_up_common(q, mode, nr_exclusive, 0, key);
  61         spin_unlock_irqrestore(&q->lock, flags);
  62 }
  63
  64 void wake_up(wait_queue_head_t *q)
  65 {
  66         __wake_up(q, TASK_NORMAL, 1, NULL);
  67 }
  68
  69 static void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr)
  70 {
  71         __wake_up_common(q, mode, nr, 0, NULL);
  72 }
  73
  74 void
  75 prepare_to_wait(wait_queue_head_t *q, wait_queue_t *wait, int state)
  76 {
  77         unsigned long flags;
  78
  79         wait->flags &= ~WQ_FLAG_EXCLUSIVE;
  80         spin_lock_irqsave(&q->lock, flags);
  81         if (list_empty(&wait->task_list))
  82                 __add_wait_queue(q, wait);
  83         set_current_state(state);
  84         spin_unlock_irqrestore(&q->lock, flags);
  85 }
  86
  87 static void
  88 prepare_to_wait_exclusive(wait_queue_head_t *q, wait_queue_t *wait, int state)
  89 {
  90         unsigned long flags;
  91
  92         wait->flags |= WQ_FLAG_EXCLUSIVE;
  93         spin_lock_irqsave(&q->lock, flags);
  94         if (list_empty(&wait->task_list))
  95                 __add_wait_queue_tail(q, wait);
  96         set_current_state(state);
  97         spin_unlock_irqrestore(&q->lock, flags);
  98 }
  99
 100 void finish_wait(wait_queue_head_t *q, wait_queue_t *wait)
 101 {
 102         unsigned long flags;
 103
 104         __set_current_state(TASK_RUNNING);
 105         /*
 106          * We can check for list emptiness outside the lock
 107          * IFF:
 108          *  - we use the "careful" check that verifies both
 109          *    the next and prev pointers, so that there cannot
 110          *    be any half-pending updates in progress on other
 111          *    CPU's that we haven't seen yet (and that might
 112          *    still change the stack area.
 113          * and
 114          *  - all other users take the lock (ie we can only
 115          *    have _one_ other CPU that looks at or modifies
 116          *    the list).
 117          */
 118         if (!list_empty_careful(&wait->task_list)) {
 119                 spin_lock_irqsave(&q->lock, flags);
 120                 list_del_init(&wait->task_list);
 121                 spin_unlock_irqrestore(&q->lock, flags);
 122         }
 123 }
 124
 125 int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
 126                           void *key)
 127 {
 128         return wake_up_process(curr->private);
 129 }
 130
 131 int autoremove_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key)
 132 {
 133         int ret = default_wake_function(wait, mode, sync, key);
 134
 135         if (ret)
 136                 list_del_init(&wait->task_list);
 137         return ret;
 138 }
 139
 140 struct wait_bit_key {
 141         void                    *flags;
 142         int                     bit_nr;
 143         unsigned long           timeout;
 144 };
 145
 146 struct wait_bit_queue {
 147         struct wait_bit_key     key;
 148         wait_queue_t            wait;
 149 };
 150
 151 static int wake_bit_function(wait_queue_t *wait, unsigned mode, int sync, void *arg)
 152 {
 153         struct wait_bit_key *key = arg;
 154         struct wait_bit_queue *wait_bit =
 155                 container_of(wait, struct wait_bit_queue, wait);
 156
 157         return (wait_bit->key.flags == key->flags &&
 158                 wait_bit->key.bit_nr == key->bit_nr &&
 159                 !test_bit(key->bit_nr, key->flags))
 160                 ? autoremove_wake_function(wait, mode, sync, key) : 0;
 161 }
 162
 163 static DECLARE_WAIT_QUEUE_HEAD(bit_wq);
 164
 165 #define __WAIT_BIT_KEY_INITIALIZER(word, bit)                           \
 166         { .flags = word, .bit_nr = bit, }
 167
 168 #define DEFINE_WAIT_BIT(name, word, bit)                                \
 169         struct wait_bit_queue name = {                                  \
 170                 .key = __WAIT_BIT_KEY_INITIALIZER(word, bit),           \
 171                 .wait   = {                                             \
 172                         .private        = current,                      \
 173                         .func           = wake_bit_function,            \
 174                         .task_list      =                               \
 175                                 LIST_HEAD_INIT((name).wait.task_list),  \
 176                 },                                                      \
 177         }
 178
 179 void wake_up_bit(void *word, int bit)
 180 {
 181         struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit);
 182
 183         if (waitqueue_active(&bit_wq))
 184                 __wake_up(&bit_wq, TASK_NORMAL, 1, &key);
 185 }
 186
 187 void __wait_on_bit(void *word, int bit, unsigned mode)
 188 {
 189         DEFINE_WAIT_BIT(wait, word, bit);
 190
 191         do {
 192                 prepare_to_wait(&bit_wq, &wait.wait, mode);
 193                 if (test_bit(wait.key.bit_nr, wait.key.flags))
 194                         schedule();
 195         } while (test_bit(wait.key.bit_nr, wait.key.flags));
 196
 197         finish_wait(&bit_wq, &wait.wait);
 198 }
 199
 200 void __wait_on_bit_lock(void *word, int bit, unsigned mode)
 201 {
 202         DEFINE_WAIT_BIT(wait, word, bit);
 203
 204         do {
 205                 prepare_to_wait_exclusive(&bit_wq, &wait.wait, mode);
 206                 if (!test_bit(wait.key.bit_nr, wait.key.flags))
 207                         continue;
 208                 schedule();
 209         } while (test_and_set_bit(wait.key.bit_nr, wait.key.flags));
 210         finish_wait(&bit_wq, &wait.wait);
 211 }
 212
 213 void complete(struct completion *x)
 214 {
 215         unsigned long flags;
 216
 217         spin_lock_irqsave(&x->wait.lock, flags);
 218         x->done++;
 219         __wake_up_locked(&x->wait, TASK_NORMAL, 1);
 220         spin_unlock_irqrestore(&x->wait.lock, flags);
 221 }
 222
 223 void wait_for_completion(struct completion *x)
 224 {
 225         spin_lock_irq(&x->wait.lock);
 226
 227         if (!x->done) {
 228                 DECLARE_WAITQUEUE(wait, current);
 229
 230                 __add_wait_queue_tail_exclusive(&x->wait, &wait);
 231                 do {
 232                         __set_current_state(TASK_UNINTERRUPTIBLE);
 233                         spin_unlock_irq(&x->wait.lock);
 234
 235                         schedule();
 236                         spin_lock_irq(&x->wait.lock);
 237                 } while (!x->done);
 238                 __remove_wait_queue(&x->wait, &wait);
 239                 if (!x->done)
 240                         goto out;
 241         }
 242         x->done--;
 243 out:
 244         spin_unlock_irq(&x->wait.lock);
 245 }