git.sesse.net Git - bcachefs-tools-debian/blob - linux/six.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include <linux/export.h>
   4 #include <linux/log2.h>
   5 #include <linux/percpu.h>
   6 #include <linux/preempt.h>
   7 #include <linux/rcupdate.h>
   8 #include <linux/sched.h>
   9 #include <linux/sched/clock.h>
  10 #include <linux/sched/rt.h>
  11 #include <linux/six.h>
  12 #include <linux/slab.h>
  13
  14 #include <trace/events/lock.h>
  15
  16 #ifdef DEBUG
  17 #define EBUG_ON(cond)           BUG_ON(cond)
  18 #else
  19 #define EBUG_ON(cond)           do {} while (0)
  20 #endif
  21
  22 #define six_acquire(l, t, r, ip)        lock_acquire(l, 0, t, r, 1, NULL, ip)
  23 #define six_release(l, ip)              lock_release(l, ip)
  24
  25 static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type);
  26
  27 struct six_lock_vals {
  28         /* Value we add to the lock in order to take the lock: */
  29         u64                     lock_val;
  30
  31         /* If the lock has this value (used as a mask), taking the lock fails: */
  32         u64                     lock_fail;
  33
  34         /* Value we add to the lock in order to release the lock: */
  35         u64                     unlock_val;
  36
  37         /* Mask that indicates lock is held for this type: */
  38         u64                     held_mask;
  39
  40         /* Waitlist we wakeup when releasing the lock: */
  41         enum six_lock_type      unlock_wakeup;
  42 };
  43
  44 #define __SIX_LOCK_HELD_read    __SIX_VAL(read_lock, ~0)
  45 #define __SIX_LOCK_HELD_intent  __SIX_VAL(intent_lock, ~0)
  46 #define __SIX_LOCK_HELD_write   __SIX_VAL(seq, 1)
  47
  48 #define LOCK_VALS {                                                     \
  49         [SIX_LOCK_read] = {                                             \
  50                 .lock_val       = __SIX_VAL(read_lock, 1),              \
  51                 .lock_fail      = __SIX_LOCK_HELD_write + __SIX_VAL(write_locking, 1),\
  52                 .unlock_val     = -__SIX_VAL(read_lock, 1),             \
  53                 .held_mask      = __SIX_LOCK_HELD_read,                 \
  54                 .unlock_wakeup  = SIX_LOCK_write,                       \
  55         },                                                              \
  56         [SIX_LOCK_intent] = {                                           \
  57                 .lock_val       = __SIX_VAL(intent_lock, 1),            \
  58                 .lock_fail      = __SIX_LOCK_HELD_intent,               \
  59                 .unlock_val     = -__SIX_VAL(intent_lock, 1),           \
  60                 .held_mask      = __SIX_LOCK_HELD_intent,               \
  61                 .unlock_wakeup  = SIX_LOCK_intent,                      \
  62         },                                                              \
  63         [SIX_LOCK_write] = {                                            \
  64                 .lock_val       = __SIX_VAL(seq, 1),                    \
  65                 .lock_fail      = __SIX_LOCK_HELD_read,                 \
  66                 .unlock_val     = __SIX_VAL(seq, 1),                    \
  67                 .held_mask      = __SIX_LOCK_HELD_write,                \
  68                 .unlock_wakeup  = SIX_LOCK_read,                        \
  69         },                                                              \
  70 }
  71
  72 static inline void six_set_owner(struct six_lock *lock, enum six_lock_type type,
  73                                  union six_lock_state old,
  74                                  struct task_struct *owner)
  75 {
  76         if (type != SIX_LOCK_intent)
  77                 return;
  78
  79         if (!old.intent_lock) {
  80                 EBUG_ON(lock->owner);
  81                 lock->owner = owner;
  82         } else {
  83                 EBUG_ON(lock->owner != current);
  84         }
  85 }
  86
  87 static inline unsigned pcpu_read_count(struct six_lock *lock)
  88 {
  89         unsigned read_count = 0;
  90         int cpu;
  91
  92         for_each_possible_cpu(cpu)
  93                 read_count += *per_cpu_ptr(lock->readers, cpu);
  94         return read_count;
  95 }
  96
  97 /* This is probably up there with the more evil things I've done */
  98 #define waitlist_bitnr(id) ilog2((((union six_lock_state) { .waiters = 1 << (id) }).l))
  99
 100 static int __do_six_trylock_type(struct six_lock *lock,
 101                                  enum six_lock_type type,
 102                                  struct task_struct *task,
 103                                  bool try)
 104 {
 105         const struct six_lock_vals l[] = LOCK_VALS;
 106         union six_lock_state old, new;
 107         int ret;
 108         u64 v;
 109
 110         EBUG_ON(type == SIX_LOCK_write && lock->owner != task);
 111         EBUG_ON(type == SIX_LOCK_write && (lock->state.seq & 1));
 112         EBUG_ON(type == SIX_LOCK_write && (try != !(lock->state.write_locking)));
 113
 114         /*
 115          * Percpu reader mode:
 116          *
 117          * The basic idea behind this algorithm is that you can implement a lock
 118          * between two threads without any atomics, just memory barriers:
 119          *
 120          * For two threads you'll need two variables, one variable for "thread a
 121          * has the lock" and another for "thread b has the lock".
 122          *
 123          * To take the lock, a thread sets its variable indicating that it holds
 124          * the lock, then issues a full memory barrier, then reads from the
 125          * other thread's variable to check if the other thread thinks it has
 126          * the lock. If we raced, we backoff and retry/sleep.
 127          */
 128
 129         if (type == SIX_LOCK_read && lock->readers) {
 130                 preempt_disable();
 131                 this_cpu_inc(*lock->readers); /* signal that we own lock */
 132
 133                 smp_mb();
 134
 135                 old.v = READ_ONCE(lock->state.v);
 136                 ret = !(old.v & l[type].lock_fail);
 137
 138                 this_cpu_sub(*lock->readers, !ret);
 139                 preempt_enable();
 140
 141                 /*
 142                  * If we failed because a writer was trying to take the
 143                  * lock, issue a wakeup because we might have caused a
 144                  * spurious trylock failure:
 145                  */
 146 #if 0
 147                 /*
 148                  * This code should be sufficient, but we're seeing unexplained
 149                  * lost wakeups:
 150                  */
 151                 if (old.write_locking)
 152                         ret = -1 - SIX_LOCK_write;
 153 #else
 154                 if (!ret)
 155                         ret = -1 - SIX_LOCK_write;
 156 #endif
 157         } else if (type == SIX_LOCK_write && lock->readers) {
 158                 if (try) {
 159                         atomic64_add(__SIX_VAL(write_locking, 1),
 160                                      &lock->state.counter);
 161                         smp_mb__after_atomic();
 162                 } else if (!(lock->state.waiters & (1 << SIX_LOCK_write))) {
 163                         atomic64_add(__SIX_VAL(waiters, 1 << SIX_LOCK_write),
 164                                      &lock->state.counter);
 165                         /*
 166                          * pairs with barrier after unlock and before checking
 167                          * for readers in unlock path
 168                          */
 169                         smp_mb__after_atomic();
 170                 }
 171
 172                 ret = !pcpu_read_count(lock);
 173
 174                 /*
 175                  * On success, we increment lock->seq; also we clear
 176                  * write_locking unless we failed from the lock path:
 177                  */
 178                 v = 0;
 179                 if (ret)
 180                         v += __SIX_VAL(seq, 1);
 181                 if (ret || try)
 182                         v -= __SIX_VAL(write_locking, 1);
 183
 184                 if (try && !ret) {
 185                         old.v = atomic64_add_return(v, &lock->state.counter);
 186                         if (old.waiters & (1 << SIX_LOCK_read))
 187                                 ret = -1 - SIX_LOCK_read;
 188                 } else {
 189                         atomic64_add(v, &lock->state.counter);
 190                 }
 191         } else {
 192                 v = READ_ONCE(lock->state.v);
 193                 do {
 194                         new.v = old.v = v;
 195
 196                         if (!(old.v & l[type].lock_fail)) {
 197                                 new.v += l[type].lock_val;
 198
 199                                 if (type == SIX_LOCK_write)
 200                                         new.write_locking = 0;
 201                         } else if (!try && !(new.waiters & (1 << type)))
 202                                 new.waiters |= 1 << type;
 203                         else
 204                                 break; /* waiting bit already set */
 205                 } while ((v = atomic64_cmpxchg_acquire(&lock->state.counter,
 206                                         old.v, new.v)) != old.v);
 207
 208                 ret = !(old.v & l[type].lock_fail);
 209
 210                 EBUG_ON(ret && !(lock->state.v & l[type].held_mask));
 211         }
 212
 213         if (ret > 0)
 214                 six_set_owner(lock, type, old, task);
 215
 216         EBUG_ON(type == SIX_LOCK_write && (try || ret > 0) && (lock->state.write_locking));
 217
 218         return ret;
 219 }
 220
 221 static inline void __six_lock_wakeup(struct six_lock *lock, enum six_lock_type lock_type)
 222 {
 223         struct six_lock_waiter *w, *next;
 224         struct task_struct *task;
 225         bool saw_one;
 226         int ret;
 227 again:
 228         ret = 0;
 229         saw_one = false;
 230         raw_spin_lock(&lock->wait_lock);
 231
 232         list_for_each_entry_safe(w, next, &lock->wait_list, list) {
 233                 if (w->lock_want != lock_type)
 234                         continue;
 235
 236                 if (saw_one && lock_type != SIX_LOCK_read)
 237                         goto unlock;
 238                 saw_one = true;
 239
 240                 ret = __do_six_trylock_type(lock, lock_type, w->task, false);
 241                 if (ret <= 0)
 242                         goto unlock;
 243
 244                 __list_del(w->list.prev, w->list.next);
 245                 task = w->task;
 246                 /*
 247                  * Do no writes to @w besides setting lock_acquired - otherwise
 248                  * we would need a memory barrier:
 249                  */
 250                 barrier();
 251                 w->lock_acquired = true;
 252                 wake_up_process(task);
 253         }
 254
 255         clear_bit(waitlist_bitnr(lock_type), (unsigned long *) &lock->state.v);
 256 unlock:
 257         raw_spin_unlock(&lock->wait_lock);
 258
 259         if (ret < 0) {
 260                 lock_type = -ret - 1;
 261                 goto again;
 262         }
 263 }
 264
 265 static inline void six_lock_wakeup(struct six_lock *lock,
 266                                    union six_lock_state state,
 267                                    enum six_lock_type lock_type)
 268 {
 269         if (lock_type == SIX_LOCK_write && state.read_lock)
 270                 return;
 271
 272         if (!(state.waiters & (1 << lock_type)))
 273                 return;
 274
 275         __six_lock_wakeup(lock, lock_type);
 276 }
 277
 278 static bool do_six_trylock_type(struct six_lock *lock,
 279                                 enum six_lock_type type,
 280                                 bool try)
 281 {
 282         int ret;
 283
 284         ret = __do_six_trylock_type(lock, type, current, try);
 285         if (ret < 0)
 286                 __six_lock_wakeup(lock, -ret - 1);
 287
 288         return ret > 0;
 289 }
 290
 291 __always_inline __flatten
 292 static bool __six_trylock_type(struct six_lock *lock, enum six_lock_type type,
 293                                unsigned long ip)
 294 {
 295         if (!do_six_trylock_type(lock, type, true))
 296                 return false;
 297
 298         if (type != SIX_LOCK_write)
 299                 six_acquire(&lock->dep_map, 1, type == SIX_LOCK_read, ip);
 300         return true;
 301 }
 302
 303 __always_inline __flatten
 304 static bool __six_relock_type(struct six_lock *lock, enum six_lock_type type,
 305                               unsigned seq, unsigned long ip)
 306 {
 307         const struct six_lock_vals l[] = LOCK_VALS;
 308         union six_lock_state old;
 309         u64 v;
 310
 311         EBUG_ON(type == SIX_LOCK_write);
 312
 313         if (type == SIX_LOCK_read &&
 314             lock->readers) {
 315                 bool ret;
 316
 317                 preempt_disable();
 318                 this_cpu_inc(*lock->readers);
 319
 320                 smp_mb();
 321
 322                 old.v = READ_ONCE(lock->state.v);
 323                 ret = !(old.v & l[type].lock_fail) && old.seq == seq;
 324
 325                 this_cpu_sub(*lock->readers, !ret);
 326                 preempt_enable();
 327
 328                 /*
 329                  * Similar to the lock path, we may have caused a spurious write
 330                  * lock fail and need to issue a wakeup:
 331                  */
 332                 if (ret)
 333                         six_acquire(&lock->dep_map, 1, type == SIX_LOCK_read, ip);
 334                 else
 335                         six_lock_wakeup(lock, old, SIX_LOCK_write);
 336
 337                 return ret;
 338         }
 339
 340         v = READ_ONCE(lock->state.v);
 341         do {
 342                 old.v = v;
 343
 344                 if (old.seq != seq || old.v & l[type].lock_fail)
 345                         return false;
 346         } while ((v = atomic64_cmpxchg_acquire(&lock->state.counter,
 347                                 old.v,
 348                                 old.v + l[type].lock_val)) != old.v);
 349
 350         six_set_owner(lock, type, old, current);
 351         if (type != SIX_LOCK_write)
 352                 six_acquire(&lock->dep_map, 1, type == SIX_LOCK_read, ip);
 353         return true;
 354 }
 355
 356 #ifdef CONFIG_LOCK_SPIN_ON_OWNER
 357
 358 static inline bool six_can_spin_on_owner(struct six_lock *lock)
 359 {
 360         struct task_struct *owner;
 361         bool ret;
 362
 363         if (need_resched())
 364                 return false;
 365
 366         rcu_read_lock();
 367         owner = READ_ONCE(lock->owner);
 368         ret = !owner || owner_on_cpu(owner);
 369         rcu_read_unlock();
 370
 371         return ret;
 372 }
 373
 374 static inline void six_set_nospin(struct six_lock *lock)
 375 {
 376         union six_lock_state old, new;
 377         u64 v = READ_ONCE(lock->state.v);
 378
 379         do {
 380                 new.v = old.v = v;
 381                 new.nospin = true;
 382         } while ((v = atomic64_cmpxchg(&lock->state.counter, old.v, new.v)) != old.v);
 383 }
 384
 385 static inline bool six_spin_on_owner(struct six_lock *lock,
 386                                      struct task_struct *owner,
 387                                      u64 end_time)
 388 {
 389         bool ret = true;
 390         unsigned loop = 0;
 391
 392         rcu_read_lock();
 393         while (lock->owner == owner) {
 394                 /*
 395                  * Ensure we emit the owner->on_cpu, dereference _after_
 396                  * checking lock->owner still matches owner. If that fails,
 397                  * owner might point to freed memory. If it still matches,
 398                  * the rcu_read_lock() ensures the memory stays valid.
 399                  */
 400                 barrier();
 401
 402                 if (!owner_on_cpu(owner) || need_resched()) {
 403                         ret = false;
 404                         break;
 405                 }
 406
 407                 if (!(++loop & 0xf) && (time_after64(sched_clock(), end_time))) {
 408                         six_set_nospin(lock);
 409                         ret = false;
 410                         break;
 411                 }
 412
 413                 cpu_relax();
 414         }
 415         rcu_read_unlock();
 416
 417         return ret;
 418 }
 419
 420 static inline bool six_optimistic_spin(struct six_lock *lock, enum six_lock_type type)
 421 {
 422         struct task_struct *task = current;
 423         u64 end_time;
 424
 425         if (type == SIX_LOCK_write)
 426                 return false;
 427
 428         preempt_disable();
 429         if (!six_can_spin_on_owner(lock))
 430                 goto fail;
 431
 432         if (!osq_lock(&lock->osq))
 433                 goto fail;
 434
 435         end_time = sched_clock() + 10 * NSEC_PER_USEC;
 436
 437         while (1) {
 438                 struct task_struct *owner;
 439
 440                 /*
 441                  * If there's an owner, wait for it to either
 442                  * release the lock or go to sleep.
 443                  */
 444                 owner = READ_ONCE(lock->owner);
 445                 if (owner && !six_spin_on_owner(lock, owner, end_time))
 446                         break;
 447
 448                 if (do_six_trylock_type(lock, type, false)) {
 449                         osq_unlock(&lock->osq);
 450                         preempt_enable();
 451                         return true;
 452                 }
 453
 454                 /*
 455                  * When there's no owner, we might have preempted between the
 456                  * owner acquiring the lock and setting the owner field. If
 457                  * we're an RT task that will live-lock because we won't let
 458                  * the owner complete.
 459                  */
 460                 if (!owner && (need_resched() || rt_task(task)))
 461                         break;
 462
 463                 /*
 464                  * The cpu_relax() call is a compiler barrier which forces
 465                  * everything in this loop to be re-loaded. We don't need
 466                  * memory barriers as we'll eventually observe the right
 467                  * values at the cost of a few extra spins.
 468                  */
 469                 cpu_relax();
 470         }
 471
 472         osq_unlock(&lock->osq);
 473 fail:
 474         preempt_enable();
 475
 476         /*
 477          * If we fell out of the spin path because of need_resched(),
 478          * reschedule now, before we try-lock again. This avoids getting
 479          * scheduled out right after we obtained the lock.
 480          */
 481         if (need_resched())
 482                 schedule();
 483
 484         return false;
 485 }
 486
 487 #else /* CONFIG_LOCK_SPIN_ON_OWNER */
 488
 489 static inline bool six_optimistic_spin(struct six_lock *lock, enum six_lock_type type)
 490 {
 491         return false;
 492 }
 493
 494 #endif
 495
 496 noinline
 497 static int __six_lock_type_slowpath(struct six_lock *lock, enum six_lock_type type,
 498                                     struct six_lock_waiter *wait,
 499                                     six_lock_should_sleep_fn should_sleep_fn, void *p,
 500                                     unsigned long ip)
 501 {
 502         union six_lock_state old;
 503         int ret = 0;
 504
 505         if (type == SIX_LOCK_write) {
 506                 EBUG_ON(lock->state.write_locking);
 507                 atomic64_add(__SIX_VAL(write_locking, 1), &lock->state.counter);
 508                 smp_mb__after_atomic();
 509         }
 510
 511         trace_contention_begin(lock, 0);
 512         lock_contended(&lock->dep_map, ip);
 513
 514         if (six_optimistic_spin(lock, type))
 515                 goto out;
 516
 517         wait->task              = current;
 518         wait->lock_want         = type;
 519         wait->lock_acquired     = false;
 520
 521         raw_spin_lock(&lock->wait_lock);
 522         if (!(lock->state.waiters & (1 << type)))
 523                 set_bit(waitlist_bitnr(type), (unsigned long *) &lock->state.v);
 524         /*
 525          * Retry taking the lock after taking waitlist lock, have raced with an
 526          * unlock:
 527          */
 528         ret = __do_six_trylock_type(lock, type, current, false);
 529         if (ret <= 0) {
 530                 wait->start_time = local_clock();
 531
 532                 if (!list_empty(&lock->wait_list)) {
 533                         struct six_lock_waiter *last =
 534                                 list_last_entry(&lock->wait_list,
 535                                         struct six_lock_waiter, list);
 536
 537                         if (time_before_eq64(wait->start_time, last->start_time))
 538                                 wait->start_time = last->start_time + 1;
 539                 }
 540
 541                 list_add_tail(&wait->list, &lock->wait_list);
 542         }
 543         raw_spin_unlock(&lock->wait_lock);
 544
 545         if (unlikely(ret > 0)) {
 546                 ret = 0;
 547                 goto out;
 548         }
 549
 550         if (unlikely(ret < 0)) {
 551                 __six_lock_wakeup(lock, -ret - 1);
 552                 ret = 0;
 553         }
 554
 555         while (1) {
 556                 set_current_state(TASK_UNINTERRUPTIBLE);
 557
 558                 if (wait->lock_acquired)
 559                         break;
 560
 561                 ret = should_sleep_fn ? should_sleep_fn(lock, p) : 0;
 562                 if (unlikely(ret)) {
 563                         raw_spin_lock(&lock->wait_lock);
 564                         if (!wait->lock_acquired)
 565                                 list_del(&wait->list);
 566                         raw_spin_unlock(&lock->wait_lock);
 567
 568                         if (wait->lock_acquired)
 569                                 do_six_unlock_type(lock, type);
 570                         break;
 571                 }
 572
 573                 schedule();
 574         }
 575
 576         __set_current_state(TASK_RUNNING);
 577 out:
 578         if (ret && type == SIX_LOCK_write && lock->state.write_locking) {
 579                 old.v = atomic64_sub_return(__SIX_VAL(write_locking, 1),
 580                                             &lock->state.counter);
 581                 six_lock_wakeup(lock, old, SIX_LOCK_read);
 582         }
 583         trace_contention_end(lock, 0);
 584
 585         return ret;
 586 }
 587
 588 __always_inline __flatten
 589 static int __six_lock_type_waiter(struct six_lock *lock, enum six_lock_type type,
 590                          struct six_lock_waiter *wait,
 591                          six_lock_should_sleep_fn should_sleep_fn, void *p,
 592                          unsigned long ip)
 593 {
 594         int ret;
 595
 596         wait->start_time = 0;
 597
 598         if (type != SIX_LOCK_write)
 599                 six_acquire(&lock->dep_map, 0, type == SIX_LOCK_read, ip);
 600
 601         ret = do_six_trylock_type(lock, type, true) ? 0
 602                 : __six_lock_type_slowpath(lock, type, wait, should_sleep_fn, p, ip);
 603
 604         if (ret && type != SIX_LOCK_write)
 605                 six_release(&lock->dep_map, ip);
 606         if (!ret)
 607                 lock_acquired(&lock->dep_map, ip);
 608
 609         return ret;
 610 }
 611
 612 __always_inline
 613 static int __six_lock_type(struct six_lock *lock, enum six_lock_type type,
 614                            six_lock_should_sleep_fn should_sleep_fn, void *p,
 615                            unsigned long ip)
 616 {
 617         struct six_lock_waiter wait;
 618
 619         return __six_lock_type_waiter(lock, type, &wait, should_sleep_fn, p, ip);
 620 }
 621
 622 __always_inline __flatten
 623 static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type)
 624 {
 625         const struct six_lock_vals l[] = LOCK_VALS;
 626         union six_lock_state state;
 627
 628         if (type == SIX_LOCK_intent)
 629                 lock->owner = NULL;
 630
 631         if (type == SIX_LOCK_read &&
 632             lock->readers) {
 633                 smp_mb(); /* unlock barrier */
 634                 this_cpu_dec(*lock->readers);
 635                 smp_mb(); /* between unlocking and checking for waiters */
 636                 state.v = READ_ONCE(lock->state.v);
 637         } else {
 638                 u64 v = l[type].unlock_val;
 639
 640                 if (type != SIX_LOCK_read)
 641                         v -= lock->state.v & __SIX_VAL(nospin, 1);
 642
 643                 EBUG_ON(!(lock->state.v & l[type].held_mask));
 644                 state.v = atomic64_add_return_release(v, &lock->state.counter);
 645         }
 646
 647         six_lock_wakeup(lock, state, l[type].unlock_wakeup);
 648 }
 649
 650 __always_inline __flatten
 651 static void __six_unlock_type(struct six_lock *lock, enum six_lock_type type,
 652                               unsigned long ip)
 653 {
 654         EBUG_ON(type == SIX_LOCK_write &&
 655                 !(lock->state.v & __SIX_LOCK_HELD_intent));
 656         EBUG_ON((type == SIX_LOCK_write ||
 657                  type == SIX_LOCK_intent) &&
 658                 lock->owner != current);
 659
 660         if (type != SIX_LOCK_write)
 661                 six_release(&lock->dep_map, ip);
 662
 663         if (type == SIX_LOCK_intent &&
 664             lock->intent_lock_recurse) {
 665                 --lock->intent_lock_recurse;
 666                 return;
 667         }
 668
 669         do_six_unlock_type(lock, type);
 670 }
 671
 672 #define __SIX_LOCK(type)                                                \
 673 bool six_trylock_ip_##type(struct six_lock *lock, unsigned long ip)     \
 674 {                                                                       \
 675         return __six_trylock_type(lock, SIX_LOCK_##type, ip);           \
 676 }                                                                       \
 677 EXPORT_SYMBOL_GPL(six_trylock_ip_##type);                               \
 678                                                                         \
 679 bool six_relock_ip_##type(struct six_lock *lock, u32 seq, unsigned long ip)\
 680 {                                                                       \
 681         return __six_relock_type(lock, SIX_LOCK_##type, seq, ip);       \
 682 }                                                                       \
 683 EXPORT_SYMBOL_GPL(six_relock_ip_##type);                                \
 684                                                                         \
 685 int six_lock_ip_##type(struct six_lock *lock,                           \
 686                     six_lock_should_sleep_fn should_sleep_fn, void *p,  \
 687                     unsigned long ip)                                   \
 688 {                                                                       \
 689         return __six_lock_type(lock, SIX_LOCK_##type, should_sleep_fn, p, ip);\
 690 }                                                                       \
 691 EXPORT_SYMBOL_GPL(six_lock_ip_##type);                                  \
 692                                                                         \
 693 int six_lock_ip_waiter_##type(struct six_lock *lock,                    \
 694                            struct six_lock_waiter *wait,                \
 695                            six_lock_should_sleep_fn should_sleep_fn, void *p,\
 696                            unsigned long ip)                            \
 697 {                                                                       \
 698         return __six_lock_type_waiter(lock, SIX_LOCK_##type, wait, should_sleep_fn, p, ip);\
 699 }                                                                       \
 700 EXPORT_SYMBOL_GPL(six_lock_ip_waiter_##type);                           \
 701                                                                         \
 702 void six_unlock_ip_##type(struct six_lock *lock, unsigned long ip)      \
 703 {                                                                       \
 704         __six_unlock_type(lock, SIX_LOCK_##type, ip);                   \
 705 }                                                                       \
 706 EXPORT_SYMBOL_GPL(six_unlock_ip_##type);
 707
 708 __SIX_LOCK(read)
 709 __SIX_LOCK(intent)
 710 __SIX_LOCK(write)
 711
 712 #undef __SIX_LOCK
 713
 714 /* Convert from intent to read: */
 715 void six_lock_downgrade(struct six_lock *lock)
 716 {
 717         six_lock_increment(lock, SIX_LOCK_read);
 718         six_unlock_intent(lock);
 719 }
 720 EXPORT_SYMBOL_GPL(six_lock_downgrade);
 721
 722 bool six_lock_tryupgrade(struct six_lock *lock)
 723 {
 724         union six_lock_state old, new;
 725         u64 v = READ_ONCE(lock->state.v);
 726
 727         do {
 728                 new.v = old.v = v;
 729
 730                 if (new.intent_lock)
 731                         return false;
 732
 733                 if (!lock->readers) {
 734                         EBUG_ON(!new.read_lock);
 735                         new.read_lock--;
 736                 }
 737
 738                 new.intent_lock = 1;
 739         } while ((v = atomic64_cmpxchg_acquire(&lock->state.counter,
 740                                 old.v, new.v)) != old.v);
 741
 742         if (lock->readers)
 743                 this_cpu_dec(*lock->readers);
 744
 745         six_set_owner(lock, SIX_LOCK_intent, old, current);
 746
 747         return true;
 748 }
 749 EXPORT_SYMBOL_GPL(six_lock_tryupgrade);
 750
 751 bool six_trylock_convert(struct six_lock *lock,
 752                          enum six_lock_type from,
 753                          enum six_lock_type to)
 754 {
 755         EBUG_ON(to == SIX_LOCK_write || from == SIX_LOCK_write);
 756
 757         if (to == from)
 758                 return true;
 759
 760         if (to == SIX_LOCK_read) {
 761                 six_lock_downgrade(lock);
 762                 return true;
 763         } else {
 764                 return six_lock_tryupgrade(lock);
 765         }
 766 }
 767 EXPORT_SYMBOL_GPL(six_trylock_convert);
 768
 769 /*
 770  * Increment read/intent lock count, assuming we already have it read or intent
 771  * locked:
 772  */
 773 void six_lock_increment(struct six_lock *lock, enum six_lock_type type)
 774 {
 775         const struct six_lock_vals l[] = LOCK_VALS;
 776
 777         six_acquire(&lock->dep_map, 0, type == SIX_LOCK_read, _RET_IP_);
 778
 779         /* XXX: assert already locked, and that we don't overflow: */
 780
 781         switch (type) {
 782         case SIX_LOCK_read:
 783                 if (lock->readers) {
 784                         this_cpu_inc(*lock->readers);
 785                 } else {
 786                         EBUG_ON(!lock->state.read_lock &&
 787                                 !lock->state.intent_lock);
 788                         atomic64_add(l[type].lock_val, &lock->state.counter);
 789                 }
 790                 break;
 791         case SIX_LOCK_intent:
 792                 EBUG_ON(!lock->state.intent_lock);
 793                 lock->intent_lock_recurse++;
 794                 break;
 795         case SIX_LOCK_write:
 796                 BUG();
 797                 break;
 798         }
 799 }
 800 EXPORT_SYMBOL_GPL(six_lock_increment);
 801
 802 void six_lock_wakeup_all(struct six_lock *lock)
 803 {
 804         union six_lock_state state = lock->state;
 805         struct six_lock_waiter *w;
 806
 807         six_lock_wakeup(lock, state, SIX_LOCK_read);
 808         six_lock_wakeup(lock, state, SIX_LOCK_intent);
 809         six_lock_wakeup(lock, state, SIX_LOCK_write);
 810
 811         raw_spin_lock(&lock->wait_lock);
 812         list_for_each_entry(w, &lock->wait_list, list)
 813                 wake_up_process(w->task);
 814         raw_spin_unlock(&lock->wait_lock);
 815 }
 816 EXPORT_SYMBOL_GPL(six_lock_wakeup_all);
 817
 818 void six_lock_pcpu_free(struct six_lock *lock)
 819 {
 820         BUG_ON(lock->readers && pcpu_read_count(lock));
 821         BUG_ON(lock->state.read_lock);
 822
 823         free_percpu(lock->readers);
 824         lock->readers = NULL;
 825 }
 826 EXPORT_SYMBOL_GPL(six_lock_pcpu_free);
 827
 828 void six_lock_pcpu_alloc(struct six_lock *lock)
 829 {
 830 #ifdef __KERNEL__
 831         if (!lock->readers)
 832                 lock->readers = alloc_percpu(unsigned);
 833 #endif
 834 }
 835 EXPORT_SYMBOL_GPL(six_lock_pcpu_alloc);
 836
 837 /*
 838  * Returns lock held counts, for both read and intent
 839  */
 840 struct six_lock_count six_lock_counts(struct six_lock *lock)
 841 {
 842         struct six_lock_count ret;
 843
 844         ret.n[SIX_LOCK_read]    = !lock->readers
 845                 ? lock->state.read_lock
 846                 : pcpu_read_count(lock);
 847         ret.n[SIX_LOCK_intent]  = lock->state.intent_lock + lock->intent_lock_recurse;
 848         ret.n[SIX_LOCK_write]   = lock->state.seq & 1;
 849
 850         return ret;
 851 }
 852 EXPORT_SYMBOL_GPL(six_lock_counts);