git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/util.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _BCACHEFS_UTIL_H
   3 #define _BCACHEFS_UTIL_H
   4
   5 #include <linux/bio.h>
   6 #include <linux/blkdev.h>
   7 #include <linux/closure.h>
   8 #include <linux/darray.h>
   9 #include <linux/errno.h>
  10 #include <linux/freezer.h>
  11 #include <linux/kernel.h>
  12 #include <linux/llist.h>
  13 #include <linux/log2.h>
  14 #include <linux/percpu.h>
  15 #include <linux/preempt.h>
  16 #include <linux/ratelimit.h>
  17 #include <linux/sched/clock.h>
  18 #include <linux/slab.h>
  19 #include <linux/time_stats.h>
  20 #include <linux/vmalloc.h>
  21 #include <linux/workqueue.h>
  22 #include <linux/mean_and_variance.h>
  23
  24 struct closure;
  25
  26 #ifdef CONFIG_BCACHEFS_DEBUG
  27 #define EBUG_ON(cond)           BUG_ON(cond)
  28 #else
  29 #define EBUG_ON(cond)
  30 #endif
  31
  32 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
  33 #define CPU_BIG_ENDIAN          0
  34 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
  35 #define CPU_BIG_ENDIAN          1
  36 #endif
  37
  38 /* type hackery */
  39
  40 #define type_is_exact(_val, _type)                                      \
  41         __builtin_types_compatible_p(typeof(_val), _type)
  42
  43 #define type_is(_val, _type)                                            \
  44         (__builtin_types_compatible_p(typeof(_val), _type) ||           \
  45          __builtin_types_compatible_p(typeof(_val), const _type))
  46
  47 /* Userspace doesn't align allocations as nicely as the kernel allocators: */
  48 static inline size_t buf_pages(void *p, size_t len)
  49 {
  50         return DIV_ROUND_UP(len +
  51                             ((unsigned long) p & (PAGE_SIZE - 1)),
  52                             PAGE_SIZE);
  53 }
  54
  55 #define HEAP(type)                                                      \
  56 struct {                                                                \
  57         size_t size, used;                                              \
  58         type *data;                                                     \
  59 }
  60
  61 #define DECLARE_HEAP(type, name) HEAP(type) name
  62
  63 #define init_heap(heap, _size, gfp)                                     \
  64 ({                                                                      \
  65         (heap)->used = 0;                                               \
  66         (heap)->size = (_size);                                         \
  67         (heap)->data = kvmalloc((heap)->size * sizeof((heap)->data[0]),\
  68                                  (gfp));                                \
  69 })
  70
  71 #define free_heap(heap)                                                 \
  72 do {                                                                    \
  73         kvfree((heap)->data);                                           \
  74         (heap)->data = NULL;                                            \
  75 } while (0)
  76
  77 #define heap_set_backpointer(h, i, _fn)                                 \
  78 do {                                                                    \
  79         void (*fn)(typeof(h), size_t) = _fn;                            \
  80         if (fn)                                                         \
  81                 fn(h, i);                                               \
  82 } while (0)
  83
  84 #define heap_swap(h, i, j, set_backpointer)                             \
  85 do {                                                                    \
  86         swap((h)->data[i], (h)->data[j]);                               \
  87         heap_set_backpointer(h, i, set_backpointer);                    \
  88         heap_set_backpointer(h, j, set_backpointer);                    \
  89 } while (0)
  90
  91 #define heap_peek(h)                                                    \
  92 ({                                                                      \
  93         EBUG_ON(!(h)->used);                                            \
  94         (h)->data[0];                                                   \
  95 })
  96
  97 #define heap_full(h)    ((h)->used == (h)->size)
  98
  99 #define heap_sift_down(h, i, cmp, set_backpointer)                      \
 100 do {                                                                    \
 101         size_t _c, _j = i;                                              \
 102                                                                         \
 103         for (; _j * 2 + 1 < (h)->used; _j = _c) {                       \
 104                 _c = _j * 2 + 1;                                        \
 105                 if (_c + 1 < (h)->used &&                               \
 106                     cmp(h, (h)->data[_c], (h)->data[_c + 1]) >= 0)      \
 107                         _c++;                                           \
 108                                                                         \
 109                 if (cmp(h, (h)->data[_c], (h)->data[_j]) >= 0)          \
 110                         break;                                          \
 111                 heap_swap(h, _c, _j, set_backpointer);                  \
 112         }                                                               \
 113 } while (0)
 114
 115 #define heap_sift_up(h, i, cmp, set_backpointer)                        \
 116 do {                                                                    \
 117         while (i) {                                                     \
 118                 size_t p = (i - 1) / 2;                                 \
 119                 if (cmp(h, (h)->data[i], (h)->data[p]) >= 0)            \
 120                         break;                                          \
 121                 heap_swap(h, i, p, set_backpointer);                    \
 122                 i = p;                                                  \
 123         }                                                               \
 124 } while (0)
 125
 126 #define __heap_add(h, d, cmp, set_backpointer)                          \
 127 ({                                                                      \
 128         size_t _i = (h)->used++;                                        \
 129         (h)->data[_i] = d;                                              \
 130         heap_set_backpointer(h, _i, set_backpointer);                   \
 131                                                                         \
 132         heap_sift_up(h, _i, cmp, set_backpointer);                      \
 133         _i;                                                             \
 134 })
 135
 136 #define heap_add(h, d, cmp, set_backpointer)                            \
 137 ({                                                                      \
 138         bool _r = !heap_full(h);                                        \
 139         if (_r)                                                         \
 140                 __heap_add(h, d, cmp, set_backpointer);                 \
 141         _r;                                                             \
 142 })
 143
 144 #define heap_add_or_replace(h, new, cmp, set_backpointer)               \
 145 do {                                                                    \
 146         if (!heap_add(h, new, cmp, set_backpointer) &&                  \
 147             cmp(h, new, heap_peek(h)) >= 0) {                           \
 148                 (h)->data[0] = new;                                     \
 149                 heap_set_backpointer(h, 0, set_backpointer);            \
 150                 heap_sift_down(h, 0, cmp, set_backpointer);             \
 151         }                                                               \
 152 } while (0)
 153
 154 #define heap_del(h, i, cmp, set_backpointer)                            \
 155 do {                                                                    \
 156         size_t _i = (i);                                                \
 157                                                                         \
 158         BUG_ON(_i >= (h)->used);                                        \
 159         (h)->used--;                                                    \
 160         if ((_i) < (h)->used) {                                         \
 161                 heap_swap(h, _i, (h)->used, set_backpointer);           \
 162                 heap_sift_up(h, _i, cmp, set_backpointer);              \
 163                 heap_sift_down(h, _i, cmp, set_backpointer);            \
 164         }                                                               \
 165 } while (0)
 166
 167 #define heap_pop(h, d, cmp, set_backpointer)                            \
 168 ({                                                                      \
 169         bool _r = (h)->used;                                            \
 170         if (_r) {                                                       \
 171                 (d) = (h)->data[0];                                     \
 172                 heap_del(h, 0, cmp, set_backpointer);                   \
 173         }                                                               \
 174         _r;                                                             \
 175 })
 176
 177 #define heap_resort(heap, cmp, set_backpointer)                         \
 178 do {                                                                    \
 179         ssize_t _i;                                                     \
 180         for (_i = (ssize_t) (heap)->used / 2 -  1; _i >= 0; --_i)       \
 181                 heap_sift_down(heap, _i, cmp, set_backpointer);         \
 182 } while (0)
 183
 184 #define ANYSINT_MAX(t)                                                  \
 185         ((((t) 1 << (sizeof(t) * 8 - 2)) - (t) 1) * (t) 2 + (t) 1)
 186
 187 #include "printbuf.h"
 188
 189 #define prt_vprintf(_out, ...)          bch2_prt_vprintf(_out, __VA_ARGS__)
 190 #define prt_printf(_out, ...)           bch2_prt_printf(_out, __VA_ARGS__)
 191 #define printbuf_str(_buf)              bch2_printbuf_str(_buf)
 192 #define printbuf_exit(_buf)             bch2_printbuf_exit(_buf)
 193
 194 #define printbuf_tabstops_reset(_buf)   bch2_printbuf_tabstops_reset(_buf)
 195 #define printbuf_tabstop_pop(_buf)      bch2_printbuf_tabstop_pop(_buf)
 196 #define printbuf_tabstop_push(_buf, _n) bch2_printbuf_tabstop_push(_buf, _n)
 197
 198 #define printbuf_indent_add(_out, _n)   bch2_printbuf_indent_add(_out, _n)
 199 #define printbuf_indent_sub(_out, _n)   bch2_printbuf_indent_sub(_out, _n)
 200
 201 #define prt_newline(_out)               bch2_prt_newline(_out)
 202 #define prt_tab(_out)                   bch2_prt_tab(_out)
 203 #define prt_tab_rjust(_out)             bch2_prt_tab_rjust(_out)
 204
 205 #define prt_bytes_indented(...)         bch2_prt_bytes_indented(__VA_ARGS__)
 206 #define prt_u64(_out, _v)               prt_printf(_out, "%llu", (u64) (_v))
 207 #define prt_human_readable_u64(...)     bch2_prt_human_readable_u64(__VA_ARGS__)
 208 #define prt_human_readable_s64(...)     bch2_prt_human_readable_s64(__VA_ARGS__)
 209 #define prt_units_u64(...)              bch2_prt_units_u64(__VA_ARGS__)
 210 #define prt_units_s64(...)              bch2_prt_units_s64(__VA_ARGS__)
 211 #define prt_string_option(...)          bch2_prt_string_option(__VA_ARGS__)
 212 #define prt_bitflags(...)               bch2_prt_bitflags(__VA_ARGS__)
 213 #define prt_bitflags_vector(...)        bch2_prt_bitflags_vector(__VA_ARGS__)
 214
 215 void bch2_pr_time_units(struct printbuf *, u64);
 216 void bch2_prt_datetime(struct printbuf *, time64_t);
 217
 218 #ifdef __KERNEL__
 219 static inline void uuid_unparse_lower(u8 *uuid, char *out)
 220 {
 221         sprintf(out, "%pUb", uuid);
 222 }
 223 #else
 224 #include <uuid/uuid.h>
 225 #endif
 226
 227 static inline void pr_uuid(struct printbuf *out, u8 *uuid)
 228 {
 229         char uuid_str[40];
 230
 231         uuid_unparse_lower(uuid, uuid_str);
 232         prt_printf(out, "%s", uuid_str);
 233 }
 234
 235 int bch2_strtoint_h(const char *, int *);
 236 int bch2_strtouint_h(const char *, unsigned int *);
 237 int bch2_strtoll_h(const char *, long long *);
 238 int bch2_strtoull_h(const char *, unsigned long long *);
 239 int bch2_strtou64_h(const char *, u64 *);
 240
 241 static inline int bch2_strtol_h(const char *cp, long *res)
 242 {
 243 #if BITS_PER_LONG == 32
 244         return bch2_strtoint_h(cp, (int *) res);
 245 #else
 246         return bch2_strtoll_h(cp, (long long *) res);
 247 #endif
 248 }
 249
 250 static inline int bch2_strtoul_h(const char *cp, long *res)
 251 {
 252 #if BITS_PER_LONG == 32
 253         return bch2_strtouint_h(cp, (unsigned int *) res);
 254 #else
 255         return bch2_strtoull_h(cp, (unsigned long long *) res);
 256 #endif
 257 }
 258
 259 #define strtoi_h(cp, res)                                               \
 260         ( type_is(*res, int)            ? bch2_strtoint_h(cp, (void *) res)\
 261         : type_is(*res, long)           ? bch2_strtol_h(cp, (void *) res)\
 262         : type_is(*res, long long)      ? bch2_strtoll_h(cp, (void *) res)\
 263         : type_is(*res, unsigned)       ? bch2_strtouint_h(cp, (void *) res)\
 264         : type_is(*res, unsigned long)  ? bch2_strtoul_h(cp, (void *) res)\
 265         : type_is(*res, unsigned long long) ? bch2_strtoull_h(cp, (void *) res)\
 266         : -EINVAL)
 267
 268 #define strtoul_safe(cp, var)                                           \
 269 ({                                                                      \
 270         unsigned long _v;                                               \
 271         int _r = kstrtoul(cp, 10, &_v);                                 \
 272         if (!_r)                                                        \
 273                 var = _v;                                               \
 274         _r;                                                             \
 275 })
 276
 277 #define strtoul_safe_clamp(cp, var, min, max)                           \
 278 ({                                                                      \
 279         unsigned long _v;                                               \
 280         int _r = kstrtoul(cp, 10, &_v);                                 \
 281         if (!_r)                                                        \
 282                 var = clamp_t(typeof(var), _v, min, max);               \
 283         _r;                                                             \
 284 })
 285
 286 #define strtoul_safe_restrict(cp, var, min, max)                        \
 287 ({                                                                      \
 288         unsigned long _v;                                               \
 289         int _r = kstrtoul(cp, 10, &_v);                                 \
 290         if (!_r && _v >= min && _v <= max)                              \
 291                 var = _v;                                               \
 292         else                                                            \
 293                 _r = -EINVAL;                                           \
 294         _r;                                                             \
 295 })
 296
 297 #define snprint(out, var)                                               \
 298         prt_printf(out,                                                 \
 299                    type_is(var, int)            ? "%i\n"                \
 300                  : type_is(var, unsigned)       ? "%u\n"                \
 301                  : type_is(var, long)           ? "%li\n"               \
 302                  : type_is(var, unsigned long)  ? "%lu\n"               \
 303                  : type_is(var, s64)            ? "%lli\n"              \
 304                  : type_is(var, u64)            ? "%llu\n"              \
 305                  : type_is(var, char *)         ? "%s\n"                \
 306                  : "%i\n", var)
 307
 308 bool bch2_is_zero(const void *, size_t);
 309
 310 u64 bch2_read_flag_list(char *, const char * const[]);
 311
 312 void bch2_prt_u64_base2_nbits(struct printbuf *, u64, unsigned);
 313 void bch2_prt_u64_base2(struct printbuf *, u64);
 314
 315 void bch2_print_string_as_lines(const char *prefix, const char *lines);
 316
 317 typedef DARRAY(unsigned long) bch_stacktrace;
 318 int bch2_save_backtrace(bch_stacktrace *stack, struct task_struct *, unsigned, gfp_t);
 319 void bch2_prt_backtrace(struct printbuf *, bch_stacktrace *);
 320 int bch2_prt_task_backtrace(struct printbuf *, struct task_struct *, unsigned, gfp_t);
 321
 322 static inline void prt_bdevname(struct printbuf *out, struct block_device *bdev)
 323 {
 324 #ifdef __KERNEL__
 325         prt_printf(out, "%pg", bdev);
 326 #else
 327         prt_str(out, bdev->name);
 328 #endif
 329 }
 330
 331 void bch2_time_stats_to_text(struct printbuf *, struct time_stats *);
 332
 333 #define ewma_add(ewma, val, weight)                                     \
 334 ({                                                                      \
 335         typeof(ewma) _ewma = (ewma);                                    \
 336         typeof(weight) _weight = (weight);                              \
 337                                                                         \
 338         (((_ewma << _weight) - _ewma) + (val)) >> _weight;              \
 339 })
 340
 341 struct bch_ratelimit {
 342         /* Next time we want to do some work, in nanoseconds */
 343         u64                     next;
 344
 345         /*
 346          * Rate at which we want to do work, in units per nanosecond
 347          * The units here correspond to the units passed to
 348          * bch2_ratelimit_increment()
 349          */
 350         unsigned                rate;
 351 };
 352
 353 static inline void bch2_ratelimit_reset(struct bch_ratelimit *d)
 354 {
 355         d->next = local_clock();
 356 }
 357
 358 u64 bch2_ratelimit_delay(struct bch_ratelimit *);
 359 void bch2_ratelimit_increment(struct bch_ratelimit *, u64);
 360
 361 struct bch_pd_controller {
 362         struct bch_ratelimit    rate;
 363         unsigned long           last_update;
 364
 365         s64                     last_actual;
 366         s64                     smoothed_derivative;
 367
 368         unsigned                p_term_inverse;
 369         unsigned                d_smooth;
 370         unsigned                d_term;
 371
 372         /* for exporting to sysfs (no effect on behavior) */
 373         s64                     last_derivative;
 374         s64                     last_proportional;
 375         s64                     last_change;
 376         s64                     last_target;
 377
 378         /*
 379          * If true, the rate will not increase if bch2_ratelimit_delay()
 380          * is not being called often enough.
 381          */
 382         bool                    backpressure;
 383 };
 384
 385 void bch2_pd_controller_update(struct bch_pd_controller *, s64, s64, int);
 386 void bch2_pd_controller_init(struct bch_pd_controller *);
 387 void bch2_pd_controller_debug_to_text(struct printbuf *, struct bch_pd_controller *);
 388
 389 #define sysfs_pd_controller_attribute(name)                             \
 390         rw_attribute(name##_rate);                                      \
 391         rw_attribute(name##_rate_bytes);                                \
 392         rw_attribute(name##_rate_d_term);                               \
 393         rw_attribute(name##_rate_p_term_inverse);                       \
 394         read_attribute(name##_rate_debug)
 395
 396 #define sysfs_pd_controller_files(name)                                 \
 397         &sysfs_##name##_rate,                                           \
 398         &sysfs_##name##_rate_bytes,                                     \
 399         &sysfs_##name##_rate_d_term,                                    \
 400         &sysfs_##name##_rate_p_term_inverse,                            \
 401         &sysfs_##name##_rate_debug
 402
 403 #define sysfs_pd_controller_show(name, var)                             \
 404 do {                                                                    \
 405         sysfs_hprint(name##_rate,               (var)->rate.rate);      \
 406         sysfs_print(name##_rate_bytes,          (var)->rate.rate);      \
 407         sysfs_print(name##_rate_d_term,         (var)->d_term);         \
 408         sysfs_print(name##_rate_p_term_inverse, (var)->p_term_inverse); \
 409                                                                         \
 410         if (attr == &sysfs_##name##_rate_debug)                         \
 411                 bch2_pd_controller_debug_to_text(out, var);             \
 412 } while (0)
 413
 414 #define sysfs_pd_controller_store(name, var)                            \
 415 do {                                                                    \
 416         sysfs_strtoul_clamp(name##_rate,                                \
 417                             (var)->rate.rate, 1, UINT_MAX);             \
 418         sysfs_strtoul_clamp(name##_rate_bytes,                          \
 419                             (var)->rate.rate, 1, UINT_MAX);             \
 420         sysfs_strtoul(name##_rate_d_term,       (var)->d_term);         \
 421         sysfs_strtoul_clamp(name##_rate_p_term_inverse,                 \
 422                             (var)->p_term_inverse, 1, INT_MAX);         \
 423 } while (0)
 424
 425 #define container_of_or_null(ptr, type, member)                         \
 426 ({                                                                      \
 427         typeof(ptr) _ptr = ptr;                                         \
 428         _ptr ? container_of(_ptr, type, member) : NULL;                 \
 429 })
 430
 431 /* Does linear interpolation between powers of two */
 432 static inline unsigned fract_exp_two(unsigned x, unsigned fract_bits)
 433 {
 434         unsigned fract = x & ~(~0 << fract_bits);
 435
 436         x >>= fract_bits;
 437         x   = 1 << x;
 438         x  += (x * fract) >> fract_bits;
 439
 440         return x;
 441 }
 442
 443 void bch2_bio_map(struct bio *bio, void *base, size_t);
 444 int bch2_bio_alloc_pages(struct bio *, size_t, gfp_t);
 445
 446 static inline sector_t bdev_sectors(struct block_device *bdev)
 447 {
 448         return bdev->bd_inode->i_size >> 9;
 449 }
 450
 451 #define closure_bio_submit(bio, cl)                                     \
 452 do {                                                                    \
 453         closure_get(cl);                                                \
 454         submit_bio(bio);                                                \
 455 } while (0)
 456
 457 #define kthread_wait(cond)                                              \
 458 ({                                                                      \
 459         int _ret = 0;                                                   \
 460                                                                         \
 461         while (1) {                                                     \
 462                 set_current_state(TASK_INTERRUPTIBLE);                  \
 463                 if (kthread_should_stop()) {                            \
 464                         _ret = -1;                                      \
 465                         break;                                          \
 466                 }                                                       \
 467                                                                         \
 468                 if (cond)                                               \
 469                         break;                                          \
 470                                                                         \
 471                 schedule();                                             \
 472         }                                                               \
 473         set_current_state(TASK_RUNNING);                                \
 474         _ret;                                                           \
 475 })
 476
 477 #define kthread_wait_freezable(cond)                                    \
 478 ({                                                                      \
 479         int _ret = 0;                                                   \
 480         while (1) {                                                     \
 481                 set_current_state(TASK_INTERRUPTIBLE);                  \
 482                 if (kthread_should_stop()) {                            \
 483                         _ret = -1;                                      \
 484                         break;                                          \
 485                 }                                                       \
 486                                                                         \
 487                 if (cond)                                               \
 488                         break;                                          \
 489                                                                         \
 490                 schedule();                                             \
 491                 try_to_freeze();                                        \
 492         }                                                               \
 493         set_current_state(TASK_RUNNING);                                \
 494         _ret;                                                           \
 495 })
 496
 497 size_t bch2_rand_range(size_t);
 498
 499 void memcpy_to_bio(struct bio *, struct bvec_iter, const void *);
 500 void memcpy_from_bio(void *, struct bio *, struct bvec_iter);
 501
 502 static inline void memcpy_u64s_small(void *dst, const void *src,
 503                                      unsigned u64s)
 504 {
 505         u64 *d = dst;
 506         const u64 *s = src;
 507
 508         while (u64s--)
 509                 *d++ = *s++;
 510 }
 511
 512 static inline void __memcpy_u64s(void *dst, const void *src,
 513                                  unsigned u64s)
 514 {
 515 #ifdef CONFIG_X86_64
 516         long d0, d1, d2;
 517
 518         asm volatile("rep ; movsq"
 519                      : "=&c" (d0), "=&D" (d1), "=&S" (d2)
 520                      : "0" (u64s), "1" (dst), "2" (src)
 521                      : "memory");
 522 #else
 523         u64 *d = dst;
 524         const u64 *s = src;
 525
 526         while (u64s--)
 527                 *d++ = *s++;
 528 #endif
 529 }
 530
 531 static inline void memcpy_u64s(void *dst, const void *src,
 532                                unsigned u64s)
 533 {
 534         EBUG_ON(!(dst >= src + u64s * sizeof(u64) ||
 535                  dst + u64s * sizeof(u64) <= src));
 536
 537         __memcpy_u64s(dst, src, u64s);
 538 }
 539
 540 static inline void __memmove_u64s_down(void *dst, const void *src,
 541                                        unsigned u64s)
 542 {
 543         __memcpy_u64s(dst, src, u64s);
 544 }
 545
 546 static inline void memmove_u64s_down(void *dst, const void *src,
 547                                      unsigned u64s)
 548 {
 549         EBUG_ON(dst > src);
 550
 551         __memmove_u64s_down(dst, src, u64s);
 552 }
 553
 554 static inline void __memmove_u64s_down_small(void *dst, const void *src,
 555                                        unsigned u64s)
 556 {
 557         memcpy_u64s_small(dst, src, u64s);
 558 }
 559
 560 static inline void memmove_u64s_down_small(void *dst, const void *src,
 561                                      unsigned u64s)
 562 {
 563         EBUG_ON(dst > src);
 564
 565         __memmove_u64s_down_small(dst, src, u64s);
 566 }
 567
 568 static inline void __memmove_u64s_up_small(void *_dst, const void *_src,
 569                                            unsigned u64s)
 570 {
 571         u64 *dst = (u64 *) _dst + u64s;
 572         u64 *src = (u64 *) _src + u64s;
 573
 574         while (u64s--)
 575                 *--dst = *--src;
 576 }
 577
 578 static inline void memmove_u64s_up_small(void *dst, const void *src,
 579                                          unsigned u64s)
 580 {
 581         EBUG_ON(dst < src);
 582
 583         __memmove_u64s_up_small(dst, src, u64s);
 584 }
 585
 586 static inline void __memmove_u64s_up(void *_dst, const void *_src,
 587                                      unsigned u64s)
 588 {
 589         u64 *dst = (u64 *) _dst + u64s - 1;
 590         u64 *src = (u64 *) _src + u64s - 1;
 591
 592 #ifdef CONFIG_X86_64
 593         long d0, d1, d2;
 594
 595         asm volatile("std ;\n"
 596                      "rep ; movsq\n"
 597                      "cld ;\n"
 598                      : "=&c" (d0), "=&D" (d1), "=&S" (d2)
 599                      : "0" (u64s), "1" (dst), "2" (src)
 600                      : "memory");
 601 #else
 602         while (u64s--)
 603                 *dst-- = *src--;
 604 #endif
 605 }
 606
 607 static inline void memmove_u64s_up(void *dst, const void *src,
 608                                    unsigned u64s)
 609 {
 610         EBUG_ON(dst < src);
 611
 612         __memmove_u64s_up(dst, src, u64s);
 613 }
 614
 615 static inline void memmove_u64s(void *dst, const void *src,
 616                                 unsigned u64s)
 617 {
 618         if (dst < src)
 619                 __memmove_u64s_down(dst, src, u64s);
 620         else
 621                 __memmove_u64s_up(dst, src, u64s);
 622 }
 623
 624 /* Set the last few bytes up to a u64 boundary given an offset into a buffer. */
 625 static inline void memset_u64s_tail(void *s, int c, unsigned bytes)
 626 {
 627         unsigned rem = round_up(bytes, sizeof(u64)) - bytes;
 628
 629         memset(s + bytes, c, rem);
 630 }
 631
 632 static inline void __move_gap(void *array, size_t element_size,
 633                               size_t nr, size_t size,
 634                               size_t old_gap, size_t new_gap)
 635 {
 636         size_t gap_end = old_gap + size - nr;
 637
 638         if (new_gap < old_gap) {
 639                 size_t move = old_gap - new_gap;
 640
 641                 memmove(array + element_size * (gap_end - move),
 642                         array + element_size * (old_gap - move),
 643                                 element_size * move);
 644         } else if (new_gap > old_gap) {
 645                 size_t move = new_gap - old_gap;
 646
 647                 memmove(array + element_size * old_gap,
 648                         array + element_size * gap_end,
 649                                 element_size * move);
 650         }
 651 }
 652
 653 /* Move the gap in a gap buffer: */
 654 #define move_gap(_array, _nr, _size, _old_gap, _new_gap)        \
 655         __move_gap(_array, sizeof(_array[0]), _nr, _size, _old_gap, _new_gap)
 656
 657 #define bubble_sort(_base, _nr, _cmp)                                   \
 658 do {                                                                    \
 659         ssize_t _i, _last;                                              \
 660         bool _swapped = true;                                           \
 661                                                                         \
 662         for (_last= (ssize_t) (_nr) - 1; _last > 0 && _swapped; --_last) {\
 663                 _swapped = false;                                       \
 664                 for (_i = 0; _i < _last; _i++)                          \
 665                         if (_cmp((_base)[_i], (_base)[_i + 1]) > 0) {   \
 666                                 swap((_base)[_i], (_base)[_i + 1]);     \
 667                                 _swapped = true;                        \
 668                         }                                               \
 669         }                                                               \
 670 } while (0)
 671
 672 static inline u64 percpu_u64_get(u64 __percpu *src)
 673 {
 674         u64 ret = 0;
 675         int cpu;
 676
 677         for_each_possible_cpu(cpu)
 678                 ret += *per_cpu_ptr(src, cpu);
 679         return ret;
 680 }
 681
 682 static inline void percpu_u64_set(u64 __percpu *dst, u64 src)
 683 {
 684         int cpu;
 685
 686         for_each_possible_cpu(cpu)
 687                 *per_cpu_ptr(dst, cpu) = 0;
 688         this_cpu_write(*dst, src);
 689 }
 690
 691 static inline void acc_u64s(u64 *acc, const u64 *src, unsigned nr)
 692 {
 693         unsigned i;
 694
 695         for (i = 0; i < nr; i++)
 696                 acc[i] += src[i];
 697 }
 698
 699 static inline void acc_u64s_percpu(u64 *acc, const u64 __percpu *src,
 700                                    unsigned nr)
 701 {
 702         int cpu;
 703
 704         for_each_possible_cpu(cpu)
 705                 acc_u64s(acc, per_cpu_ptr(src, cpu), nr);
 706 }
 707
 708 static inline void percpu_memset(void __percpu *p, int c, size_t bytes)
 709 {
 710         int cpu;
 711
 712         for_each_possible_cpu(cpu)
 713                 memset(per_cpu_ptr(p, cpu), c, bytes);
 714 }
 715
 716 u64 *bch2_acc_percpu_u64s(u64 __percpu *, unsigned);
 717
 718 #define cmp_int(l, r)           ((l > r) - (l < r))
 719
 720 static inline int u8_cmp(u8 l, u8 r)
 721 {
 722         return cmp_int(l, r);
 723 }
 724
 725 static inline int cmp_le32(__le32 l, __le32 r)
 726 {
 727         return cmp_int(le32_to_cpu(l), le32_to_cpu(r));
 728 }
 729
 730 #include <linux/uuid.h>
 731
 732 #define QSTR(n) { { { .len = strlen(n) } }, .name = n }
 733
 734 static inline bool qstr_eq(const struct qstr l, const struct qstr r)
 735 {
 736         return l.len == r.len && !memcmp(l.name, r.name, l.len);
 737 }
 738
 739 void bch2_darray_str_exit(darray_str *);
 740 int bch2_split_devs(const char *, darray_str *);
 741
 742 #endif /* _BCACHEFS_UTIL_H */