]> git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/btree_types.h
Update bcachefs sources to 24c6361e20 bcachefs: Fix a trans path overflow in bch2_btr...
[bcachefs-tools-debian] / libbcachefs / btree_types.h
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _BCACHEFS_BTREE_TYPES_H
3 #define _BCACHEFS_BTREE_TYPES_H
4
5 #include <linux/list.h>
6 #include <linux/rhashtable.h>
7 #include <linux/six.h>
8
9 #include "bkey_methods.h"
10 #include "buckets_types.h"
11 #include "darray.h"
12 #include "journal_types.h"
13
14 struct open_bucket;
15 struct btree_update;
16 struct btree_trans;
17
18 #define MAX_BSETS               3U
19
20 struct btree_nr_keys {
21
22         /*
23          * Amount of live metadata (i.e. size of node after a compaction) in
24          * units of u64s
25          */
26         u16                     live_u64s;
27         u16                     bset_u64s[MAX_BSETS];
28
29         /* live keys only: */
30         u16                     packed_keys;
31         u16                     unpacked_keys;
32 };
33
34 struct bset_tree {
35         /*
36          * We construct a binary tree in an array as if the array
37          * started at 1, so that things line up on the same cachelines
38          * better: see comments in bset.c at cacheline_to_bkey() for
39          * details
40          */
41
42         /* size of the binary tree and prev array */
43         u16                     size;
44
45         /* function of size - precalculated for to_inorder() */
46         u16                     extra;
47
48         u16                     data_offset;
49         u16                     aux_data_offset;
50         u16                     end_offset;
51 };
52
53 struct btree_write {
54         struct journal_entry_pin        journal;
55 };
56
57 struct btree_alloc {
58         struct open_buckets     ob;
59         __BKEY_PADDED(k, BKEY_BTREE_PTR_VAL_U64s_MAX);
60 };
61
62 struct btree_bkey_cached_common {
63         struct six_lock         lock;
64         u8                      level;
65         u8                      btree_id;
66         bool                    cached;
67 };
68
69 struct btree {
70         struct btree_bkey_cached_common c;
71
72         struct rhash_head       hash;
73         u64                     hash_val;
74
75         unsigned long           flags;
76         u16                     written;
77         u8                      nsets;
78         u8                      nr_key_bits;
79         u16                     version_ondisk;
80
81         struct bkey_format      format;
82
83         struct btree_node       *data;
84         void                    *aux_data;
85
86         /*
87          * Sets of sorted keys - the real btree node - plus a binary search tree
88          *
89          * set[0] is special; set[0]->tree, set[0]->prev and set[0]->data point
90          * to the memory we have allocated for this btree node. Additionally,
91          * set[0]->data points to the entire btree node as it exists on disk.
92          */
93         struct bset_tree        set[MAX_BSETS];
94
95         struct btree_nr_keys    nr;
96         u16                     sib_u64s[2];
97         u16                     whiteout_u64s;
98         u8                      byte_order;
99         u8                      unpack_fn_len;
100
101         struct btree_write      writes[2];
102
103         /* Key/pointer for this btree node */
104         __BKEY_PADDED(key, BKEY_BTREE_PTR_VAL_U64s_MAX);
105
106         /*
107          * XXX: add a delete sequence number, so when bch2_btree_node_relock()
108          * fails because the lock sequence number has changed - i.e. the
109          * contents were modified - we can still relock the node if it's still
110          * the one we want, without redoing the traversal
111          */
112
113         /*
114          * For asynchronous splits/interior node updates:
115          * When we do a split, we allocate new child nodes and update the parent
116          * node to point to them: we update the parent in memory immediately,
117          * but then we must wait until the children have been written out before
118          * the update to the parent can be written - this is a list of the
119          * btree_updates that are blocking this node from being
120          * written:
121          */
122         struct list_head        write_blocked;
123
124         /*
125          * Also for asynchronous splits/interior node updates:
126          * If a btree node isn't reachable yet, we don't want to kick off
127          * another write - because that write also won't yet be reachable and
128          * marking it as completed before it's reachable would be incorrect:
129          */
130         unsigned long           will_make_reachable;
131
132         struct open_buckets     ob;
133
134         /* lru list */
135         struct list_head        list;
136 };
137
138 struct btree_cache {
139         struct rhashtable       table;
140         bool                    table_init_done;
141         /*
142          * We never free a struct btree, except on shutdown - we just put it on
143          * the btree_cache_freed list and reuse it later. This simplifies the
144          * code, and it doesn't cost us much memory as the memory usage is
145          * dominated by buffers that hold the actual btree node data and those
146          * can be freed - and the number of struct btrees allocated is
147          * effectively bounded.
148          *
149          * btree_cache_freeable effectively is a small cache - we use it because
150          * high order page allocations can be rather expensive, and it's quite
151          * common to delete and allocate btree nodes in quick succession. It
152          * should never grow past ~2-3 nodes in practice.
153          */
154         struct mutex            lock;
155         struct list_head        live;
156         struct list_head        freeable;
157         struct list_head        freed_pcpu;
158         struct list_head        freed_nonpcpu;
159
160         /* Number of elements in live + freeable lists */
161         unsigned                used;
162         unsigned                reserve;
163         atomic_t                dirty;
164         struct shrinker         shrink;
165
166         /*
167          * If we need to allocate memory for a new btree node and that
168          * allocation fails, we can cannibalize another node in the btree cache
169          * to satisfy the allocation - lock to guarantee only one thread does
170          * this at a time:
171          */
172         struct task_struct      *alloc_lock;
173         struct closure_waitlist alloc_wait;
174 };
175
176 struct btree_node_iter {
177         struct btree_node_iter_set {
178                 u16     k, end;
179         } data[MAX_BSETS];
180 };
181
182 /*
183  * Iterate over all possible positions, synthesizing deleted keys for holes:
184  */
185 #define BTREE_ITER_SLOTS                (1 << 0)
186 #define BTREE_ITER_ALL_LEVELS           (1 << 1)
187 /*
188  * Indicates that intent locks should be taken on leaf nodes, because we expect
189  * to be doing updates:
190  */
191 #define BTREE_ITER_INTENT               (1 << 2)
192 /*
193  * Causes the btree iterator code to prefetch additional btree nodes from disk:
194  */
195 #define BTREE_ITER_PREFETCH             (1 << 3)
196 /*
197  * Used in bch2_btree_iter_traverse(), to indicate whether we're searching for
198  * @pos or the first key strictly greater than @pos
199  */
200 #define BTREE_ITER_IS_EXTENTS           (1 << 4)
201 #define BTREE_ITER_NOT_EXTENTS          (1 << 5)
202 #define BTREE_ITER_CACHED               (1 << 6)
203 #define BTREE_ITER_WITH_KEY_CACHE       (1 << 7)
204 #define BTREE_ITER_WITH_UPDATES         (1 << 8)
205 #define BTREE_ITER_WITH_JOURNAL         (1 << 9)
206 #define __BTREE_ITER_ALL_SNAPSHOTS      (1 << 10)
207 #define BTREE_ITER_ALL_SNAPSHOTS        (1 << 11)
208 #define BTREE_ITER_FILTER_SNAPSHOTS     (1 << 12)
209 #define BTREE_ITER_NOPRESERVE           (1 << 13)
210
211 enum btree_path_uptodate {
212         BTREE_ITER_UPTODATE             = 0,
213         BTREE_ITER_NEED_RELOCK          = 1,
214         BTREE_ITER_NEED_TRAVERSE        = 2,
215 };
216
217 struct btree_path {
218         u8                      idx;
219         u8                      sorted_idx;
220         u8                      ref;
221         u8                      intent_ref;
222
223         /* btree_iter_copy starts here: */
224         struct bpos             pos;
225
226         enum btree_id           btree_id:4;
227         bool                    cached:1;
228         bool                    preserve:1;
229         enum btree_path_uptodate uptodate:2;
230         /*
231          * When true, failing to relock this path will cause the transaction to
232          * restart:
233          */
234         bool                    should_be_locked:1;
235         unsigned                level:3,
236                                 locks_want:4;
237         u8                      nodes_locked;
238
239         struct btree_path_level {
240                 struct btree    *b;
241                 struct btree_node_iter iter;
242                 u32             lock_seq;
243 #ifdef CONFIG_BCACHEFS_LOCK_TIME_STATS
244                 u64             lock_taken_time;
245 #endif
246         }                       l[BTREE_MAX_DEPTH];
247 #ifdef CONFIG_BCACHEFS_DEBUG
248         unsigned long           ip_allocated;
249 #endif
250 };
251
252 static inline struct btree_path_level *path_l(struct btree_path *path)
253 {
254         return path->l + path->level;
255 }
256
257 /*
258  * @pos                 - iterator's current position
259  * @level               - current btree depth
260  * @locks_want          - btree level below which we start taking intent locks
261  * @nodes_locked        - bitmask indicating which nodes in @nodes are locked
262  * @nodes_intent_locked - bitmask indicating which locks are intent locks
263  */
264 struct btree_iter {
265         struct btree_trans      *trans;
266         struct btree_path       *path;
267         struct btree_path       *update_path;
268         struct btree_path       *key_cache_path;
269
270         enum btree_id           btree_id:4;
271         unsigned                min_depth:3;
272         unsigned                advanced:1;
273
274         /* btree_iter_copy starts here: */
275         u16                     flags;
276
277         /* When we're filtering by snapshot, the snapshot ID we're looking for: */
278         unsigned                snapshot;
279
280         struct bpos             pos;
281         struct bpos             pos_after_commit;
282         /*
283          * Current unpacked key - so that bch2_btree_iter_next()/
284          * bch2_btree_iter_next_slot() can correctly advance pos.
285          */
286         struct bkey             k;
287
288         /* BTREE_ITER_WITH_JOURNAL: */
289         size_t                  journal_idx;
290         struct bpos             journal_pos;
291 #ifdef CONFIG_BCACHEFS_DEBUG
292         unsigned long           ip_allocated;
293 #endif
294 };
295
296 struct btree_key_cache_freelist {
297         struct bkey_cached      *objs[16];
298         unsigned                nr;
299 };
300
301 struct btree_key_cache {
302         struct mutex            lock;
303         struct rhashtable       table;
304         bool                    table_init_done;
305         struct list_head        freed_pcpu;
306         struct list_head        freed_nonpcpu;
307         struct shrinker         shrink;
308         unsigned                shrink_iter;
309         struct btree_key_cache_freelist __percpu *pcpu_freed;
310
311         atomic_long_t           nr_freed;
312         atomic_long_t           nr_keys;
313         atomic_long_t           nr_dirty;
314 };
315
316 struct bkey_cached_key {
317         u32                     btree_id;
318         struct bpos             pos;
319 } __attribute__((packed, aligned(4)));
320
321 #define BKEY_CACHED_ACCESSED            0
322 #define BKEY_CACHED_DIRTY               1
323
324 struct bkey_cached {
325         struct btree_bkey_cached_common c;
326
327         unsigned long           flags;
328         u16                     u64s;
329         bool                    valid;
330         u32                     btree_trans_barrier_seq;
331         struct bkey_cached_key  key;
332
333         struct rhash_head       hash;
334         struct list_head        list;
335
336         struct journal_preres   res;
337         struct journal_entry_pin journal;
338
339         struct bkey_i           *k;
340 };
341
342 static inline struct bpos btree_node_pos(struct btree_bkey_cached_common *b)
343 {
344         return !b->cached
345                 ? container_of(b, struct btree, c)->key.k.p
346                 : container_of(b, struct bkey_cached, c)->key.pos;
347 }
348
349 struct btree_insert_entry {
350         unsigned                flags;
351         u8                      bkey_type;
352         enum btree_id           btree_id:8;
353         u8                      level:4;
354         bool                    cached:1;
355         bool                    insert_trigger_run:1;
356         bool                    overwrite_trigger_run:1;
357         bool                    key_cache_already_flushed:1;
358         /*
359          * @old_k may be a key from the journal; @old_btree_u64s always refers
360          * to the size of the key being overwritten in the btree:
361          */
362         u8                      old_btree_u64s;
363         struct bkey_i           *k;
364         struct btree_path       *path;
365         /* key being overwritten: */
366         struct bkey             old_k;
367         const struct bch_val    *old_v;
368         unsigned long           ip_allocated;
369 };
370
371 #ifndef CONFIG_LOCKDEP
372 #define BTREE_ITER_MAX          64
373 #else
374 #define BTREE_ITER_MAX          32
375 #endif
376
377 struct btree_trans_commit_hook;
378 typedef int (btree_trans_commit_hook_fn)(struct btree_trans *, struct btree_trans_commit_hook *);
379
380 struct btree_trans_commit_hook {
381         btree_trans_commit_hook_fn      *fn;
382         struct btree_trans_commit_hook  *next;
383 };
384
385 #define BTREE_TRANS_MEM_MAX     (1U << 16)
386
387 #define BTREE_TRANS_MAX_LOCK_HOLD_TIME_NS       10000
388
389 struct btree_trans {
390         struct bch_fs           *c;
391         const char              *fn;
392         struct closure          ref;
393         struct list_head        list;
394         u64                     last_begin_time;
395
396         u8                      lock_may_not_fail;
397         u8                      lock_must_abort;
398         struct btree_bkey_cached_common *locking;
399         struct six_lock_waiter  locking_wait;
400
401         int                     srcu_idx;
402
403         u8                      fn_idx;
404         u8                      nr_sorted;
405         u8                      nr_updates;
406         u8                      traverse_all_idx;
407         bool                    used_mempool:1;
408         bool                    in_traverse_all:1;
409         bool                    memory_allocation_failure:1;
410         bool                    is_initial_gc:1;
411         enum bch_errcode        restarted:16;
412         u32                     restart_count;
413         unsigned long           last_restarted_ip;
414
415         /*
416          * For when bch2_trans_update notices we'll be splitting a compressed
417          * extent:
418          */
419         unsigned                extra_journal_res;
420         unsigned                nr_max_paths;
421
422         u64                     paths_allocated;
423
424         unsigned                mem_top;
425         unsigned                mem_max;
426         unsigned                mem_bytes;
427         void                    *mem;
428
429         u8                      sorted[BTREE_ITER_MAX];
430         struct btree_path       *paths;
431         struct btree_insert_entry *updates;
432
433         /* update path: */
434         struct btree_trans_commit_hook *hooks;
435         DARRAY(u64)             extra_journal_entries;
436         struct journal_entry_pin *journal_pin;
437
438         struct journal_res      journal_res;
439         struct journal_preres   journal_preres;
440         u64                     *journal_seq;
441         struct disk_reservation *disk_res;
442         unsigned                flags;
443         unsigned                journal_u64s;
444         unsigned                journal_preres_u64s;
445         struct replicas_delta_list *fs_usage_deltas;
446 };
447
448 #define BTREE_FLAGS()                                                   \
449         x(read_in_flight)                                               \
450         x(read_error)                                                   \
451         x(dirty)                                                        \
452         x(need_write)                                                   \
453         x(write_blocked)                                                \
454         x(will_make_reachable)                                          \
455         x(noevict)                                                      \
456         x(write_idx)                                                    \
457         x(accessed)                                                     \
458         x(write_in_flight)                                              \
459         x(write_in_flight_inner)                                        \
460         x(just_written)                                                 \
461         x(dying)                                                        \
462         x(fake)                                                         \
463         x(need_rewrite)                                                 \
464         x(never_write)
465
466 enum btree_flags {
467 #define x(flag) BTREE_NODE_##flag,
468         BTREE_FLAGS()
469 #undef x
470 };
471
472 #define x(flag)                                                         \
473 static inline bool btree_node_ ## flag(struct btree *b)                 \
474 {       return test_bit(BTREE_NODE_ ## flag, &b->flags); }              \
475                                                                         \
476 static inline void set_btree_node_ ## flag(struct btree *b)             \
477 {       set_bit(BTREE_NODE_ ## flag, &b->flags); }                      \
478                                                                         \
479 static inline void clear_btree_node_ ## flag(struct btree *b)           \
480 {       clear_bit(BTREE_NODE_ ## flag, &b->flags); }
481
482 BTREE_FLAGS()
483 #undef x
484
485 static inline struct btree_write *btree_current_write(struct btree *b)
486 {
487         return b->writes + btree_node_write_idx(b);
488 }
489
490 static inline struct btree_write *btree_prev_write(struct btree *b)
491 {
492         return b->writes + (btree_node_write_idx(b) ^ 1);
493 }
494
495 static inline struct bset_tree *bset_tree_last(struct btree *b)
496 {
497         EBUG_ON(!b->nsets);
498         return b->set + b->nsets - 1;
499 }
500
501 static inline void *
502 __btree_node_offset_to_ptr(const struct btree *b, u16 offset)
503 {
504         return (void *) ((u64 *) b->data + 1 + offset);
505 }
506
507 static inline u16
508 __btree_node_ptr_to_offset(const struct btree *b, const void *p)
509 {
510         u16 ret = (u64 *) p - 1 - (u64 *) b->data;
511
512         EBUG_ON(__btree_node_offset_to_ptr(b, ret) != p);
513         return ret;
514 }
515
516 static inline struct bset *bset(const struct btree *b,
517                                 const struct bset_tree *t)
518 {
519         return __btree_node_offset_to_ptr(b, t->data_offset);
520 }
521
522 static inline void set_btree_bset_end(struct btree *b, struct bset_tree *t)
523 {
524         t->end_offset =
525                 __btree_node_ptr_to_offset(b, vstruct_last(bset(b, t)));
526 }
527
528 static inline void set_btree_bset(struct btree *b, struct bset_tree *t,
529                                   const struct bset *i)
530 {
531         t->data_offset = __btree_node_ptr_to_offset(b, i);
532         set_btree_bset_end(b, t);
533 }
534
535 static inline struct bset *btree_bset_first(struct btree *b)
536 {
537         return bset(b, b->set);
538 }
539
540 static inline struct bset *btree_bset_last(struct btree *b)
541 {
542         return bset(b, bset_tree_last(b));
543 }
544
545 static inline u16
546 __btree_node_key_to_offset(const struct btree *b, const struct bkey_packed *k)
547 {
548         return __btree_node_ptr_to_offset(b, k);
549 }
550
551 static inline struct bkey_packed *
552 __btree_node_offset_to_key(const struct btree *b, u16 k)
553 {
554         return __btree_node_offset_to_ptr(b, k);
555 }
556
557 static inline unsigned btree_bkey_first_offset(const struct bset_tree *t)
558 {
559         return t->data_offset + offsetof(struct bset, _data) / sizeof(u64);
560 }
561
562 #define btree_bkey_first(_b, _t)                                        \
563 ({                                                                      \
564         EBUG_ON(bset(_b, _t)->start !=                                  \
565                 __btree_node_offset_to_key(_b, btree_bkey_first_offset(_t)));\
566                                                                         \
567         bset(_b, _t)->start;                                            \
568 })
569
570 #define btree_bkey_last(_b, _t)                                         \
571 ({                                                                      \
572         EBUG_ON(__btree_node_offset_to_key(_b, (_t)->end_offset) !=     \
573                 vstruct_last(bset(_b, _t)));                            \
574                                                                         \
575         __btree_node_offset_to_key(_b, (_t)->end_offset);               \
576 })
577
578 static inline unsigned bset_u64s(struct bset_tree *t)
579 {
580         return t->end_offset - t->data_offset -
581                 sizeof(struct bset) / sizeof(u64);
582 }
583
584 static inline unsigned bset_dead_u64s(struct btree *b, struct bset_tree *t)
585 {
586         return bset_u64s(t) - b->nr.bset_u64s[t - b->set];
587 }
588
589 static inline unsigned bset_byte_offset(struct btree *b, void *i)
590 {
591         return i - (void *) b->data;
592 }
593
594 enum btree_node_type {
595 #define x(kwd, val) BKEY_TYPE_##kwd = val,
596         BCH_BTREE_IDS()
597 #undef x
598         BKEY_TYPE_btree,
599 };
600
601 /* Type of a key in btree @id at level @level: */
602 static inline enum btree_node_type __btree_node_type(unsigned level, enum btree_id id)
603 {
604         return level ? BKEY_TYPE_btree : (enum btree_node_type) id;
605 }
606
607 /* Type of keys @b contains: */
608 static inline enum btree_node_type btree_node_type(struct btree *b)
609 {
610         return __btree_node_type(b->c.level, b->c.btree_id);
611 }
612
613 #define BTREE_NODE_TYPE_HAS_TRANS_TRIGGERS              \
614         ((1U << BKEY_TYPE_extents)|                     \
615          (1U << BKEY_TYPE_alloc)|                       \
616          (1U << BKEY_TYPE_inodes)|                      \
617          (1U << BKEY_TYPE_stripes)|                     \
618          (1U << BKEY_TYPE_reflink)|                     \
619          (1U << BKEY_TYPE_btree))
620
621 #define BTREE_NODE_TYPE_HAS_MEM_TRIGGERS                \
622         ((1U << BKEY_TYPE_alloc)|                       \
623          (1U << BKEY_TYPE_inodes)|                      \
624          (1U << BKEY_TYPE_stripes)|                     \
625          (1U << BKEY_TYPE_snapshots))
626
627 #define BTREE_NODE_TYPE_HAS_TRIGGERS                    \
628         (BTREE_NODE_TYPE_HAS_TRANS_TRIGGERS|            \
629          BTREE_NODE_TYPE_HAS_MEM_TRIGGERS)
630
631 #define BTREE_ID_IS_EXTENTS                             \
632         ((1U << BTREE_ID_extents)|                      \
633          (1U << BTREE_ID_reflink)|                      \
634          (1U << BTREE_ID_freespace))
635
636 static inline bool btree_node_type_is_extents(enum btree_node_type type)
637 {
638         return (1U << type) & BTREE_ID_IS_EXTENTS;
639 }
640
641 #define BTREE_ID_HAS_SNAPSHOTS                          \
642         ((1U << BTREE_ID_extents)|                      \
643          (1U << BTREE_ID_inodes)|                       \
644          (1U << BTREE_ID_dirents)|                      \
645          (1U << BTREE_ID_xattrs))
646
647 #define BTREE_ID_HAS_PTRS                               \
648         ((1U << BTREE_ID_extents)|                      \
649          (1U << BTREE_ID_reflink))
650
651 static inline bool btree_type_has_snapshots(enum btree_id id)
652 {
653         return (1 << id) & BTREE_ID_HAS_SNAPSHOTS;
654 }
655
656 static inline bool btree_type_has_ptrs(enum btree_id id)
657 {
658         return (1 << id) & BTREE_ID_HAS_PTRS;
659 }
660
661 static inline bool btree_node_type_needs_gc(enum btree_node_type type)
662 {
663         return BTREE_NODE_TYPE_HAS_TRIGGERS & (1U << type);
664 }
665
666 struct btree_root {
667         struct btree            *b;
668
669         /* On disk root - see async splits: */
670         __BKEY_PADDED(key, BKEY_BTREE_PTR_VAL_U64s_MAX);
671         u8                      level;
672         u8                      alive;
673         s8                      error;
674 };
675
676 enum btree_insert_ret {
677         BTREE_INSERT_OK,
678         /* leaf node needs to be split */
679         BTREE_INSERT_BTREE_NODE_FULL,
680         BTREE_INSERT_NEED_MARK_REPLICAS,
681         BTREE_INSERT_NEED_JOURNAL_RES,
682         BTREE_INSERT_NEED_JOURNAL_RECLAIM,
683 };
684
685 enum btree_gc_coalesce_fail_reason {
686         BTREE_GC_COALESCE_FAIL_RESERVE_GET,
687         BTREE_GC_COALESCE_FAIL_KEYLIST_REALLOC,
688         BTREE_GC_COALESCE_FAIL_FORMAT_FITS,
689 };
690
691 enum btree_node_sibling {
692         btree_prev_sib,
693         btree_next_sib,
694 };
695
696 #endif /* _BCACHEFS_BTREE_TYPES_H */