]> git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/extents.h
fe92737354bd202815ea70fb627b383824911520
[bcachefs-tools-debian] / libbcachefs / extents.h
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _BCACHEFS_EXTENTS_H
3 #define _BCACHEFS_EXTENTS_H
4
5 #include "bcachefs.h"
6 #include "bkey.h"
7 #include "extents_types.h"
8
9 struct bch_fs;
10 struct btree_trans;
11 struct btree_insert_entry;
12
13 /* extent entries: */
14
15 #define extent_entry_last(_e)           bkey_val_end(_e)
16
17 #define entry_to_ptr(_entry)                                            \
18 ({                                                                      \
19         EBUG_ON((_entry) && !extent_entry_is_ptr(_entry));              \
20                                                                         \
21         __builtin_choose_expr(                                          \
22                 type_is_exact(_entry, const union bch_extent_entry *),  \
23                 (const struct bch_extent_ptr *) (_entry),               \
24                 (struct bch_extent_ptr *) (_entry));                    \
25 })
26
27 /* downcast, preserves const */
28 #define to_entry(_entry)                                                \
29 ({                                                                      \
30         BUILD_BUG_ON(!type_is(_entry, union bch_extent_crc *) &&        \
31                      !type_is(_entry, struct bch_extent_ptr *) &&       \
32                      !type_is(_entry, struct bch_extent_stripe_ptr *)); \
33                                                                         \
34         __builtin_choose_expr(                                          \
35                 (type_is_exact(_entry, const union bch_extent_crc *) || \
36                  type_is_exact(_entry, const struct bch_extent_ptr *) ||\
37                  type_is_exact(_entry, const struct bch_extent_stripe_ptr *)),\
38                 (const union bch_extent_entry *) (_entry),              \
39                 (union bch_extent_entry *) (_entry));                   \
40 })
41
42 static inline unsigned
43 __extent_entry_type(const union bch_extent_entry *e)
44 {
45         return e->type ? __ffs(e->type) : BCH_EXTENT_ENTRY_MAX;
46 }
47
48 static inline enum bch_extent_entry_type
49 extent_entry_type(const union bch_extent_entry *e)
50 {
51         int ret = __ffs(e->type);
52
53         EBUG_ON(ret < 0 || ret >= BCH_EXTENT_ENTRY_MAX);
54
55         return ret;
56 }
57
58 static inline size_t extent_entry_bytes(const union bch_extent_entry *entry)
59 {
60         switch (extent_entry_type(entry)) {
61 #define x(f, n)                                         \
62         case BCH_EXTENT_ENTRY_##f:                      \
63                 return sizeof(struct bch_extent_##f);
64         BCH_EXTENT_ENTRY_TYPES()
65 #undef x
66         default:
67                 BUG();
68         }
69 }
70
71 static inline size_t extent_entry_u64s(const union bch_extent_entry *entry)
72 {
73         return extent_entry_bytes(entry) / sizeof(u64);
74 }
75
76 static inline bool extent_entry_is_ptr(const union bch_extent_entry *e)
77 {
78         switch (extent_entry_type(e)) {
79         case BCH_EXTENT_ENTRY_ptr:
80                 return true;
81         default:
82                 return false;
83         }
84 }
85
86 static inline bool extent_entry_is_crc(const union bch_extent_entry *e)
87 {
88         switch (extent_entry_type(e)) {
89         case BCH_EXTENT_ENTRY_crc32:
90         case BCH_EXTENT_ENTRY_crc64:
91         case BCH_EXTENT_ENTRY_crc128:
92                 return true;
93         default:
94                 return false;
95         }
96 }
97
98 union bch_extent_crc {
99         u8                              type;
100         struct bch_extent_crc32         crc32;
101         struct bch_extent_crc64         crc64;
102         struct bch_extent_crc128        crc128;
103 };
104
105 #define __entry_to_crc(_entry)                                          \
106         __builtin_choose_expr(                                          \
107                 type_is_exact(_entry, const union bch_extent_entry *),  \
108                 (const union bch_extent_crc *) (_entry),                \
109                 (union bch_extent_crc *) (_entry))
110
111 #define entry_to_crc(_entry)                                            \
112 ({                                                                      \
113         EBUG_ON((_entry) && !extent_entry_is_crc(_entry));              \
114                                                                         \
115         __entry_to_crc(_entry);                                         \
116 })
117
118 static inline struct bch_extent_crc_unpacked
119 bch2_extent_crc_unpack(const struct bkey *k, const union bch_extent_crc *crc)
120 {
121 #define common_fields(_crc)                                             \
122                 .csum_type              = _crc.csum_type,               \
123                 .compression_type       = _crc.compression_type,        \
124                 .compressed_size        = _crc._compressed_size + 1,    \
125                 .uncompressed_size      = _crc._uncompressed_size + 1,  \
126                 .offset                 = _crc.offset,                  \
127                 .live_size              = k->size
128
129         if (!crc)
130                 return (struct bch_extent_crc_unpacked) {
131                         .compressed_size        = k->size,
132                         .uncompressed_size      = k->size,
133                         .live_size              = k->size,
134                 };
135
136         switch (extent_entry_type(to_entry(crc))) {
137         case BCH_EXTENT_ENTRY_crc32: {
138                 struct bch_extent_crc_unpacked ret = (struct bch_extent_crc_unpacked) {
139                         common_fields(crc->crc32),
140                 };
141
142                 *((__le32 *) &ret.csum.lo) = crc->crc32.csum;
143
144                 memcpy(&ret.csum.lo, &crc->crc32.csum,
145                        sizeof(crc->crc32.csum));
146
147                 return ret;
148         }
149         case BCH_EXTENT_ENTRY_crc64: {
150                 struct bch_extent_crc_unpacked ret = (struct bch_extent_crc_unpacked) {
151                         common_fields(crc->crc64),
152                         .nonce                  = crc->crc64.nonce,
153                         .csum.lo                = (__force __le64) crc->crc64.csum_lo,
154                 };
155
156                 *((__le16 *) &ret.csum.hi) = crc->crc64.csum_hi;
157
158                 return ret;
159         }
160         case BCH_EXTENT_ENTRY_crc128: {
161                 struct bch_extent_crc_unpacked ret = (struct bch_extent_crc_unpacked) {
162                         common_fields(crc->crc128),
163                         .nonce                  = crc->crc128.nonce,
164                         .csum                   = crc->crc128.csum,
165                 };
166
167                 return ret;
168         }
169         default:
170                 BUG();
171         }
172 #undef common_fields
173 }
174
175 /* bkey_ptrs: generically over any key type that has ptrs */
176
177 struct bkey_ptrs_c {
178         const union bch_extent_entry    *start;
179         const union bch_extent_entry    *end;
180 };
181
182 struct bkey_ptrs {
183         union bch_extent_entry  *start;
184         union bch_extent_entry  *end;
185 };
186
187 /* iterate over bkey ptrs */
188
189 #define extent_entry_next(_entry)                                       \
190         ((typeof(_entry)) ((void *) (_entry) + extent_entry_bytes(_entry)))
191
192 #define __bkey_extent_entry_for_each_from(_start, _end, _entry)         \
193         for ((_entry) = (_start);                                       \
194              (_entry) < (_end);                                         \
195              (_entry) = extent_entry_next(_entry))
196
197 #define __bkey_ptr_next(_ptr, _end)                                     \
198 ({                                                                      \
199         typeof(_end) _entry;                                            \
200                                                                         \
201         __bkey_extent_entry_for_each_from(to_entry(_ptr), _end, _entry) \
202                 if (extent_entry_is_ptr(_entry))                        \
203                         break;                                          \
204                                                                         \
205         _entry < (_end) ? entry_to_ptr(_entry) : NULL;                  \
206 })
207
208 #define bkey_extent_entry_for_each_from(_p, _entry, _start)             \
209         __bkey_extent_entry_for_each_from(_start, (_p).end, _entry)
210
211 #define bkey_extent_entry_for_each(_p, _entry)                          \
212         bkey_extent_entry_for_each_from(_p, _entry, _p.start)
213
214 #define __bkey_for_each_ptr(_start, _end, _ptr)                         \
215         for ((_ptr) = (_start);                                         \
216              ((_ptr) = __bkey_ptr_next(_ptr, _end));                    \
217              (_ptr)++)
218
219 #define bkey_ptr_next(_p, _ptr)                                         \
220         __bkey_ptr_next(_ptr, (_p).end)
221
222 #define bkey_for_each_ptr(_p, _ptr)                                     \
223         __bkey_for_each_ptr(&(_p).start->ptr, (_p).end, _ptr)
224
225 #define __bkey_ptr_next_decode(_k, _end, _ptr, _entry)                  \
226 ({                                                                      \
227         __label__ out;                                                  \
228                                                                         \
229         (_ptr).idx      = 0;                                            \
230         (_ptr).ec_nr    = 0;                                            \
231                                                                         \
232         __bkey_extent_entry_for_each_from(_entry, _end, _entry)         \
233                 switch (extent_entry_type(_entry)) {                    \
234                 case BCH_EXTENT_ENTRY_ptr:                              \
235                         (_ptr).ptr              = _entry->ptr;          \
236                         goto out;                                       \
237                 case BCH_EXTENT_ENTRY_crc32:                            \
238                 case BCH_EXTENT_ENTRY_crc64:                            \
239                 case BCH_EXTENT_ENTRY_crc128:                           \
240                         (_ptr).crc = bch2_extent_crc_unpack(_k,         \
241                                         entry_to_crc(_entry));          \
242                         break;                                          \
243                 case BCH_EXTENT_ENTRY_stripe_ptr:                       \
244                         (_ptr).ec[(_ptr).ec_nr++] = _entry->stripe_ptr; \
245                         break;                                          \
246                 }                                                       \
247 out:                                                                    \
248         _entry < (_end);                                                \
249 })
250
251 #define __bkey_for_each_ptr_decode(_k, _start, _end, _ptr, _entry)      \
252         for ((_ptr).crc = bch2_extent_crc_unpack(_k, NULL),             \
253              (_entry) = _start;                                         \
254              __bkey_ptr_next_decode(_k, _end, _ptr, _entry);            \
255              (_entry) = extent_entry_next(_entry))
256
257 #define bkey_for_each_ptr_decode(_k, _p, _ptr, _entry)                  \
258         __bkey_for_each_ptr_decode(_k, (_p).start, (_p).end,            \
259                                    _ptr, _entry)
260
261 /* utility code common to all keys with pointers: */
262
263 static inline struct bkey_ptrs_c bch2_bkey_ptrs_c(struct bkey_s_c k)
264 {
265         switch (k.k->type) {
266         case KEY_TYPE_btree_ptr: {
267                 struct bkey_s_c_btree_ptr e = bkey_s_c_to_btree_ptr(k);
268                 return (struct bkey_ptrs_c) {
269                         to_entry(&e.v->start[0]),
270                         to_entry(bkey_val_end(e))
271                 };
272         }
273         case KEY_TYPE_extent: {
274                 struct bkey_s_c_extent e = bkey_s_c_to_extent(k);
275                 return (struct bkey_ptrs_c) {
276                         e.v->start,
277                         extent_entry_last(e)
278                 };
279         }
280         case KEY_TYPE_stripe: {
281                 struct bkey_s_c_stripe s = bkey_s_c_to_stripe(k);
282                 return (struct bkey_ptrs_c) {
283                         to_entry(&s.v->ptrs[0]),
284                         to_entry(&s.v->ptrs[s.v->nr_blocks]),
285                 };
286         }
287         default:
288                 return (struct bkey_ptrs_c) { NULL, NULL };
289         }
290 }
291
292 static inline struct bkey_ptrs bch2_bkey_ptrs(struct bkey_s k)
293 {
294         struct bkey_ptrs_c p = bch2_bkey_ptrs_c(k.s_c);
295
296         return (struct bkey_ptrs) {
297                 (void *) p.start,
298                 (void *) p.end
299         };
300 }
301
302 static inline struct bch_devs_list bch2_bkey_devs(struct bkey_s_c k)
303 {
304         struct bch_devs_list ret = (struct bch_devs_list) { 0 };
305         struct bkey_ptrs_c p = bch2_bkey_ptrs_c(k);
306         const struct bch_extent_ptr *ptr;
307
308         bkey_for_each_ptr(p, ptr)
309                 ret.devs[ret.nr++] = ptr->dev;
310
311         return ret;
312 }
313
314 static inline struct bch_devs_list bch2_bkey_dirty_devs(struct bkey_s_c k)
315 {
316         struct bch_devs_list ret = (struct bch_devs_list) { 0 };
317         struct bkey_ptrs_c p = bch2_bkey_ptrs_c(k);
318         const struct bch_extent_ptr *ptr;
319
320         bkey_for_each_ptr(p, ptr)
321                 if (!ptr->cached)
322                         ret.devs[ret.nr++] = ptr->dev;
323
324         return ret;
325 }
326
327 static inline struct bch_devs_list bch2_bkey_cached_devs(struct bkey_s_c k)
328 {
329         struct bch_devs_list ret = (struct bch_devs_list) { 0 };
330         struct bkey_ptrs_c p = bch2_bkey_ptrs_c(k);
331         const struct bch_extent_ptr *ptr;
332
333         bkey_for_each_ptr(p, ptr)
334                 if (ptr->cached)
335                         ret.devs[ret.nr++] = ptr->dev;
336
337         return ret;
338 }
339
340 static inline bool bch2_bkey_has_device(struct bkey_s_c k, unsigned dev)
341 {
342         struct bkey_ptrs_c p = bch2_bkey_ptrs_c(k);
343         const struct bch_extent_ptr *ptr;
344
345         bkey_for_each_ptr(p, ptr)
346                 if (ptr->dev == dev)
347                         return ptr;
348
349         return NULL;
350 }
351
352 unsigned bch2_bkey_nr_ptrs(struct bkey_s_c);
353 unsigned bch2_bkey_nr_dirty_ptrs(struct bkey_s_c);
354 unsigned bch2_bkey_durability(struct bch_fs *, struct bkey_s_c);
355
356 void bch2_mark_io_failure(struct bch_io_failures *,
357                           struct extent_ptr_decoded *);
358 int bch2_bkey_pick_read_device(struct bch_fs *, struct bkey_s_c,
359                                struct bch_io_failures *,
360                                struct extent_ptr_decoded *);
361
362 void bch2_bkey_ptrs_to_text(struct printbuf *, struct bch_fs *,
363                             struct bkey_s_c);
364 const char *bch2_bkey_ptrs_invalid(const struct bch_fs *, struct bkey_s_c);
365
366 /* bch_btree_ptr: */
367
368 const char *bch2_btree_ptr_invalid(const struct bch_fs *, struct bkey_s_c);
369 void bch2_btree_ptr_debugcheck(struct bch_fs *, struct btree *,
370                                struct bkey_s_c);
371 void bch2_btree_ptr_to_text(struct printbuf *, struct bch_fs *,
372                             struct bkey_s_c);
373 void bch2_ptr_swab(const struct bkey_format *, struct bkey_packed *);
374
375 #define bch2_bkey_ops_btree_ptr (struct bkey_ops) {             \
376         .key_invalid    = bch2_btree_ptr_invalid,               \
377         .key_debugcheck = bch2_btree_ptr_debugcheck,            \
378         .val_to_text    = bch2_btree_ptr_to_text,               \
379         .swab           = bch2_ptr_swab,                        \
380 }
381
382 /* bch_extent: */
383
384 const char *bch2_extent_invalid(const struct bch_fs *, struct bkey_s_c);
385 void bch2_extent_debugcheck(struct bch_fs *, struct btree *, struct bkey_s_c);
386 void bch2_extent_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c);
387 bool bch2_extent_normalize(struct bch_fs *, struct bkey_s);
388 enum merge_result bch2_extent_merge(struct bch_fs *,
389                                     struct bkey_s, struct bkey_s);
390
391 #define bch2_bkey_ops_extent (struct bkey_ops) {                \
392         .key_invalid    = bch2_extent_invalid,                  \
393         .key_debugcheck = bch2_extent_debugcheck,               \
394         .val_to_text    = bch2_extent_to_text,                  \
395         .swab           = bch2_ptr_swab,                        \
396         .key_normalize  = bch2_extent_normalize,                \
397         .key_merge      = bch2_extent_merge,                    \
398 }
399
400 /* bch_reservation: */
401
402 const char *bch2_reservation_invalid(const struct bch_fs *, struct bkey_s_c);
403 void bch2_reservation_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c);
404 enum merge_result bch2_reservation_merge(struct bch_fs *,
405                                          struct bkey_s, struct bkey_s);
406
407 #define bch2_bkey_ops_reservation (struct bkey_ops) {           \
408         .key_invalid    = bch2_reservation_invalid,             \
409         .val_to_text    = bch2_reservation_to_text,             \
410         .key_merge      = bch2_reservation_merge,               \
411 }
412
413 void bch2_extent_trim_atomic(struct bkey_i *, struct btree_iter *);
414 bool bch2_extent_is_atomic(struct bkey_i *, struct btree_iter *);
415
416 enum btree_insert_ret
417 bch2_extent_can_insert(struct btree_trans *, struct btree_insert_entry *,
418                        unsigned *);
419 void bch2_insert_fixup_extent(struct btree_trans *,
420                               struct btree_insert_entry *);
421
422 void bch2_extent_mark_replicas_cached(struct bch_fs *, struct bkey_s_extent,
423                                       unsigned, unsigned);
424
425 const struct bch_extent_ptr *
426 bch2_extent_has_device(struct bkey_s_c_extent, unsigned);
427 const struct bch_extent_ptr *
428 bch2_extent_has_group(struct bch_fs *, struct bkey_s_c_extent, unsigned);
429 const struct bch_extent_ptr *
430 bch2_extent_has_target(struct bch_fs *, struct bkey_s_c_extent, unsigned);
431
432 unsigned bch2_extent_is_compressed(struct bkey_s_c);
433
434 bool bch2_extent_matches_ptr(struct bch_fs *, struct bkey_s_c_extent,
435                              struct bch_extent_ptr, u64);
436
437 static inline bool bkey_extent_is_data(const struct bkey *k)
438 {
439         switch (k->type) {
440         case KEY_TYPE_btree_ptr:
441         case KEY_TYPE_extent:
442                 return true;
443         default:
444                 return false;
445         }
446 }
447
448 static inline bool bkey_extent_is_allocation(const struct bkey *k)
449 {
450         switch (k->type) {
451         case KEY_TYPE_extent:
452         case KEY_TYPE_reservation:
453                 return true;
454         default:
455                 return false;
456         }
457 }
458
459 static inline bool bch2_extent_is_fully_allocated(struct bkey_s_c k)
460 {
461         return bkey_extent_is_allocation(k.k) &&
462                 !bch2_extent_is_compressed(k);
463 }
464
465 void bch2_bkey_append_ptr(struct bkey_i *, struct bch_extent_ptr);
466 void bch2_bkey_drop_device(struct bkey_s, unsigned);
467
468 /* Extent entry iteration: */
469
470 #define extent_for_each_entry_from(_e, _entry, _start)                  \
471         __bkey_extent_entry_for_each_from(_start,                       \
472                                 extent_entry_last(_e),_entry)
473
474 #define extent_for_each_entry(_e, _entry)                               \
475         extent_for_each_entry_from(_e, _entry, (_e).v->start)
476
477 #define extent_ptr_next(_e, _ptr)                                       \
478         __bkey_ptr_next(_ptr, extent_entry_last(_e))
479
480 #define extent_for_each_ptr(_e, _ptr)                                   \
481         __bkey_for_each_ptr(&(_e).v->start->ptr, extent_entry_last(_e), _ptr)
482
483 #define extent_crc_next(_e, _crc, _iter)                                \
484 ({                                                                      \
485         extent_for_each_entry_from(_e, _iter, _iter)                    \
486                 if (extent_entry_is_crc(_iter)) {                       \
487                         (_crc) = bch2_extent_crc_unpack((_e).k, entry_to_crc(_iter));\
488                         break;                                          \
489                 }                                                       \
490                                                                         \
491         (_iter) < extent_entry_last(_e);                                \
492 })
493
494 #define extent_for_each_crc(_e, _crc, _iter)                            \
495         for ((_crc) = bch2_extent_crc_unpack((_e).k, NULL),             \
496              (_iter) = (_e).v->start;                                   \
497              extent_crc_next(_e, _crc, _iter);                          \
498              (_iter) = extent_entry_next(_iter))
499
500 #define extent_for_each_ptr_decode(_e, _ptr, _entry)                    \
501         __bkey_for_each_ptr_decode((_e).k, (_e).v->start,               \
502                                    extent_entry_last(_e), _ptr, _entry)
503
504 void bch2_extent_crc_append(struct bkey_i_extent *,
505                             struct bch_extent_crc_unpacked);
506 void bch2_extent_ptr_decoded_append(struct bkey_i_extent *,
507                                     struct extent_ptr_decoded *);
508
509 static inline void __extent_entry_push(struct bkey_i_extent *e)
510 {
511         union bch_extent_entry *entry = extent_entry_last(extent_i_to_s(e));
512
513         EBUG_ON(bkey_val_u64s(&e->k) + extent_entry_u64s(entry) >
514                 BKEY_EXTENT_VAL_U64s_MAX);
515
516         e->k.u64s += extent_entry_u64s(entry);
517 }
518
519 bool bch2_can_narrow_extent_crcs(struct bkey_s_c_extent,
520                                  struct bch_extent_crc_unpacked);
521 bool bch2_extent_narrow_crcs(struct bkey_i_extent *, struct bch_extent_crc_unpacked);
522
523 union bch_extent_entry *bch2_bkey_drop_ptr(struct bkey_s,
524                                            struct bch_extent_ptr *);
525
526 #define bch2_bkey_drop_ptrs(_k, _ptr, _cond)                            \
527 do {                                                                    \
528         struct bkey_ptrs _ptrs = bch2_bkey_ptrs(_k);                    \
529                                                                         \
530         _ptr = &_ptrs.start->ptr;                                       \
531                                                                         \
532         while ((_ptr = bkey_ptr_next(_ptrs, _ptr))) {                   \
533                 if (_cond) {                                            \
534                         _ptr = (void *) bch2_bkey_drop_ptr(_k, _ptr);   \
535                         _ptrs = bch2_bkey_ptrs(_k);                     \
536                         continue;                                       \
537                 }                                                       \
538                                                                         \
539                 (_ptr)++;                                               \
540         }                                                               \
541 } while (0)
542
543 bool __bch2_cut_front(struct bpos, struct bkey_s);
544
545 static inline bool bch2_cut_front(struct bpos where, struct bkey_i *k)
546 {
547         return __bch2_cut_front(where, bkey_i_to_s(k));
548 }
549
550 bool bch2_cut_back(struct bpos, struct bkey *);
551 void bch2_key_resize(struct bkey *, unsigned);
552
553 /*
554  * In extent_sort_fix_overlapping(), insert_fixup_extent(),
555  * extent_merge_inline() - we're modifying keys in place that are packed. To do
556  * that we have to unpack the key, modify the unpacked key - then this
557  * copies/repacks the unpacked to the original as necessary.
558  */
559 static inline void extent_save(struct btree *b, struct bkey_packed *dst,
560                                struct bkey *src)
561 {
562         struct bkey_format *f = &b->format;
563         struct bkey_i *dst_unpacked;
564
565         if ((dst_unpacked = packed_to_bkey(dst)))
566                 dst_unpacked->k = *src;
567         else
568                 BUG_ON(!bch2_bkey_pack_key(dst, src, f));
569 }
570
571 bool bch2_check_range_allocated(struct bch_fs *, struct bpos, u64, unsigned);
572 unsigned bch2_bkey_nr_ptrs_allocated(struct bkey_s_c);
573
574 #endif /* _BCACHEFS_EXTENTS_H */