git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/journal_reclaim.c

   1
   2 #include "bcachefs.h"
   3 #include "journal.h"
   4 #include "journal_reclaim.h"
   5 #include "replicas.h"
   6 #include "super.h"
   7
   8 /*
   9  * Journal entry pinning - machinery for holding a reference on a given journal
  10  * entry, holding it open to ensure it gets replayed during recovery:
  11  */
  12
  13 static inline void __journal_pin_add(struct journal *j,
  14                                      u64 seq,
  15                                      struct journal_entry_pin *pin,
  16                                      journal_pin_flush_fn flush_fn)
  17 {
  18         struct journal_entry_pin_list *pin_list = journal_seq_pin(j, seq);
  19
  20         BUG_ON(journal_pin_active(pin));
  21         BUG_ON(!atomic_read(&pin_list->count));
  22
  23         atomic_inc(&pin_list->count);
  24         pin->seq        = seq;
  25         pin->flush      = flush_fn;
  26
  27         if (flush_fn)
  28                 list_add(&pin->list, &pin_list->list);
  29         else
  30                 INIT_LIST_HEAD(&pin->list);
  31
  32         /*
  33          * If the journal is currently full,  we might want to call flush_fn
  34          * immediately:
  35          */
  36         journal_wake(j);
  37 }
  38
  39 void bch2_journal_pin_add(struct journal *j, u64 seq,
  40                           struct journal_entry_pin *pin,
  41                           journal_pin_flush_fn flush_fn)
  42 {
  43         spin_lock(&j->lock);
  44         __journal_pin_add(j, seq, pin, flush_fn);
  45         spin_unlock(&j->lock);
  46 }
  47
  48 static inline void __journal_pin_drop(struct journal *j,
  49                                       struct journal_entry_pin *pin)
  50 {
  51         struct journal_entry_pin_list *pin_list;
  52
  53         if (!journal_pin_active(pin))
  54                 return;
  55
  56         pin_list = journal_seq_pin(j, pin->seq);
  57         pin->seq = 0;
  58         list_del_init(&pin->list);
  59
  60         /*
  61          * Unpinning a journal entry make make journal_next_bucket() succeed, if
  62          * writing a new last_seq will now make another bucket available:
  63          */
  64         if (atomic_dec_and_test(&pin_list->count) &&
  65             pin_list == &fifo_peek_front(&j->pin))
  66                 bch2_journal_reclaim_fast(j);
  67 }
  68
  69 void bch2_journal_pin_drop(struct journal *j,
  70                            struct journal_entry_pin *pin)
  71 {
  72         spin_lock(&j->lock);
  73         __journal_pin_drop(j, pin);
  74         spin_unlock(&j->lock);
  75 }
  76
  77 void bch2_journal_pin_update(struct journal *j, u64 seq,
  78                              struct journal_entry_pin *pin,
  79                              journal_pin_flush_fn flush_fn)
  80 {
  81         spin_lock(&j->lock);
  82
  83         if (pin->seq != seq) {
  84                 __journal_pin_drop(j, pin);
  85                 __journal_pin_add(j, seq, pin, flush_fn);
  86         } else {
  87                 struct journal_entry_pin_list *pin_list =
  88                         journal_seq_pin(j, seq);
  89
  90                 list_move(&pin->list, &pin_list->list);
  91         }
  92
  93         spin_unlock(&j->lock);
  94 }
  95
  96 void bch2_journal_pin_add_if_older(struct journal *j,
  97                                   struct journal_entry_pin *src_pin,
  98                                   struct journal_entry_pin *pin,
  99                                   journal_pin_flush_fn flush_fn)
 100 {
 101         spin_lock(&j->lock);
 102
 103         if (journal_pin_active(src_pin) &&
 104             (!journal_pin_active(pin) ||
 105              src_pin->seq < pin->seq)) {
 106                 __journal_pin_drop(j, pin);
 107                 __journal_pin_add(j, src_pin->seq, pin, flush_fn);
 108         }
 109
 110         spin_unlock(&j->lock);
 111 }
 112
 113 void bch2_journal_pin_flush(struct journal *j, struct journal_entry_pin *pin)
 114 {
 115         BUG_ON(journal_pin_active(pin));
 116
 117         wait_event(j->pin_flush_wait, j->flush_in_progress != pin);
 118 }
 119
 120 /*
 121  * Journal reclaim: flush references to open journal entries to reclaim space in
 122  * the journal
 123  *
 124  * May be done by the journal code in the background as needed to free up space
 125  * for more journal entries, or as part of doing a clean shutdown, or to migrate
 126  * data off of a specific device:
 127  */
 128
 129 /**
 130  * bch2_journal_reclaim_fast - do the fast part of journal reclaim
 131  *
 132  * Called from IO submission context, does not block. Cleans up after btree
 133  * write completions by advancing the journal pin and each cache's last_idx,
 134  * kicking off discards and background reclaim as necessary.
 135  */
 136 void bch2_journal_reclaim_fast(struct journal *j)
 137 {
 138         struct journal_entry_pin_list temp;
 139         bool popped = false;
 140
 141         lockdep_assert_held(&j->lock);
 142
 143         /*
 144          * Unpin journal entries whose reference counts reached zero, meaning
 145          * all btree nodes got written out
 146          */
 147         while (!fifo_empty(&j->pin) &&
 148                !atomic_read(&fifo_peek_front(&j->pin).count)) {
 149                 BUG_ON(!list_empty(&fifo_peek_front(&j->pin).list));
 150                 BUG_ON(!fifo_pop(&j->pin, temp));
 151                 popped = true;
 152         }
 153
 154         if (popped)
 155                 journal_wake(j);
 156 }
 157
 158 static void journal_pin_mark_flushing(struct journal *j,
 159                                       struct journal_entry_pin *pin,
 160                                       u64 seq)
 161 {
 162         lockdep_assert_held(&j->reclaim_lock);
 163
 164         list_move(&pin->list, &journal_seq_pin(j, seq)->flushed);
 165         BUG_ON(j->flush_in_progress);
 166         j->flush_in_progress = pin;
 167 }
 168
 169 static void journal_pin_flush(struct journal *j,
 170                               struct journal_entry_pin *pin,
 171                               u64 seq)
 172 {
 173         pin->flush(j, pin, seq);
 174
 175         BUG_ON(j->flush_in_progress != pin);
 176         j->flush_in_progress = NULL;
 177         wake_up(&j->pin_flush_wait);
 178 }
 179
 180 static struct journal_entry_pin *
 181 journal_get_next_pin(struct journal *j, u64 seq_to_flush, u64 *seq)
 182 {
 183         struct journal_entry_pin_list *pin_list;
 184         struct journal_entry_pin *ret = NULL;
 185
 186         /* no need to iterate over empty fifo entries: */
 187         bch2_journal_reclaim_fast(j);
 188
 189         fifo_for_each_entry_ptr(pin_list, &j->pin, *seq)
 190                 if (*seq > seq_to_flush ||
 191                     (ret = list_first_entry_or_null(&pin_list->list,
 192                                 struct journal_entry_pin, list)))
 193                         break;
 194
 195         return ret;
 196 }
 197
 198 static bool should_discard_bucket(struct journal *j, struct journal_device *ja)
 199 {
 200         bool ret;
 201
 202         spin_lock(&j->lock);
 203         ret = ja->nr &&
 204                 (ja->last_idx != ja->cur_idx &&
 205                  ja->bucket_seq[ja->last_idx] < j->last_seq_ondisk);
 206         spin_unlock(&j->lock);
 207
 208         return ret;
 209 }
 210
 211 /**
 212  * bch2_journal_reclaim_work - free up journal buckets
 213  *
 214  * Background journal reclaim writes out btree nodes. It should be run
 215  * early enough so that we never completely run out of journal buckets.
 216  *
 217  * High watermarks for triggering background reclaim:
 218  * - FIFO has fewer than 512 entries left
 219  * - fewer than 25% journal buckets free
 220  *
 221  * Background reclaim runs until low watermarks are reached:
 222  * - FIFO has more than 1024 entries left
 223  * - more than 50% journal buckets free
 224  *
 225  * As long as a reclaim can complete in the time it takes to fill up
 226  * 512 journal entries or 25% of all journal buckets, then
 227  * journal_next_bucket() should not stall.
 228  */
 229 void bch2_journal_reclaim_work(struct work_struct *work)
 230 {
 231         struct bch_fs *c = container_of(to_delayed_work(work),
 232                                 struct bch_fs, journal.reclaim_work);
 233         struct journal *j = &c->journal;
 234         struct bch_dev *ca;
 235         struct journal_entry_pin *pin;
 236         u64 seq, seq_to_flush = 0;
 237         unsigned iter, bucket_to_flush;
 238         unsigned long next_flush;
 239         bool reclaim_lock_held = false, need_flush;
 240
 241         /*
 242          * Advance last_idx to point to the oldest journal entry containing
 243          * btree node updates that have not yet been written out
 244          */
 245         for_each_rw_member(ca, c, iter) {
 246                 struct journal_device *ja = &ca->journal;
 247
 248                 if (!ja->nr)
 249                         continue;
 250
 251                 while (should_discard_bucket(j, ja)) {
 252                         if (!reclaim_lock_held) {
 253                                 /*
 254                                  * ugh:
 255                                  * might be called from __journal_res_get()
 256                                  * under wait_event() - have to go back to
 257                                  * TASK_RUNNING before doing something that
 258                                  * would block, but only if we're doing work:
 259                                  */
 260                                 __set_current_state(TASK_RUNNING);
 261
 262                                 mutex_lock(&j->reclaim_lock);
 263                                 reclaim_lock_held = true;
 264                                 /* recheck under reclaim_lock: */
 265                                 continue;
 266                         }
 267
 268                         if (ca->mi.discard &&
 269                             blk_queue_discard(bdev_get_queue(ca->disk_sb.bdev)))
 270                                 blkdev_issue_discard(ca->disk_sb.bdev,
 271                                         bucket_to_sector(ca,
 272                                                 ja->buckets[ja->last_idx]),
 273                                         ca->mi.bucket_size, GFP_NOIO, 0);
 274
 275                         spin_lock(&j->lock);
 276                         ja->last_idx = (ja->last_idx + 1) % ja->nr;
 277                         spin_unlock(&j->lock);
 278
 279                         journal_wake(j);
 280                 }
 281
 282                 /*
 283                  * Write out enough btree nodes to free up 50% journal
 284                  * buckets
 285                  */
 286                 spin_lock(&j->lock);
 287                 bucket_to_flush = (ja->cur_idx + (ja->nr >> 1)) % ja->nr;
 288                 seq_to_flush = max_t(u64, seq_to_flush,
 289                                      ja->bucket_seq[bucket_to_flush]);
 290                 spin_unlock(&j->lock);
 291         }
 292
 293         /* Also flush if the pin fifo is more than half full */
 294         spin_lock(&j->lock);
 295         seq_to_flush = max_t(s64, seq_to_flush,
 296                              (s64) journal_cur_seq(j) -
 297                              (j->pin.size >> 1));
 298
 299         /*
 300          * If it's been longer than j->reclaim_delay_ms since we last flushed,
 301          * make sure to flush at least one journal pin:
 302          */
 303         next_flush = j->last_flushed + msecs_to_jiffies(j->reclaim_delay_ms);
 304         need_flush = time_after(jiffies, next_flush);
 305
 306         while ((pin = journal_get_next_pin(j, need_flush
 307                                            ? U64_MAX
 308                                            : seq_to_flush, &seq))) {
 309                 if (!reclaim_lock_held) {
 310                         spin_unlock(&j->lock);
 311                         __set_current_state(TASK_RUNNING);
 312                         mutex_lock(&j->reclaim_lock);
 313                         reclaim_lock_held = true;
 314                         spin_lock(&j->lock);
 315                         continue;
 316                 }
 317
 318                 journal_pin_mark_flushing(j, pin, seq);
 319                 spin_unlock(&j->lock);
 320
 321                 journal_pin_flush(j, pin, seq);
 322
 323                 need_flush = false;
 324                 j->last_flushed = jiffies;
 325
 326                 spin_lock(&j->lock);
 327         }
 328
 329         spin_unlock(&j->lock);
 330
 331         if (reclaim_lock_held)
 332                 mutex_unlock(&j->reclaim_lock);
 333
 334         if (!test_bit(BCH_FS_RO, &c->flags))
 335                 queue_delayed_work(system_freezable_wq, &j->reclaim_work,
 336                                    msecs_to_jiffies(j->reclaim_delay_ms));
 337 }
 338
 339 static int journal_flush_done(struct journal *j, u64 seq_to_flush,
 340                               struct journal_entry_pin **pin,
 341                               u64 *pin_seq)
 342 {
 343         int ret;
 344
 345         *pin = NULL;
 346
 347         ret = bch2_journal_error(j);
 348         if (ret)
 349                 return ret;
 350
 351         spin_lock(&j->lock);
 352         /*
 353          * If journal replay hasn't completed, the unreplayed journal entries
 354          * hold refs on their corresponding sequence numbers
 355          */
 356         ret = (*pin = journal_get_next_pin(j, seq_to_flush, pin_seq)) != NULL ||
 357                 !test_bit(JOURNAL_REPLAY_DONE, &j->flags) ||
 358                 journal_last_seq(j) > seq_to_flush ||
 359                 (fifo_used(&j->pin) == 1 &&
 360                  atomic_read(&fifo_peek_front(&j->pin).count) == 1);
 361         if (*pin)
 362                 journal_pin_mark_flushing(j, *pin, *pin_seq);
 363
 364         spin_unlock(&j->lock);
 365
 366         return ret;
 367 }
 368
 369 void bch2_journal_flush_pins(struct journal *j, u64 seq_to_flush)
 370 {
 371         struct journal_entry_pin *pin;
 372         u64 pin_seq;
 373
 374         if (!test_bit(JOURNAL_STARTED, &j->flags))
 375                 return;
 376
 377         mutex_lock(&j->reclaim_lock);
 378
 379         while (1) {
 380                 wait_event(j->wait, journal_flush_done(j, seq_to_flush,
 381                                                        &pin, &pin_seq));
 382                 if (!pin)
 383                         break;
 384
 385                 journal_pin_flush(j, pin, pin_seq);
 386         }
 387
 388         mutex_unlock(&j->reclaim_lock);
 389 }
 390
 391 int bch2_journal_flush_device_pins(struct journal *j, int dev_idx)
 392 {
 393         struct bch_fs *c = container_of(j, struct bch_fs, journal);
 394         struct journal_entry_pin_list *p;
 395         struct bch_devs_list devs;
 396         u64 iter, seq = 0;
 397         int ret = 0;
 398
 399         spin_lock(&j->lock);
 400         fifo_for_each_entry_ptr(p, &j->pin, iter)
 401                 if (dev_idx >= 0
 402                     ? bch2_dev_list_has_dev(p->devs, dev_idx)
 403                     : p->devs.nr < c->opts.metadata_replicas)
 404                         seq = iter;
 405         spin_unlock(&j->lock);
 406
 407         bch2_journal_flush_pins(j, seq);
 408
 409         ret = bch2_journal_error(j);
 410         if (ret)
 411                 return ret;
 412
 413         mutex_lock(&c->replicas_gc_lock);
 414         bch2_replicas_gc_start(c, 1 << BCH_DATA_JOURNAL);
 415
 416         seq = 0;
 417
 418         spin_lock(&j->lock);
 419         while (!ret && seq < j->pin.back) {
 420                 seq = max(seq, journal_last_seq(j));
 421                 devs = journal_seq_pin(j, seq)->devs;
 422                 seq++;
 423
 424                 spin_unlock(&j->lock);
 425                 ret = bch2_mark_replicas(c, BCH_DATA_JOURNAL, devs);
 426                 spin_lock(&j->lock);
 427         }
 428         spin_unlock(&j->lock);
 429
 430         ret = bch2_replicas_gc_end(c, ret);
 431         mutex_unlock(&c->replicas_gc_lock);
 432
 433         return ret;
 434 }