git.sesse.net Git - bcachefs-tools-debian/blob - libbcache/writeback.c

   1 /*
   2  * background writeback - scan btree for dirty data and write it to the backing
   3  * device
   4  *
   5  * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
   6  * Copyright 2012 Google, Inc.
   7  */
   8
   9 #include "bcache.h"
  10 #include "btree_update.h"
  11 #include "clock.h"
  12 #include "debug.h"
  13 #include "error.h"
  14 #include "extents.h"
  15 #include "io.h"
  16 #include "keybuf.h"
  17 #include "keylist.h"
  18 #include "writeback.h"
  19
  20 #include <linux/delay.h>
  21 #include <linux/freezer.h>
  22 #include <linux/kthread.h>
  23 #include <trace/events/bcache.h>
  24
  25 /* Rate limiting */
  26
  27 static void __update_writeback_rate(struct cached_dev *dc)
  28 {
  29         struct cache_set *c = dc->disk.c;
  30         u64 cache_dirty_target =
  31                 div_u64(c->capacity * dc->writeback_percent, 100);
  32         s64 target = div64_u64(cache_dirty_target *
  33                                bdev_sectors(dc->disk_sb.bdev),
  34                                c->cached_dev_sectors);
  35         s64 dirty = bcache_dev_sectors_dirty(&dc->disk);
  36
  37         bch_pd_controller_update(&dc->writeback_pd, target << 9,
  38                                  dirty << 9, -1);
  39 }
  40
  41 static void update_writeback_rate(struct work_struct *work)
  42 {
  43         struct cached_dev *dc = container_of(to_delayed_work(work),
  44                                              struct cached_dev,
  45                                              writeback_pd_update);
  46
  47         down_read(&dc->writeback_lock);
  48
  49         if (atomic_read(&dc->has_dirty) &&
  50             dc->writeback_percent &&
  51             !test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags))
  52                 __update_writeback_rate(dc);
  53         else
  54                 dc->writeback_pd.rate.rate = UINT_MAX;
  55
  56         up_read(&dc->writeback_lock);
  57
  58         schedule_delayed_work(&dc->writeback_pd_update,
  59                               dc->writeback_pd_update_seconds * HZ);
  60 }
  61
  62 struct dirty_io {
  63         struct closure          cl;
  64         struct bch_replace_info replace;
  65         struct cached_dev       *dc;
  66         struct cache            *ca;
  67         struct keybuf_key       *w;
  68         struct bch_extent_ptr   ptr;
  69         int                     error;
  70         bool                    from_mempool;
  71         /* Must be last */
  72         struct bio              bio;
  73 };
  74
  75 #define DIRTY_IO_MEMPOOL_BVECS          64
  76 #define DIRTY_IO_MEMPOOL_SECTORS        (DIRTY_IO_MEMPOOL_BVECS * PAGE_SECTORS)
  77
  78 static void dirty_init(struct dirty_io *io)
  79 {
  80         struct bio *bio = &io->bio;
  81
  82         bio_init(bio);
  83         if (!io->dc->writeback_percent)
  84                 bio_set_prio(bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
  85
  86         bio->bi_iter.bi_size    = io->replace.key.k.size << 9;
  87         bio->bi_max_vecs        =
  88                 DIV_ROUND_UP(io->replace.key.k.size, PAGE_SECTORS);
  89         bio->bi_io_vec          = bio->bi_inline_vecs;
  90         bch_bio_map(bio, NULL);
  91 }
  92
  93 static void dirty_io_destructor(struct closure *cl)
  94 {
  95         struct dirty_io *io = container_of(cl, struct dirty_io, cl);
  96
  97         if (io->from_mempool)
  98                 mempool_free(io, &io->dc->writeback_io_pool);
  99         else
 100                 kfree(io);
 101 }
 102
 103 static void write_dirty_finish(struct closure *cl)
 104 {
 105         struct dirty_io *io = container_of(cl, struct dirty_io, cl);
 106         struct cached_dev *dc = io->dc;
 107         struct bio_vec *bv;
 108         int i;
 109
 110         bio_for_each_segment_all(bv, &io->bio, i)
 111                 mempool_free(bv->bv_page, &dc->writeback_page_pool);
 112
 113         if (!io->error) {
 114                 BKEY_PADDED(k) tmp;
 115                 int ret;
 116
 117                 bkey_copy(&tmp.k, &io->replace.key);
 118                 io->replace.hook.fn = bch_extent_cmpxchg;
 119                 bkey_extent_set_cached(&tmp.k.k, true);
 120
 121                 ret = bch_btree_insert(dc->disk.c, BTREE_ID_EXTENTS, &tmp.k,
 122                                        NULL, &io->replace.hook, NULL, 0);
 123                 if (io->replace.successes == 0)
 124                         trace_bcache_writeback_collision(&io->replace.key.k);
 125
 126                 atomic_long_inc(ret
 127                                 ? &dc->disk.c->writeback_keys_failed
 128                                 : &dc->disk.c->writeback_keys_done);
 129         }
 130
 131         bch_keybuf_put(&dc->writeback_keys, io->w);
 132
 133         closure_return_with_destructor(cl, dirty_io_destructor);
 134 }
 135
 136 static void dirty_endio(struct bio *bio)
 137 {
 138         struct dirty_io *io = container_of(bio, struct dirty_io, bio);
 139
 140         if (bio->bi_error) {
 141                 trace_bcache_writeback_error(&io->replace.key.k,
 142                                              op_is_write(bio_op(&io->bio)),
 143                                              bio->bi_error);
 144                 io->error = bio->bi_error;
 145         }
 146
 147         closure_put(&io->cl);
 148 }
 149
 150 static void write_dirty(struct closure *cl)
 151 {
 152         struct dirty_io *io = container_of(cl, struct dirty_io, cl);
 153
 154         if (!io->error) {
 155                 dirty_init(io);
 156                 bio_set_op_attrs(&io->bio, REQ_OP_WRITE, 0);
 157                 io->bio.bi_iter.bi_sector =
 158                         bkey_start_offset(&io->replace.key.k);
 159                 io->bio.bi_bdev         = io->dc->disk_sb.bdev;
 160                 io->bio.bi_end_io       = dirty_endio;
 161
 162                 closure_bio_submit(&io->bio, cl);
 163         }
 164
 165         continue_at(cl, write_dirty_finish, io->dc->disk.c->wq);
 166 }
 167
 168 static void read_dirty_endio(struct bio *bio)
 169 {
 170         struct dirty_io *io = container_of(bio, struct dirty_io, bio);
 171
 172         cache_nonfatal_io_err_on(bio->bi_error, io->ca, "writeback read");
 173
 174         bch_account_io_completion(io->ca);
 175
 176         if (ptr_stale(io->ca, &io->ptr))
 177                 bio->bi_error = -EINTR;
 178
 179         dirty_endio(bio);
 180 }
 181
 182 static void read_dirty_submit(struct closure *cl)
 183 {
 184         struct dirty_io *io = container_of(cl, struct dirty_io, cl);
 185
 186         closure_bio_submit(&io->bio, cl);
 187
 188         continue_at(cl, write_dirty, system_freezable_wq);
 189 }
 190
 191 static u64 read_dirty(struct cached_dev *dc)
 192 {
 193         struct keybuf_key *w;
 194         struct dirty_io *io;
 195         struct closure cl;
 196         unsigned i;
 197         struct bio_vec *bv;
 198         u64 sectors_written = 0;
 199         BKEY_PADDED(k) tmp;
 200
 201         closure_init_stack(&cl);
 202
 203         while (!bch_ratelimit_wait_freezable_stoppable(&dc->writeback_pd.rate)) {
 204                 w = bch_keybuf_next(&dc->writeback_keys);
 205                 if (!w)
 206                         break;
 207
 208                 sectors_written += w->key.k.size;
 209                 bkey_copy(&tmp.k, &w->key);
 210
 211                 while (tmp.k.k.size) {
 212                         struct extent_pick_ptr pick;
 213
 214                         bch_extent_pick_ptr(dc->disk.c,
 215                                             bkey_i_to_s_c(&tmp.k),
 216                                             &pick);
 217                         if (IS_ERR_OR_NULL(pick.ca))
 218                                 break;
 219
 220                         io = kzalloc(sizeof(*io) + sizeof(struct bio_vec) *
 221                                      DIV_ROUND_UP(tmp.k.k.size,
 222                                                   PAGE_SECTORS),
 223                                      GFP_KERNEL);
 224                         if (!io) {
 225                                 trace_bcache_writeback_alloc_fail(pick.ca->set,
 226                                                                   tmp.k.k.size);
 227                                 io = mempool_alloc(&dc->writeback_io_pool,
 228                                                    GFP_KERNEL);
 229                                 memset(io, 0, sizeof(*io) +
 230                                        sizeof(struct bio_vec) *
 231                                        DIRTY_IO_MEMPOOL_BVECS);
 232                                 io->from_mempool = true;
 233
 234                                 bkey_copy(&io->replace.key, &tmp.k);
 235
 236                                 if (DIRTY_IO_MEMPOOL_SECTORS <
 237                                     io->replace.key.k.size)
 238                                         bch_key_resize(&io->replace.key.k,
 239                                                 DIRTY_IO_MEMPOOL_SECTORS);
 240                         } else {
 241                                 bkey_copy(&io->replace.key, &tmp.k);
 242                         }
 243
 244                         io->dc          = dc;
 245                         io->ca          = pick.ca;
 246                         io->w           = w;
 247                         io->ptr         = pick.ptr;
 248                         atomic_inc(&w->ref);
 249
 250                         dirty_init(io);
 251                         bio_set_op_attrs(&io->bio, REQ_OP_READ, 0);
 252                         io->bio.bi_iter.bi_sector = pick.ptr.offset;
 253                         io->bio.bi_bdev         = pick.ca->disk_sb.bdev;
 254                         io->bio.bi_end_io       = read_dirty_endio;
 255
 256                         bio_for_each_segment_all(bv, &io->bio, i) {
 257                                 bv->bv_page =
 258                                         mempool_alloc(&dc->writeback_page_pool,
 259                                                       i ? GFP_NOWAIT
 260                                                       : GFP_KERNEL);
 261                                 if (!bv->bv_page) {
 262                                         BUG_ON(!i);
 263                                         io->bio.bi_vcnt = i;
 264
 265                                         io->bio.bi_iter.bi_size =
 266                                                 io->bio.bi_vcnt * PAGE_SIZE;
 267
 268                                         bch_key_resize(&io->replace.key.k,
 269                                                        bio_sectors(&io->bio));
 270                                         break;
 271                                 }
 272                         }
 273
 274                         bch_cut_front(io->replace.key.k.p, &tmp.k);
 275                         trace_bcache_writeback(&io->replace.key.k);
 276
 277                         bch_ratelimit_increment(&dc->writeback_pd.rate,
 278                                                 io->replace.key.k.size << 9);
 279
 280                         closure_call(&io->cl, read_dirty_submit, NULL, &cl);
 281                 }
 282
 283                 bch_keybuf_put(&dc->writeback_keys, w);
 284         }
 285
 286         /*
 287          * Wait for outstanding writeback IOs to finish (and keybuf slots to be
 288          * freed) before refilling again
 289          */
 290         closure_sync(&cl);
 291
 292         return sectors_written;
 293 }
 294
 295 /* Scan for dirty data */
 296
 297 static void __bcache_dev_sectors_dirty_add(struct bcache_device *d,
 298                                            u64 offset, int nr_sectors)
 299 {
 300         unsigned stripe_offset, stripe, sectors_dirty;
 301
 302         if (!d)
 303                 return;
 304
 305         if (!d->stripe_sectors_dirty)
 306                 return;
 307
 308         stripe = offset_to_stripe(d, offset);
 309         stripe_offset = offset & (d->stripe_size - 1);
 310
 311         while (nr_sectors) {
 312                 int s = min_t(unsigned, abs(nr_sectors),
 313                               d->stripe_size - stripe_offset);
 314
 315                 if (nr_sectors < 0)
 316                         s = -s;
 317
 318                 if (stripe >= d->nr_stripes)
 319                         return;
 320
 321                 sectors_dirty = atomic_add_return(s,
 322                                         d->stripe_sectors_dirty + stripe);
 323                 if (sectors_dirty == d->stripe_size)
 324                         set_bit(stripe, d->full_dirty_stripes);
 325                 else
 326                         clear_bit(stripe, d->full_dirty_stripes);
 327
 328                 nr_sectors -= s;
 329                 stripe_offset = 0;
 330                 stripe++;
 331         }
 332 }
 333
 334 void bcache_dev_sectors_dirty_add(struct cache_set *c, unsigned inode,
 335                                   u64 offset, int nr_sectors)
 336 {
 337         struct bcache_device *d;
 338
 339         rcu_read_lock();
 340         d = bch_dev_find(c, inode);
 341         if (d)
 342                 __bcache_dev_sectors_dirty_add(d, offset, nr_sectors);
 343         rcu_read_unlock();
 344 }
 345
 346 static bool dirty_pred(struct keybuf *buf, struct bkey_s_c k)
 347 {
 348         struct cached_dev *dc = container_of(buf, struct cached_dev, writeback_keys);
 349
 350         BUG_ON(k.k->p.inode != bcache_dev_inum(&dc->disk));
 351
 352         return bkey_extent_is_data(k.k) &&
 353                 !bkey_extent_is_cached(k.k);
 354 }
 355
 356 static void refill_full_stripes(struct cached_dev *dc)
 357 {
 358         struct keybuf *buf = &dc->writeback_keys;
 359         unsigned inode = bcache_dev_inum(&dc->disk);
 360         unsigned start_stripe, stripe, next_stripe;
 361         bool wrapped = false;
 362
 363         stripe = offset_to_stripe(&dc->disk, buf->last_scanned.offset);
 364
 365         if (stripe >= dc->disk.nr_stripes)
 366                 stripe = 0;
 367
 368         start_stripe = stripe;
 369
 370         while (1) {
 371                 stripe = find_next_bit(dc->disk.full_dirty_stripes,
 372                                        dc->disk.nr_stripes, stripe);
 373
 374                 if (stripe == dc->disk.nr_stripes)
 375                         goto next;
 376
 377                 next_stripe = find_next_zero_bit(dc->disk.full_dirty_stripes,
 378                                                  dc->disk.nr_stripes, stripe);
 379
 380                 buf->last_scanned = POS(inode,
 381                                         stripe * dc->disk.stripe_size);
 382
 383                 bch_refill_keybuf(dc->disk.c, buf,
 384                                   POS(inode,
 385                                       next_stripe * dc->disk.stripe_size),
 386                                   dirty_pred);
 387
 388                 if (array_freelist_empty(&buf->freelist))
 389                         return;
 390
 391                 stripe = next_stripe;
 392 next:
 393                 if (wrapped && stripe > start_stripe)
 394                         return;
 395
 396                 if (stripe == dc->disk.nr_stripes) {
 397                         stripe = 0;
 398                         wrapped = true;
 399                 }
 400         }
 401 }
 402
 403 static u64 bch_writeback(struct cached_dev *dc)
 404 {
 405         struct keybuf *buf = &dc->writeback_keys;
 406         unsigned inode = bcache_dev_inum(&dc->disk);
 407         struct bpos start = POS(inode, 0);
 408         struct bpos end = POS(inode, KEY_OFFSET_MAX);
 409         struct bpos start_pos;
 410         u64 sectors_written = 0;
 411
 412         buf->last_scanned = POS(inode, 0);
 413
 414         while (bkey_cmp(buf->last_scanned, end) < 0 &&
 415                !kthread_should_stop()) {
 416                 down_write(&dc->writeback_lock);
 417
 418                 if (!atomic_read(&dc->has_dirty)) {
 419                         up_write(&dc->writeback_lock);
 420                         set_current_state(TASK_INTERRUPTIBLE);
 421
 422                         if (kthread_should_stop())
 423                                 return sectors_written;
 424
 425                         schedule();
 426                         try_to_freeze();
 427                         return sectors_written;
 428                 }
 429
 430                 if (bkey_cmp(buf->last_scanned, end) >= 0)
 431                         buf->last_scanned = POS(inode, 0);
 432
 433                 if (dc->partial_stripes_expensive) {
 434                         refill_full_stripes(dc);
 435                         if (array_freelist_empty(&buf->freelist))
 436                                 goto refill_done;
 437                 }
 438
 439                 start_pos = buf->last_scanned;
 440                 bch_refill_keybuf(dc->disk.c, buf, end, dirty_pred);
 441
 442                 if (bkey_cmp(buf->last_scanned, end) >= 0) {
 443                         /*
 444                          * If we get to the end start scanning again from the
 445                          * beginning, and only scan up to where we initially
 446                          * started scanning from:
 447                          */
 448                         buf->last_scanned = start;
 449                         bch_refill_keybuf(dc->disk.c, buf, start_pos,
 450                                           dirty_pred);
 451                 }
 452
 453                 if (RB_EMPTY_ROOT(&dc->writeback_keys.keys)) {
 454                         atomic_set(&dc->has_dirty, 0);
 455                         cached_dev_put(dc);
 456                         SET_BDEV_STATE(dc->disk_sb.sb, BDEV_STATE_CLEAN);
 457                         bch_write_bdev_super(dc, NULL);
 458                 }
 459
 460 refill_done:
 461                 up_write(&dc->writeback_lock);
 462
 463                 bch_ratelimit_reset(&dc->writeback_pd.rate);
 464                 sectors_written += read_dirty(dc);
 465         }
 466
 467         return sectors_written;
 468 }
 469
 470 static int bch_writeback_thread(void *arg)
 471 {
 472         struct cached_dev *dc = arg;
 473         struct cache_set *c = dc->disk.c;
 474         struct io_clock *clock = &c->io_clock[WRITE];
 475         unsigned long last;
 476         u64 sectors_written;
 477
 478         set_freezable();
 479
 480         while (!kthread_should_stop()) {
 481                 if (kthread_wait_freezable(dc->writeback_running ||
 482                                 test_bit(BCACHE_DEV_DETACHING,
 483                                          &dc->disk.flags)))
 484                         break;
 485
 486                 last = atomic_long_read(&clock->now);
 487
 488                 sectors_written = bch_writeback(dc);
 489
 490                 if (sectors_written < c->capacity >> 4)
 491                         bch_kthread_io_clock_wait(clock,
 492                                           last + (c->capacity >> 5));
 493         }
 494
 495         return 0;
 496 }
 497
 498 /**
 499  * bch_keylist_recalc_oldest_gens - update oldest_gen pointers from writeback keys
 500  *
 501  * This prevents us from wrapping around gens for a bucket only referenced from
 502  * writeback keybufs. We don't actually care that the data in those buckets is
 503  * marked live, only that we don't wrap the gens.
 504  */
 505 void bch_writeback_recalc_oldest_gens(struct cache_set *c)
 506 {
 507         struct radix_tree_iter iter;
 508         void **slot;
 509
 510         rcu_read_lock();
 511
 512         radix_tree_for_each_slot(slot, &c->devices, &iter, 0) {
 513                 struct bcache_device *d;
 514                 struct cached_dev *dc;
 515
 516                 d = radix_tree_deref_slot(slot);
 517
 518                 if (!CACHED_DEV(&d->inode.v))
 519                         continue;
 520                 dc = container_of(d, struct cached_dev, disk);
 521
 522                 bch_keybuf_recalc_oldest_gens(c, &dc->writeback_keys);
 523         }
 524
 525         rcu_read_unlock();
 526 }
 527
 528 /* Init */
 529
 530 void bch_sectors_dirty_init(struct cached_dev *dc, struct cache_set *c)
 531 {
 532         struct bcache_device *d = &dc->disk;
 533         struct btree_iter iter;
 534         struct bkey_s_c k;
 535
 536         /*
 537          * We have to do this before the disk is added to the radix tree or we
 538          * race with moving GC
 539          */
 540         for_each_btree_key(&iter, c, BTREE_ID_EXTENTS,
 541                            POS(bcache_dev_inum(d), 0), k) {
 542                 if (k.k->p.inode > bcache_dev_inum(d))
 543                         break;
 544
 545                 if (bkey_extent_is_data(k.k) &&
 546                     !bkey_extent_is_cached(k.k))
 547                         __bcache_dev_sectors_dirty_add(d,
 548                                                        bkey_start_offset(k.k),
 549                                                        k.k->size);
 550
 551                 bch_btree_iter_cond_resched(&iter);
 552         }
 553         bch_btree_iter_unlock(&iter);
 554
 555         dc->writeback_pd.last_actual = bcache_dev_sectors_dirty(d);
 556 }
 557
 558 void bch_cached_dev_writeback_stop(struct cached_dev *dc)
 559 {
 560         cancel_delayed_work_sync(&dc->writeback_pd_update);
 561         if (!IS_ERR_OR_NULL(dc->writeback_thread)) {
 562                 kthread_stop(dc->writeback_thread);
 563                 dc->writeback_thread = NULL;
 564         }
 565 }
 566
 567 void bch_cached_dev_writeback_free(struct cached_dev *dc)
 568 {
 569         struct bcache_device *d = &dc->disk;
 570
 571         mempool_exit(&dc->writeback_page_pool);
 572         mempool_exit(&dc->writeback_io_pool);
 573         kvfree(d->full_dirty_stripes);
 574         kvfree(d->stripe_sectors_dirty);
 575 }
 576
 577 int bch_cached_dev_writeback_init(struct cached_dev *dc)
 578 {
 579         struct bcache_device *d = &dc->disk;
 580         sector_t sectors;
 581         size_t n;
 582
 583         sectors = get_capacity(dc->disk.disk);
 584
 585         if (!d->stripe_size) {
 586 #ifdef CONFIG_BCACHE_DEBUG
 587                 d->stripe_size = 1 << 0;
 588 #else
 589                 d->stripe_size = 1 << 31;
 590 #endif
 591         }
 592
 593         pr_debug("stripe size: %d sectors", d->stripe_size);
 594         d->nr_stripes = DIV_ROUND_UP_ULL(sectors, d->stripe_size);
 595
 596         if (!d->nr_stripes ||
 597             d->nr_stripes > INT_MAX ||
 598             d->nr_stripes > SIZE_MAX / sizeof(atomic_t)) {
 599                 pr_err("nr_stripes too large or invalid: %u (start sector beyond end of disk?)",
 600                         (unsigned)d->nr_stripes);
 601                 return -ENOMEM;
 602         }
 603
 604         n = d->nr_stripes * sizeof(atomic_t);
 605         d->stripe_sectors_dirty = n < PAGE_SIZE << 6
 606                 ? kzalloc(n, GFP_KERNEL)
 607                 : vzalloc(n);
 608         if (!d->stripe_sectors_dirty) {
 609                 pr_err("cannot allocate stripe_sectors_dirty");
 610                 return -ENOMEM;
 611         }
 612
 613         n = BITS_TO_LONGS(d->nr_stripes) * sizeof(unsigned long);
 614         d->full_dirty_stripes = n < PAGE_SIZE << 6
 615                 ? kzalloc(n, GFP_KERNEL)
 616                 : vzalloc(n);
 617         if (!d->full_dirty_stripes) {
 618                 pr_err("cannot allocate full_dirty_stripes");
 619                 return -ENOMEM;
 620         }
 621
 622         if (mempool_init_kmalloc_pool(&dc->writeback_io_pool, 4,
 623                                       sizeof(struct dirty_io) +
 624                                       sizeof(struct bio_vec) *
 625                                       DIRTY_IO_MEMPOOL_BVECS) ||
 626             mempool_init_page_pool(&dc->writeback_page_pool,
 627                                    (64 << 10) / PAGE_SIZE, 0))
 628                 return -ENOMEM;
 629
 630         init_rwsem(&dc->writeback_lock);
 631         bch_keybuf_init(&dc->writeback_keys);
 632
 633         dc->writeback_metadata          = true;
 634         dc->writeback_running           = true;
 635         dc->writeback_percent           = 10;
 636         dc->writeback_pd_update_seconds = 5;
 637
 638         bch_pd_controller_init(&dc->writeback_pd);
 639         INIT_DELAYED_WORK(&dc->writeback_pd_update, update_writeback_rate);
 640
 641         return 0;
 642 }
 643
 644 int bch_cached_dev_writeback_start(struct cached_dev *dc)
 645 {
 646         dc->writeback_thread = kthread_create(bch_writeback_thread, dc,
 647                                               "bcache_writeback");
 648         if (IS_ERR(dc->writeback_thread))
 649                 return PTR_ERR(dc->writeback_thread);
 650
 651         schedule_delayed_work(&dc->writeback_pd_update,
 652                               dc->writeback_pd_update_seconds * HZ);
 653
 654         bch_writeback_queue(dc);
 655
 656         return 0;
 657 }