git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/fs-io-buffered.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #ifndef NO_BCACHEFS_FS
   3
   4 #include "bcachefs.h"
   5 #include "alloc_foreground.h"
   6 #include "bkey_buf.h"
   7 #include "fs-io.h"
   8 #include "fs-io-buffered.h"
   9 #include "fs-io-direct.h"
  10 #include "fs-io-pagecache.h"
  11 #include "io_read.h"
  12 #include "io_write.h"
  13
  14 #include <linux/backing-dev.h>
  15 #include <linux/pagemap.h>
  16 #include <linux/writeback.h>
  17
  18 static inline bool bio_full(struct bio *bio, unsigned len)
  19 {
  20         if (bio->bi_vcnt >= bio->bi_max_vecs)
  21                 return true;
  22         if (bio->bi_iter.bi_size > UINT_MAX - len)
  23                 return true;
  24         return false;
  25 }
  26
  27 /* readpage(s): */
  28
  29 static void bch2_readpages_end_io(struct bio *bio)
  30 {
  31         struct folio_iter fi;
  32
  33         bio_for_each_folio_all(fi, bio) {
  34                 if (!bio->bi_status) {
  35                         folio_mark_uptodate(fi.folio);
  36                 } else {
  37                         folio_clear_uptodate(fi.folio);
  38                         folio_set_error(fi.folio);
  39                 }
  40                 folio_unlock(fi.folio);
  41         }
  42
  43         bio_put(bio);
  44 }
  45
  46 struct readpages_iter {
  47         struct address_space    *mapping;
  48         unsigned                idx;
  49         folios                  folios;
  50 };
  51
  52 static int readpages_iter_init(struct readpages_iter *iter,
  53                                struct readahead_control *ractl)
  54 {
  55         struct folio **fi;
  56         int ret;
  57
  58         memset(iter, 0, sizeof(*iter));
  59
  60         iter->mapping = ractl->mapping;
  61
  62         ret = bch2_filemap_get_contig_folios_d(iter->mapping,
  63                                 ractl->_index << PAGE_SHIFT,
  64                                 (ractl->_index + ractl->_nr_pages) << PAGE_SHIFT,
  65                                 0, mapping_gfp_mask(iter->mapping),
  66                                 &iter->folios);
  67         if (ret)
  68                 return ret;
  69
  70         darray_for_each(iter->folios, fi) {
  71                 ractl->_nr_pages -= 1U << folio_order(*fi);
  72                 __bch2_folio_create(*fi, __GFP_NOFAIL|GFP_KERNEL);
  73                 folio_put(*fi);
  74                 folio_put(*fi);
  75         }
  76
  77         return 0;
  78 }
  79
  80 static inline struct folio *readpage_iter_peek(struct readpages_iter *iter)
  81 {
  82         if (iter->idx >= iter->folios.nr)
  83                 return NULL;
  84         return iter->folios.data[iter->idx];
  85 }
  86
  87 static inline void readpage_iter_advance(struct readpages_iter *iter)
  88 {
  89         iter->idx++;
  90 }
  91
  92 static bool extent_partial_reads_expensive(struct bkey_s_c k)
  93 {
  94         struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
  95         struct bch_extent_crc_unpacked crc;
  96         const union bch_extent_entry *i;
  97
  98         bkey_for_each_crc(k.k, ptrs, crc, i)
  99                 if (crc.csum_type || crc.compression_type)
 100                         return true;
 101         return false;
 102 }
 103
 104 static int readpage_bio_extend(struct btree_trans *trans,
 105                                struct readpages_iter *iter,
 106                                struct bio *bio,
 107                                unsigned sectors_this_extent,
 108                                bool get_more)
 109 {
 110         /* Don't hold btree locks while allocating memory: */
 111         bch2_trans_unlock(trans);
 112
 113         while (bio_sectors(bio) < sectors_this_extent &&
 114                bio->bi_vcnt < bio->bi_max_vecs) {
 115                 struct folio *folio = readpage_iter_peek(iter);
 116                 int ret;
 117
 118                 if (folio) {
 119                         readpage_iter_advance(iter);
 120                 } else {
 121                         pgoff_t folio_offset = bio_end_sector(bio) >> PAGE_SECTORS_SHIFT;
 122
 123                         if (!get_more)
 124                                 break;
 125
 126                         folio = xa_load(&iter->mapping->i_pages, folio_offset);
 127                         if (folio && !xa_is_value(folio))
 128                                 break;
 129
 130                         folio = filemap_alloc_folio(readahead_gfp_mask(iter->mapping), 0);
 131                         if (!folio)
 132                                 break;
 133
 134                         if (!__bch2_folio_create(folio, GFP_KERNEL)) {
 135                                 folio_put(folio);
 136                                 break;
 137                         }
 138
 139                         ret = filemap_add_folio(iter->mapping, folio, folio_offset, GFP_KERNEL);
 140                         if (ret) {
 141                                 __bch2_folio_release(folio);
 142                                 folio_put(folio);
 143                                 break;
 144                         }
 145
 146                         folio_put(folio);
 147                 }
 148
 149                 BUG_ON(folio_sector(folio) != bio_end_sector(bio));
 150
 151                 BUG_ON(!bio_add_folio(bio, folio, folio_size(folio), 0));
 152         }
 153
 154         return bch2_trans_relock(trans);
 155 }
 156
 157 static void bchfs_read(struct btree_trans *trans,
 158                        struct bch_read_bio *rbio,
 159                        subvol_inum inum,
 160                        struct readpages_iter *readpages_iter)
 161 {
 162         struct bch_fs *c = trans->c;
 163         struct btree_iter iter;
 164         struct bkey_buf sk;
 165         int flags = BCH_READ_RETRY_IF_STALE|
 166                 BCH_READ_MAY_PROMOTE;
 167         u32 snapshot;
 168         int ret = 0;
 169
 170         rbio->c = c;
 171         rbio->start_time = local_clock();
 172         rbio->subvol = inum.subvol;
 173
 174         bch2_bkey_buf_init(&sk);
 175 retry:
 176         bch2_trans_begin(trans);
 177         iter = (struct btree_iter) { NULL };
 178
 179         ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot);
 180         if (ret)
 181                 goto err;
 182
 183         bch2_trans_iter_init(trans, &iter, BTREE_ID_extents,
 184                              SPOS(inum.inum, rbio->bio.bi_iter.bi_sector, snapshot),
 185                              BTREE_ITER_SLOTS);
 186         while (1) {
 187                 struct bkey_s_c k;
 188                 unsigned bytes, sectors, offset_into_extent;
 189                 enum btree_id data_btree = BTREE_ID_extents;
 190
 191                 /*
 192                  * read_extent -> io_time_reset may cause a transaction restart
 193                  * without returning an error, we need to check for that here:
 194                  */
 195                 ret = bch2_trans_relock(trans);
 196                 if (ret)
 197                         break;
 198
 199                 bch2_btree_iter_set_pos(&iter,
 200                                 POS(inum.inum, rbio->bio.bi_iter.bi_sector));
 201
 202                 k = bch2_btree_iter_peek_slot(&iter);
 203                 ret = bkey_err(k);
 204                 if (ret)
 205                         break;
 206
 207                 offset_into_extent = iter.pos.offset -
 208                         bkey_start_offset(k.k);
 209                 sectors = k.k->size - offset_into_extent;
 210
 211                 bch2_bkey_buf_reassemble(&sk, c, k);
 212
 213                 ret = bch2_read_indirect_extent(trans, &data_btree,
 214                                         &offset_into_extent, &sk);
 215                 if (ret)
 216                         break;
 217
 218                 k = bkey_i_to_s_c(sk.k);
 219
 220                 sectors = min(sectors, k.k->size - offset_into_extent);
 221
 222                 if (readpages_iter) {
 223                         ret = readpage_bio_extend(trans, readpages_iter, &rbio->bio, sectors,
 224                                                   extent_partial_reads_expensive(k));
 225                         if (ret)
 226                                 break;
 227                 }
 228
 229                 bytes = min(sectors, bio_sectors(&rbio->bio)) << 9;
 230                 swap(rbio->bio.bi_iter.bi_size, bytes);
 231
 232                 if (rbio->bio.bi_iter.bi_size == bytes)
 233                         flags |= BCH_READ_LAST_FRAGMENT;
 234
 235                 bch2_bio_page_state_set(&rbio->bio, k);
 236
 237                 bch2_read_extent(trans, rbio, iter.pos,
 238                                  data_btree, k, offset_into_extent, flags);
 239
 240                 if (flags & BCH_READ_LAST_FRAGMENT)
 241                         break;
 242
 243                 swap(rbio->bio.bi_iter.bi_size, bytes);
 244                 bio_advance(&rbio->bio, bytes);
 245
 246                 ret = btree_trans_too_many_iters(trans);
 247                 if (ret)
 248                         break;
 249         }
 250 err:
 251         bch2_trans_iter_exit(trans, &iter);
 252
 253         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
 254                 goto retry;
 255
 256         if (ret) {
 257                 bch_err_inum_offset_ratelimited(c,
 258                                 iter.pos.inode,
 259                                 iter.pos.offset << 9,
 260                                 "read error %i from btree lookup", ret);
 261                 rbio->bio.bi_status = BLK_STS_IOERR;
 262                 bio_endio(&rbio->bio);
 263         }
 264
 265         bch2_bkey_buf_exit(&sk, c);
 266 }
 267
 268 void bch2_readahead(struct readahead_control *ractl)
 269 {
 270         struct bch_inode_info *inode = to_bch_ei(ractl->mapping->host);
 271         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 272         struct bch_io_opts opts;
 273         struct btree_trans *trans = bch2_trans_get(c);
 274         struct folio *folio;
 275         struct readpages_iter readpages_iter;
 276         int ret;
 277
 278         bch2_inode_opts_get(&opts, c, &inode->ei_inode);
 279
 280         ret = readpages_iter_init(&readpages_iter, ractl);
 281         BUG_ON(ret);
 282
 283         bch2_pagecache_add_get(inode);
 284
 285         while ((folio = readpage_iter_peek(&readpages_iter))) {
 286                 unsigned n = min_t(unsigned,
 287                                    readpages_iter.folios.nr -
 288                                    readpages_iter.idx,
 289                                    BIO_MAX_VECS);
 290                 struct bch_read_bio *rbio =
 291                         rbio_init(bio_alloc_bioset(NULL, n, REQ_OP_READ,
 292                                                    GFP_KERNEL, &c->bio_read),
 293                                   opts);
 294
 295                 readpage_iter_advance(&readpages_iter);
 296
 297                 rbio->bio.bi_iter.bi_sector = folio_sector(folio);
 298                 rbio->bio.bi_end_io = bch2_readpages_end_io;
 299                 BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0));
 300
 301                 bchfs_read(trans, rbio, inode_inum(inode),
 302                            &readpages_iter);
 303                 bch2_trans_unlock(trans);
 304         }
 305
 306         bch2_pagecache_add_put(inode);
 307
 308         bch2_trans_put(trans);
 309         darray_exit(&readpages_iter.folios);
 310 }
 311
 312 static void __bchfs_readfolio(struct bch_fs *c, struct bch_read_bio *rbio,
 313                              subvol_inum inum, struct folio *folio)
 314 {
 315         bch2_folio_create(folio, __GFP_NOFAIL);
 316
 317         rbio->bio.bi_opf = REQ_OP_READ|REQ_SYNC;
 318         rbio->bio.bi_iter.bi_sector = folio_sector(folio);
 319         BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0));
 320
 321         bch2_trans_run(c, (bchfs_read(trans, rbio, inum, NULL), 0));
 322 }
 323
 324 static void bch2_read_single_folio_end_io(struct bio *bio)
 325 {
 326         complete(bio->bi_private);
 327 }
 328
 329 int bch2_read_single_folio(struct folio *folio, struct address_space *mapping)
 330 {
 331         struct bch_inode_info *inode = to_bch_ei(mapping->host);
 332         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 333         struct bch_read_bio *rbio;
 334         struct bch_io_opts opts;
 335         int ret;
 336         DECLARE_COMPLETION_ONSTACK(done);
 337
 338         bch2_inode_opts_get(&opts, c, &inode->ei_inode);
 339
 340         rbio = rbio_init(bio_alloc_bioset(NULL, 1, REQ_OP_READ, GFP_KERNEL, &c->bio_read),
 341                          opts);
 342         rbio->bio.bi_private = &done;
 343         rbio->bio.bi_end_io = bch2_read_single_folio_end_io;
 344
 345         __bchfs_readfolio(c, rbio, inode_inum(inode), folio);
 346         wait_for_completion(&done);
 347
 348         ret = blk_status_to_errno(rbio->bio.bi_status);
 349         bio_put(&rbio->bio);
 350
 351         if (ret < 0)
 352                 return ret;
 353
 354         folio_mark_uptodate(folio);
 355         return 0;
 356 }
 357
 358 int bch2_read_folio(struct file *file, struct folio *folio)
 359 {
 360         int ret;
 361
 362         ret = bch2_read_single_folio(folio, folio->mapping);
 363         folio_unlock(folio);
 364         return bch2_err_class(ret);
 365 }
 366
 367 /* writepages: */
 368
 369 struct bch_writepage_io {
 370         struct bch_inode_info           *inode;
 371
 372         /* must be last: */
 373         struct bch_write_op             op;
 374 };
 375
 376 struct bch_writepage_state {
 377         struct bch_writepage_io *io;
 378         struct bch_io_opts      opts;
 379         struct bch_folio_sector *tmp;
 380         unsigned                tmp_sectors;
 381 };
 382
 383 static inline struct bch_writepage_state bch_writepage_state_init(struct bch_fs *c,
 384                                                                   struct bch_inode_info *inode)
 385 {
 386         struct bch_writepage_state ret = { 0 };
 387
 388         bch2_inode_opts_get(&ret.opts, c, &inode->ei_inode);
 389         return ret;
 390 }
 391
 392 /*
 393  * Determine when a writepage io is full. We have to limit writepage bios to a
 394  * single page per bvec (i.e. 1MB with 4k pages) because that is the limit to
 395  * what the bounce path in bch2_write_extent() can handle. In theory we could
 396  * loosen this restriction for non-bounce I/O, but we don't have that context
 397  * here. Ideally, we can up this limit and make it configurable in the future
 398  * when the bounce path can be enhanced to accommodate larger source bios.
 399  */
 400 static inline bool bch_io_full(struct bch_writepage_io *io, unsigned len)
 401 {
 402         struct bio *bio = &io->op.wbio.bio;
 403         return bio_full(bio, len) ||
 404                 (bio->bi_iter.bi_size + len > BIO_MAX_VECS * PAGE_SIZE);
 405 }
 406
 407 static void bch2_writepage_io_done(struct bch_write_op *op)
 408 {
 409         struct bch_writepage_io *io =
 410                 container_of(op, struct bch_writepage_io, op);
 411         struct bch_fs *c = io->op.c;
 412         struct bio *bio = &io->op.wbio.bio;
 413         struct folio_iter fi;
 414         unsigned i;
 415
 416         if (io->op.error) {
 417                 set_bit(EI_INODE_ERROR, &io->inode->ei_flags);
 418
 419                 bio_for_each_folio_all(fi, bio) {
 420                         struct bch_folio *s;
 421
 422                         folio_set_error(fi.folio);
 423                         mapping_set_error(fi.folio->mapping, -EIO);
 424
 425                         s = __bch2_folio(fi.folio);
 426                         spin_lock(&s->lock);
 427                         for (i = 0; i < folio_sectors(fi.folio); i++)
 428                                 s->s[i].nr_replicas = 0;
 429                         spin_unlock(&s->lock);
 430                 }
 431         }
 432
 433         if (io->op.flags & BCH_WRITE_WROTE_DATA_INLINE) {
 434                 bio_for_each_folio_all(fi, bio) {
 435                         struct bch_folio *s;
 436
 437                         s = __bch2_folio(fi.folio);
 438                         spin_lock(&s->lock);
 439                         for (i = 0; i < folio_sectors(fi.folio); i++)
 440                                 s->s[i].nr_replicas = 0;
 441                         spin_unlock(&s->lock);
 442                 }
 443         }
 444
 445         /*
 446          * racing with fallocate can cause us to add fewer sectors than
 447          * expected - but we shouldn't add more sectors than expected:
 448          */
 449         WARN_ON_ONCE(io->op.i_sectors_delta > 0);
 450
 451         /*
 452          * (error (due to going RO) halfway through a page can screw that up
 453          * slightly)
 454          * XXX wtf?
 455            BUG_ON(io->op.op.i_sectors_delta >= PAGE_SECTORS);
 456          */
 457
 458         /*
 459          * PageWriteback is effectively our ref on the inode - fixup i_blocks
 460          * before calling end_page_writeback:
 461          */
 462         bch2_i_sectors_acct(c, io->inode, NULL, io->op.i_sectors_delta);
 463
 464         bio_for_each_folio_all(fi, bio) {
 465                 struct bch_folio *s = __bch2_folio(fi.folio);
 466
 467                 if (atomic_dec_and_test(&s->write_count))
 468                         folio_end_writeback(fi.folio);
 469         }
 470
 471         bio_put(&io->op.wbio.bio);
 472 }
 473
 474 static void bch2_writepage_do_io(struct bch_writepage_state *w)
 475 {
 476         struct bch_writepage_io *io = w->io;
 477
 478         w->io = NULL;
 479         closure_call(&io->op.cl, bch2_write, NULL, NULL);
 480 }
 481
 482 /*
 483  * Get a bch_writepage_io and add @page to it - appending to an existing one if
 484  * possible, else allocating a new one:
 485  */
 486 static void bch2_writepage_io_alloc(struct bch_fs *c,
 487                                     struct writeback_control *wbc,
 488                                     struct bch_writepage_state *w,
 489                                     struct bch_inode_info *inode,
 490                                     u64 sector,
 491                                     unsigned nr_replicas)
 492 {
 493         struct bch_write_op *op;
 494
 495         w->io = container_of(bio_alloc_bioset(NULL, BIO_MAX_VECS,
 496                                               REQ_OP_WRITE,
 497                                               GFP_KERNEL,
 498                                               &c->writepage_bioset),
 499                              struct bch_writepage_io, op.wbio.bio);
 500
 501         w->io->inode            = inode;
 502         op                      = &w->io->op;
 503         bch2_write_op_init(op, c, w->opts);
 504         op->target              = w->opts.foreground_target;
 505         op->nr_replicas         = nr_replicas;
 506         op->res.nr_replicas     = nr_replicas;
 507         op->write_point         = writepoint_hashed(inode->ei_last_dirtied);
 508         op->subvol              = inode->ei_subvol;
 509         op->pos                 = POS(inode->v.i_ino, sector);
 510         op->end_io              = bch2_writepage_io_done;
 511         op->devs_need_flush     = &inode->ei_devs_need_flush;
 512         op->wbio.bio.bi_iter.bi_sector = sector;
 513         op->wbio.bio.bi_opf     = wbc_to_write_flags(wbc);
 514 }
 515
 516 static int __bch2_writepage(struct folio *folio,
 517                             struct writeback_control *wbc,
 518                             void *data)
 519 {
 520         struct bch_inode_info *inode = to_bch_ei(folio->mapping->host);
 521         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 522         struct bch_writepage_state *w = data;
 523         struct bch_folio *s;
 524         unsigned i, offset, f_sectors, nr_replicas_this_write = U32_MAX;
 525         loff_t i_size = i_size_read(&inode->v);
 526         int ret;
 527
 528         EBUG_ON(!folio_test_uptodate(folio));
 529
 530         /* Is the folio fully inside i_size? */
 531         if (folio_end_pos(folio) <= i_size)
 532                 goto do_io;
 533
 534         /* Is the folio fully outside i_size? (truncate in progress) */
 535         if (folio_pos(folio) >= i_size) {
 536                 folio_unlock(folio);
 537                 return 0;
 538         }
 539
 540         /*
 541          * The folio straddles i_size.  It must be zeroed out on each and every
 542          * writepage invocation because it may be mmapped.  "A file is mapped
 543          * in multiples of the folio size.  For a file that is not a multiple of
 544          * the  folio size, the remaining memory is zeroed when mapped, and
 545          * writes to that region are not written out to the file."
 546          */
 547         folio_zero_segment(folio,
 548                            i_size - folio_pos(folio),
 549                            folio_size(folio));
 550 do_io:
 551         f_sectors = folio_sectors(folio);
 552         s = bch2_folio(folio);
 553
 554         if (f_sectors > w->tmp_sectors) {
 555                 kfree(w->tmp);
 556                 w->tmp = kcalloc(f_sectors, sizeof(struct bch_folio_sector), __GFP_NOFAIL);
 557                 w->tmp_sectors = f_sectors;
 558         }
 559
 560         /*
 561          * Things get really hairy with errors during writeback:
 562          */
 563         ret = bch2_get_folio_disk_reservation(c, inode, folio, false);
 564         BUG_ON(ret);
 565
 566         /* Before unlocking the page, get copy of reservations: */
 567         spin_lock(&s->lock);
 568         memcpy(w->tmp, s->s, sizeof(struct bch_folio_sector) * f_sectors);
 569
 570         for (i = 0; i < f_sectors; i++) {
 571                 if (s->s[i].state < SECTOR_dirty)
 572                         continue;
 573
 574                 nr_replicas_this_write =
 575                         min_t(unsigned, nr_replicas_this_write,
 576                               s->s[i].nr_replicas +
 577                               s->s[i].replicas_reserved);
 578         }
 579
 580         for (i = 0; i < f_sectors; i++) {
 581                 if (s->s[i].state < SECTOR_dirty)
 582                         continue;
 583
 584                 s->s[i].nr_replicas = w->opts.compression
 585                         ? 0 : nr_replicas_this_write;
 586
 587                 s->s[i].replicas_reserved = 0;
 588                 bch2_folio_sector_set(folio, s, i, SECTOR_allocated);
 589         }
 590         spin_unlock(&s->lock);
 591
 592         BUG_ON(atomic_read(&s->write_count));
 593         atomic_set(&s->write_count, 1);
 594
 595         BUG_ON(folio_test_writeback(folio));
 596         folio_start_writeback(folio);
 597
 598         folio_unlock(folio);
 599
 600         offset = 0;
 601         while (1) {
 602                 unsigned sectors = 0, dirty_sectors = 0, reserved_sectors = 0;
 603                 u64 sector;
 604
 605                 while (offset < f_sectors &&
 606                        w->tmp[offset].state < SECTOR_dirty)
 607                         offset++;
 608
 609                 if (offset == f_sectors)
 610                         break;
 611
 612                 while (offset + sectors < f_sectors &&
 613                        w->tmp[offset + sectors].state >= SECTOR_dirty) {
 614                         reserved_sectors += w->tmp[offset + sectors].replicas_reserved;
 615                         dirty_sectors += w->tmp[offset + sectors].state == SECTOR_dirty;
 616                         sectors++;
 617                 }
 618                 BUG_ON(!sectors);
 619
 620                 sector = folio_sector(folio) + offset;
 621
 622                 if (w->io &&
 623                     (w->io->op.res.nr_replicas != nr_replicas_this_write ||
 624                      bch_io_full(w->io, sectors << 9) ||
 625                      bio_end_sector(&w->io->op.wbio.bio) != sector))
 626                         bch2_writepage_do_io(w);
 627
 628                 if (!w->io)
 629                         bch2_writepage_io_alloc(c, wbc, w, inode, sector,
 630                                                 nr_replicas_this_write);
 631
 632                 atomic_inc(&s->write_count);
 633
 634                 BUG_ON(inode != w->io->inode);
 635                 BUG_ON(!bio_add_folio(&w->io->op.wbio.bio, folio,
 636                                      sectors << 9, offset << 9));
 637
 638                 /* Check for writing past i_size: */
 639                 WARN_ONCE((bio_end_sector(&w->io->op.wbio.bio) << 9) >
 640                           round_up(i_size, block_bytes(c)) &&
 641                           !test_bit(BCH_FS_EMERGENCY_RO, &c->flags),
 642                           "writing past i_size: %llu > %llu (unrounded %llu)\n",
 643                           bio_end_sector(&w->io->op.wbio.bio) << 9,
 644                           round_up(i_size, block_bytes(c)),
 645                           i_size);
 646
 647                 w->io->op.res.sectors += reserved_sectors;
 648                 w->io->op.i_sectors_delta -= dirty_sectors;
 649                 w->io->op.new_i_size = i_size;
 650
 651                 offset += sectors;
 652         }
 653
 654         if (atomic_dec_and_test(&s->write_count))
 655                 folio_end_writeback(folio);
 656
 657         return 0;
 658 }
 659
 660 int bch2_writepages(struct address_space *mapping, struct writeback_control *wbc)
 661 {
 662         struct bch_fs *c = mapping->host->i_sb->s_fs_info;
 663         struct bch_writepage_state w =
 664                 bch_writepage_state_init(c, to_bch_ei(mapping->host));
 665         struct blk_plug plug;
 666         int ret;
 667
 668         blk_start_plug(&plug);
 669         ret = write_cache_pages(mapping, wbc, __bch2_writepage, &w);
 670         if (w.io)
 671                 bch2_writepage_do_io(&w);
 672         blk_finish_plug(&plug);
 673         kfree(w.tmp);
 674         return bch2_err_class(ret);
 675 }
 676
 677 /* buffered writes: */
 678
 679 int bch2_write_begin(struct file *file, struct address_space *mapping,
 680                      loff_t pos, unsigned len,
 681                      struct page **pagep, void **fsdata)
 682 {
 683         struct bch_inode_info *inode = to_bch_ei(mapping->host);
 684         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 685         struct bch2_folio_reservation *res;
 686         struct folio *folio;
 687         unsigned offset;
 688         int ret = -ENOMEM;
 689
 690         res = kmalloc(sizeof(*res), GFP_KERNEL);
 691         if (!res)
 692                 return -ENOMEM;
 693
 694         bch2_folio_reservation_init(c, inode, res);
 695         *fsdata = res;
 696
 697         bch2_pagecache_add_get(inode);
 698
 699         folio = __filemap_get_folio(mapping, pos >> PAGE_SHIFT,
 700                                 FGP_LOCK|FGP_WRITE|FGP_CREAT|FGP_STABLE,
 701                                 mapping_gfp_mask(mapping));
 702         if (IS_ERR_OR_NULL(folio))
 703                 goto err_unlock;
 704
 705         offset = pos - folio_pos(folio);
 706         len = min_t(size_t, len, folio_end_pos(folio) - pos);
 707
 708         if (folio_test_uptodate(folio))
 709                 goto out;
 710
 711         /* If we're writing entire folio, don't need to read it in first: */
 712         if (!offset && len == folio_size(folio))
 713                 goto out;
 714
 715         if (!offset && pos + len >= inode->v.i_size) {
 716                 folio_zero_segment(folio, len, folio_size(folio));
 717                 flush_dcache_folio(folio);
 718                 goto out;
 719         }
 720
 721         if (folio_pos(folio) >= inode->v.i_size) {
 722                 folio_zero_segments(folio, 0, offset, offset + len, folio_size(folio));
 723                 flush_dcache_folio(folio);
 724                 goto out;
 725         }
 726 readpage:
 727         ret = bch2_read_single_folio(folio, mapping);
 728         if (ret)
 729                 goto err;
 730 out:
 731         ret = bch2_folio_set(c, inode_inum(inode), &folio, 1);
 732         if (ret)
 733                 goto err;
 734
 735         ret = bch2_folio_reservation_get(c, inode, folio, res, offset, len);
 736         if (ret) {
 737                 if (!folio_test_uptodate(folio)) {
 738                         /*
 739                          * If the folio hasn't been read in, we won't know if we
 740                          * actually need a reservation - we don't actually need
 741                          * to read here, we just need to check if the folio is
 742                          * fully backed by uncompressed data:
 743                          */
 744                         goto readpage;
 745                 }
 746
 747                 goto err;
 748         }
 749
 750         *pagep = &folio->page;
 751         return 0;
 752 err:
 753         folio_unlock(folio);
 754         folio_put(folio);
 755         *pagep = NULL;
 756 err_unlock:
 757         bch2_pagecache_add_put(inode);
 758         kfree(res);
 759         *fsdata = NULL;
 760         return bch2_err_class(ret);
 761 }
 762
 763 int bch2_write_end(struct file *file, struct address_space *mapping,
 764                    loff_t pos, unsigned len, unsigned copied,
 765                    struct page *page, void *fsdata)
 766 {
 767         struct bch_inode_info *inode = to_bch_ei(mapping->host);
 768         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 769         struct bch2_folio_reservation *res = fsdata;
 770         struct folio *folio = page_folio(page);
 771         unsigned offset = pos - folio_pos(folio);
 772
 773         lockdep_assert_held(&inode->v.i_rwsem);
 774         BUG_ON(offset + copied > folio_size(folio));
 775
 776         if (unlikely(copied < len && !folio_test_uptodate(folio))) {
 777                 /*
 778                  * The folio needs to be read in, but that would destroy
 779                  * our partial write - simplest thing is to just force
 780                  * userspace to redo the write:
 781                  */
 782                 folio_zero_range(folio, 0, folio_size(folio));
 783                 flush_dcache_folio(folio);
 784                 copied = 0;
 785         }
 786
 787         spin_lock(&inode->v.i_lock);
 788         if (pos + copied > inode->v.i_size)
 789                 i_size_write(&inode->v, pos + copied);
 790         spin_unlock(&inode->v.i_lock);
 791
 792         if (copied) {
 793                 if (!folio_test_uptodate(folio))
 794                         folio_mark_uptodate(folio);
 795
 796                 bch2_set_folio_dirty(c, inode, folio, res, offset, copied);
 797
 798                 inode->ei_last_dirtied = (unsigned long) current;
 799         }
 800
 801         folio_unlock(folio);
 802         folio_put(folio);
 803         bch2_pagecache_add_put(inode);
 804
 805         bch2_folio_reservation_put(c, inode, res);
 806         kfree(res);
 807
 808         return copied;
 809 }
 810
 811 static noinline void folios_trunc(folios *fs, struct folio **fi)
 812 {
 813         while (fs->data + fs->nr > fi) {
 814                 struct folio *f = darray_pop(fs);
 815
 816                 folio_unlock(f);
 817                 folio_put(f);
 818         }
 819 }
 820
 821 static int __bch2_buffered_write(struct bch_inode_info *inode,
 822                                  struct address_space *mapping,
 823                                  struct iov_iter *iter,
 824                                  loff_t pos, unsigned len)
 825 {
 826         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 827         struct bch2_folio_reservation res;
 828         folios fs;
 829         struct folio **fi, *f;
 830         unsigned copied = 0, f_offset, f_copied;
 831         u64 end = pos + len, f_pos, f_len;
 832         loff_t last_folio_pos = inode->v.i_size;
 833         int ret = 0;
 834
 835         BUG_ON(!len);
 836
 837         bch2_folio_reservation_init(c, inode, &res);
 838         darray_init(&fs);
 839
 840         ret = bch2_filemap_get_contig_folios_d(mapping, pos, end,
 841                                    FGP_LOCK|FGP_WRITE|FGP_STABLE|FGP_CREAT,
 842                                    mapping_gfp_mask(mapping),
 843                                    &fs);
 844         if (ret)
 845                 goto out;
 846
 847         BUG_ON(!fs.nr);
 848
 849         f = darray_first(fs);
 850         if (pos != folio_pos(f) && !folio_test_uptodate(f)) {
 851                 ret = bch2_read_single_folio(f, mapping);
 852                 if (ret)
 853                         goto out;
 854         }
 855
 856         f = darray_last(fs);
 857         end = min(end, folio_end_pos(f));
 858         last_folio_pos = folio_pos(f);
 859         if (end != folio_end_pos(f) && !folio_test_uptodate(f)) {
 860                 if (end >= inode->v.i_size) {
 861                         folio_zero_range(f, 0, folio_size(f));
 862                 } else {
 863                         ret = bch2_read_single_folio(f, mapping);
 864                         if (ret)
 865                                 goto out;
 866                 }
 867         }
 868
 869         ret = bch2_folio_set(c, inode_inum(inode), fs.data, fs.nr);
 870         if (ret)
 871                 goto out;
 872
 873         f_pos = pos;
 874         f_offset = pos - folio_pos(darray_first(fs));
 875         darray_for_each(fs, fi) {
 876                 f = *fi;
 877                 f_len = min(end, folio_end_pos(f)) - f_pos;
 878
 879                 /*
 880                  * XXX: per POSIX and fstests generic/275, on -ENOSPC we're
 881                  * supposed to write as much as we have disk space for.
 882                  *
 883                  * On failure here we should still write out a partial page if
 884                  * we aren't completely out of disk space - we don't do that
 885                  * yet:
 886                  */
 887                 ret = bch2_folio_reservation_get(c, inode, f, &res, f_offset, f_len);
 888                 if (unlikely(ret)) {
 889                         folios_trunc(&fs, fi);
 890                         if (!fs.nr)
 891                                 goto out;
 892
 893                         end = min(end, folio_end_pos(darray_last(fs)));
 894                         break;
 895                 }
 896
 897                 f_pos = folio_end_pos(f);
 898                 f_offset = 0;
 899         }
 900
 901         if (mapping_writably_mapped(mapping))
 902                 darray_for_each(fs, fi)
 903                         flush_dcache_folio(*fi);
 904
 905         f_pos = pos;
 906         f_offset = pos - folio_pos(darray_first(fs));
 907         darray_for_each(fs, fi) {
 908                 f = *fi;
 909                 f_len = min(end, folio_end_pos(f)) - f_pos;
 910                 f_copied = copy_page_from_iter_atomic(&f->page, f_offset, f_len, iter);
 911                 if (!f_copied) {
 912                         folios_trunc(&fs, fi);
 913                         break;
 914                 }
 915
 916                 if (!folio_test_uptodate(f) &&
 917                     f_copied != folio_size(f) &&
 918                     pos + copied + f_copied < inode->v.i_size) {
 919                         iov_iter_revert(iter, f_copied);
 920                         folio_zero_range(f, 0, folio_size(f));
 921                         folios_trunc(&fs, fi);
 922                         break;
 923                 }
 924
 925                 flush_dcache_folio(f);
 926                 copied += f_copied;
 927
 928                 if (f_copied != f_len) {
 929                         folios_trunc(&fs, fi + 1);
 930                         break;
 931                 }
 932
 933                 f_pos = folio_end_pos(f);
 934                 f_offset = 0;
 935         }
 936
 937         if (!copied)
 938                 goto out;
 939
 940         end = pos + copied;
 941
 942         spin_lock(&inode->v.i_lock);
 943         if (end > inode->v.i_size)
 944                 i_size_write(&inode->v, end);
 945         spin_unlock(&inode->v.i_lock);
 946
 947         f_pos = pos;
 948         f_offset = pos - folio_pos(darray_first(fs));
 949         darray_for_each(fs, fi) {
 950                 f = *fi;
 951                 f_len = min(end, folio_end_pos(f)) - f_pos;
 952
 953                 if (!folio_test_uptodate(f))
 954                         folio_mark_uptodate(f);
 955
 956                 bch2_set_folio_dirty(c, inode, f, &res, f_offset, f_len);
 957
 958                 f_pos = folio_end_pos(f);
 959                 f_offset = 0;
 960         }
 961
 962         inode->ei_last_dirtied = (unsigned long) current;
 963 out:
 964         darray_for_each(fs, fi) {
 965                 folio_unlock(*fi);
 966                 folio_put(*fi);
 967         }
 968
 969         /*
 970          * If the last folio added to the mapping starts beyond current EOF, we
 971          * performed a short write but left around at least one post-EOF folio.
 972          * Clean up the mapping before we return.
 973          */
 974         if (last_folio_pos >= inode->v.i_size)
 975                 truncate_pagecache(&inode->v, inode->v.i_size);
 976
 977         darray_exit(&fs);
 978         bch2_folio_reservation_put(c, inode, &res);
 979
 980         return copied ?: ret;
 981 }
 982
 983 static ssize_t bch2_buffered_write(struct kiocb *iocb, struct iov_iter *iter)
 984 {
 985         struct file *file = iocb->ki_filp;
 986         struct address_space *mapping = file->f_mapping;
 987         struct bch_inode_info *inode = file_bch_inode(file);
 988         loff_t pos = iocb->ki_pos;
 989         ssize_t written = 0;
 990         int ret = 0;
 991
 992         bch2_pagecache_add_get(inode);
 993
 994         do {
 995                 unsigned offset = pos & (PAGE_SIZE - 1);
 996                 unsigned bytes = iov_iter_count(iter);
 997 again:
 998                 /*
 999                  * Bring in the user page that we will copy from _first_.
1000                  * Otherwise there's a nasty deadlock on copying from the
1001                  * same page as we're writing to, without it being marked
1002                  * up-to-date.
1003                  *
1004                  * Not only is this an optimisation, but it is also required
1005                  * to check that the address is actually valid, when atomic
1006                  * usercopies are used, below.
1007                  */
1008                 if (unlikely(fault_in_iov_iter_readable(iter, bytes))) {
1009                         bytes = min_t(unsigned long, iov_iter_count(iter),
1010                                       PAGE_SIZE - offset);
1011
1012                         if (unlikely(fault_in_iov_iter_readable(iter, bytes))) {
1013                                 ret = -EFAULT;
1014                                 break;
1015                         }
1016                 }
1017
1018                 if (unlikely(fatal_signal_pending(current))) {
1019                         ret = -EINTR;
1020                         break;
1021                 }
1022
1023                 ret = __bch2_buffered_write(inode, mapping, iter, pos, bytes);
1024                 if (unlikely(ret < 0))
1025                         break;
1026
1027                 cond_resched();
1028
1029                 if (unlikely(ret == 0)) {
1030                         /*
1031                          * If we were unable to copy any data at all, we must
1032                          * fall back to a single segment length write.
1033                          *
1034                          * If we didn't fallback here, we could livelock
1035                          * because not all segments in the iov can be copied at
1036                          * once without a pagefault.
1037                          */
1038                         bytes = min_t(unsigned long, PAGE_SIZE - offset,
1039                                       iov_iter_single_seg_count(iter));
1040                         goto again;
1041                 }
1042                 pos += ret;
1043                 written += ret;
1044                 ret = 0;
1045
1046                 balance_dirty_pages_ratelimited(mapping);
1047         } while (iov_iter_count(iter));
1048
1049         bch2_pagecache_add_put(inode);
1050
1051         return written ? written : ret;
1052 }
1053
1054 ssize_t bch2_write_iter(struct kiocb *iocb, struct iov_iter *from)
1055 {
1056         struct file *file = iocb->ki_filp;
1057         struct bch_inode_info *inode = file_bch_inode(file);
1058         ssize_t ret;
1059
1060         if (iocb->ki_flags & IOCB_DIRECT) {
1061                 ret = bch2_direct_write(iocb, from);
1062                 goto out;
1063         }
1064
1065         inode_lock(&inode->v);
1066
1067         ret = generic_write_checks(iocb, from);
1068         if (ret <= 0)
1069                 goto unlock;
1070
1071         ret = file_remove_privs(file);
1072         if (ret)
1073                 goto unlock;
1074
1075         ret = file_update_time(file);
1076         if (ret)
1077                 goto unlock;
1078
1079         ret = bch2_buffered_write(iocb, from);
1080         if (likely(ret > 0))
1081                 iocb->ki_pos += ret;
1082 unlock:
1083         inode_unlock(&inode->v);
1084
1085         if (ret > 0)
1086                 ret = generic_write_sync(iocb, ret);
1087 out:
1088         return bch2_err_class(ret);
1089 }
1090
1091 void bch2_fs_fs_io_buffered_exit(struct bch_fs *c)
1092 {
1093         bioset_exit(&c->writepage_bioset);
1094 }
1095
1096 int bch2_fs_fs_io_buffered_init(struct bch_fs *c)
1097 {
1098         if (bioset_init(&c->writepage_bioset,
1099                         4, offsetof(struct bch_writepage_io, op.wbio.bio),
1100                         BIOSET_NEED_BVECS))
1101                 return -BCH_ERR_ENOMEM_writepage_bioset_init;
1102
1103         return 0;
1104 }
1105
1106 #endif /* NO_BCACHEFS_FS */