git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/fs-io-buffered.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #ifndef NO_BCACHEFS_FS
   3
   4 #include "bcachefs.h"
   5 #include "alloc_foreground.h"
   6 #include "bkey_buf.h"
   7 #include "fs-io.h"
   8 #include "fs-io-buffered.h"
   9 #include "fs-io-direct.h"
  10 #include "fs-io-pagecache.h"
  11 #include "io_read.h"
  12 #include "io_write.h"
  13
  14 #include <linux/backing-dev.h>
  15 #include <linux/pagemap.h>
  16 #include <linux/writeback.h>
  17
  18 static inline bool bio_full(struct bio *bio, unsigned len)
  19 {
  20         if (bio->bi_vcnt >= bio->bi_max_vecs)
  21                 return true;
  22         if (bio->bi_iter.bi_size > UINT_MAX - len)
  23                 return true;
  24         return false;
  25 }
  26
  27 /* readpage(s): */
  28
  29 static void bch2_readpages_end_io(struct bio *bio)
  30 {
  31         struct folio_iter fi;
  32
  33         bio_for_each_folio_all(fi, bio) {
  34                 if (!bio->bi_status) {
  35                         folio_mark_uptodate(fi.folio);
  36                 } else {
  37                         folio_clear_uptodate(fi.folio);
  38                         folio_set_error(fi.folio);
  39                 }
  40                 folio_unlock(fi.folio);
  41         }
  42
  43         bio_put(bio);
  44 }
  45
  46 struct readpages_iter {
  47         struct address_space    *mapping;
  48         unsigned                idx;
  49         folios                  folios;
  50 };
  51
  52 static int readpages_iter_init(struct readpages_iter *iter,
  53                                struct readahead_control *ractl)
  54 {
  55         memset(iter, 0, sizeof(*iter));
  56
  57         iter->mapping = ractl->mapping;
  58
  59         int ret = bch2_filemap_get_contig_folios_d(iter->mapping,
  60                                 ractl->_index << PAGE_SHIFT,
  61                                 (ractl->_index + ractl->_nr_pages) << PAGE_SHIFT,
  62                                 0, mapping_gfp_mask(iter->mapping),
  63                                 &iter->folios);
  64         if (ret)
  65                 return ret;
  66
  67         darray_for_each(iter->folios, fi) {
  68                 ractl->_nr_pages -= 1U << folio_order(*fi);
  69                 __bch2_folio_create(*fi, __GFP_NOFAIL|GFP_KERNEL);
  70                 folio_put(*fi);
  71                 folio_put(*fi);
  72         }
  73
  74         return 0;
  75 }
  76
  77 static inline struct folio *readpage_iter_peek(struct readpages_iter *iter)
  78 {
  79         if (iter->idx >= iter->folios.nr)
  80                 return NULL;
  81         return iter->folios.data[iter->idx];
  82 }
  83
  84 static inline void readpage_iter_advance(struct readpages_iter *iter)
  85 {
  86         iter->idx++;
  87 }
  88
  89 static bool extent_partial_reads_expensive(struct bkey_s_c k)
  90 {
  91         struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
  92         struct bch_extent_crc_unpacked crc;
  93         const union bch_extent_entry *i;
  94
  95         bkey_for_each_crc(k.k, ptrs, crc, i)
  96                 if (crc.csum_type || crc.compression_type)
  97                         return true;
  98         return false;
  99 }
 100
 101 static int readpage_bio_extend(struct btree_trans *trans,
 102                                struct readpages_iter *iter,
 103                                struct bio *bio,
 104                                unsigned sectors_this_extent,
 105                                bool get_more)
 106 {
 107         /* Don't hold btree locks while allocating memory: */
 108         bch2_trans_unlock(trans);
 109
 110         while (bio_sectors(bio) < sectors_this_extent &&
 111                bio->bi_vcnt < bio->bi_max_vecs) {
 112                 struct folio *folio = readpage_iter_peek(iter);
 113                 int ret;
 114
 115                 if (folio) {
 116                         readpage_iter_advance(iter);
 117                 } else {
 118                         pgoff_t folio_offset = bio_end_sector(bio) >> PAGE_SECTORS_SHIFT;
 119
 120                         if (!get_more)
 121                                 break;
 122
 123                         folio = xa_load(&iter->mapping->i_pages, folio_offset);
 124                         if (folio && !xa_is_value(folio))
 125                                 break;
 126
 127                         folio = filemap_alloc_folio(readahead_gfp_mask(iter->mapping), 0);
 128                         if (!folio)
 129                                 break;
 130
 131                         if (!__bch2_folio_create(folio, GFP_KERNEL)) {
 132                                 folio_put(folio);
 133                                 break;
 134                         }
 135
 136                         ret = filemap_add_folio(iter->mapping, folio, folio_offset, GFP_KERNEL);
 137                         if (ret) {
 138                                 __bch2_folio_release(folio);
 139                                 folio_put(folio);
 140                                 break;
 141                         }
 142
 143                         folio_put(folio);
 144                 }
 145
 146                 BUG_ON(folio_sector(folio) != bio_end_sector(bio));
 147
 148                 BUG_ON(!bio_add_folio(bio, folio, folio_size(folio), 0));
 149         }
 150
 151         return bch2_trans_relock(trans);
 152 }
 153
 154 static void bchfs_read(struct btree_trans *trans,
 155                        struct bch_read_bio *rbio,
 156                        subvol_inum inum,
 157                        struct readpages_iter *readpages_iter)
 158 {
 159         struct bch_fs *c = trans->c;
 160         struct btree_iter iter;
 161         struct bkey_buf sk;
 162         int flags = BCH_READ_RETRY_IF_STALE|
 163                 BCH_READ_MAY_PROMOTE;
 164         u32 snapshot;
 165         int ret = 0;
 166
 167         rbio->c = c;
 168         rbio->start_time = local_clock();
 169         rbio->subvol = inum.subvol;
 170
 171         bch2_bkey_buf_init(&sk);
 172 retry:
 173         bch2_trans_begin(trans);
 174         iter = (struct btree_iter) { NULL };
 175
 176         ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot);
 177         if (ret)
 178                 goto err;
 179
 180         bch2_trans_iter_init(trans, &iter, BTREE_ID_extents,
 181                              SPOS(inum.inum, rbio->bio.bi_iter.bi_sector, snapshot),
 182                              BTREE_ITER_SLOTS);
 183         while (1) {
 184                 struct bkey_s_c k;
 185                 unsigned bytes, sectors, offset_into_extent;
 186                 enum btree_id data_btree = BTREE_ID_extents;
 187
 188                 /*
 189                  * read_extent -> io_time_reset may cause a transaction restart
 190                  * without returning an error, we need to check for that here:
 191                  */
 192                 ret = bch2_trans_relock(trans);
 193                 if (ret)
 194                         break;
 195
 196                 bch2_btree_iter_set_pos(&iter,
 197                                 POS(inum.inum, rbio->bio.bi_iter.bi_sector));
 198
 199                 k = bch2_btree_iter_peek_slot(&iter);
 200                 ret = bkey_err(k);
 201                 if (ret)
 202                         break;
 203
 204                 offset_into_extent = iter.pos.offset -
 205                         bkey_start_offset(k.k);
 206                 sectors = k.k->size - offset_into_extent;
 207
 208                 bch2_bkey_buf_reassemble(&sk, c, k);
 209
 210                 ret = bch2_read_indirect_extent(trans, &data_btree,
 211                                         &offset_into_extent, &sk);
 212                 if (ret)
 213                         break;
 214
 215                 k = bkey_i_to_s_c(sk.k);
 216
 217                 sectors = min(sectors, k.k->size - offset_into_extent);
 218
 219                 if (readpages_iter) {
 220                         ret = readpage_bio_extend(trans, readpages_iter, &rbio->bio, sectors,
 221                                                   extent_partial_reads_expensive(k));
 222                         if (ret)
 223                                 break;
 224                 }
 225
 226                 bytes = min(sectors, bio_sectors(&rbio->bio)) << 9;
 227                 swap(rbio->bio.bi_iter.bi_size, bytes);
 228
 229                 if (rbio->bio.bi_iter.bi_size == bytes)
 230                         flags |= BCH_READ_LAST_FRAGMENT;
 231
 232                 bch2_bio_page_state_set(&rbio->bio, k);
 233
 234                 bch2_read_extent(trans, rbio, iter.pos,
 235                                  data_btree, k, offset_into_extent, flags);
 236
 237                 if (flags & BCH_READ_LAST_FRAGMENT)
 238                         break;
 239
 240                 swap(rbio->bio.bi_iter.bi_size, bytes);
 241                 bio_advance(&rbio->bio, bytes);
 242
 243                 ret = btree_trans_too_many_iters(trans);
 244                 if (ret)
 245                         break;
 246         }
 247 err:
 248         bch2_trans_iter_exit(trans, &iter);
 249
 250         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
 251                 goto retry;
 252
 253         if (ret) {
 254                 bch_err_inum_offset_ratelimited(c,
 255                                 iter.pos.inode,
 256                                 iter.pos.offset << 9,
 257                                 "read error %i from btree lookup", ret);
 258                 rbio->bio.bi_status = BLK_STS_IOERR;
 259                 bio_endio(&rbio->bio);
 260         }
 261
 262         bch2_bkey_buf_exit(&sk, c);
 263 }
 264
 265 void bch2_readahead(struct readahead_control *ractl)
 266 {
 267         struct bch_inode_info *inode = to_bch_ei(ractl->mapping->host);
 268         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 269         struct bch_io_opts opts;
 270         struct btree_trans *trans = bch2_trans_get(c);
 271         struct folio *folio;
 272         struct readpages_iter readpages_iter;
 273         int ret;
 274
 275         bch2_inode_opts_get(&opts, c, &inode->ei_inode);
 276
 277         ret = readpages_iter_init(&readpages_iter, ractl);
 278         BUG_ON(ret);
 279
 280         bch2_pagecache_add_get(inode);
 281
 282         while ((folio = readpage_iter_peek(&readpages_iter))) {
 283                 unsigned n = min_t(unsigned,
 284                                    readpages_iter.folios.nr -
 285                                    readpages_iter.idx,
 286                                    BIO_MAX_VECS);
 287                 struct bch_read_bio *rbio =
 288                         rbio_init(bio_alloc_bioset(NULL, n, REQ_OP_READ,
 289                                                    GFP_KERNEL, &c->bio_read),
 290                                   opts);
 291
 292                 readpage_iter_advance(&readpages_iter);
 293
 294                 rbio->bio.bi_iter.bi_sector = folio_sector(folio);
 295                 rbio->bio.bi_end_io = bch2_readpages_end_io;
 296                 BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0));
 297
 298                 bchfs_read(trans, rbio, inode_inum(inode),
 299                            &readpages_iter);
 300                 bch2_trans_unlock(trans);
 301         }
 302
 303         bch2_pagecache_add_put(inode);
 304
 305         bch2_trans_put(trans);
 306         darray_exit(&readpages_iter.folios);
 307 }
 308
 309 static void __bchfs_readfolio(struct bch_fs *c, struct bch_read_bio *rbio,
 310                              subvol_inum inum, struct folio *folio)
 311 {
 312         bch2_folio_create(folio, __GFP_NOFAIL);
 313
 314         rbio->bio.bi_opf = REQ_OP_READ|REQ_SYNC;
 315         rbio->bio.bi_iter.bi_sector = folio_sector(folio);
 316         BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0));
 317
 318         bch2_trans_run(c, (bchfs_read(trans, rbio, inum, NULL), 0));
 319 }
 320
 321 static void bch2_read_single_folio_end_io(struct bio *bio)
 322 {
 323         complete(bio->bi_private);
 324 }
 325
 326 int bch2_read_single_folio(struct folio *folio, struct address_space *mapping)
 327 {
 328         struct bch_inode_info *inode = to_bch_ei(mapping->host);
 329         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 330         struct bch_read_bio *rbio;
 331         struct bch_io_opts opts;
 332         int ret;
 333         DECLARE_COMPLETION_ONSTACK(done);
 334
 335         bch2_inode_opts_get(&opts, c, &inode->ei_inode);
 336
 337         rbio = rbio_init(bio_alloc_bioset(NULL, 1, REQ_OP_READ, GFP_KERNEL, &c->bio_read),
 338                          opts);
 339         rbio->bio.bi_private = &done;
 340         rbio->bio.bi_end_io = bch2_read_single_folio_end_io;
 341
 342         __bchfs_readfolio(c, rbio, inode_inum(inode), folio);
 343         wait_for_completion(&done);
 344
 345         ret = blk_status_to_errno(rbio->bio.bi_status);
 346         bio_put(&rbio->bio);
 347
 348         if (ret < 0)
 349                 return ret;
 350
 351         folio_mark_uptodate(folio);
 352         return 0;
 353 }
 354
 355 int bch2_read_folio(struct file *file, struct folio *folio)
 356 {
 357         int ret;
 358
 359         ret = bch2_read_single_folio(folio, folio->mapping);
 360         folio_unlock(folio);
 361         return bch2_err_class(ret);
 362 }
 363
 364 /* writepages: */
 365
 366 struct bch_writepage_io {
 367         struct bch_inode_info           *inode;
 368
 369         /* must be last: */
 370         struct bch_write_op             op;
 371 };
 372
 373 struct bch_writepage_state {
 374         struct bch_writepage_io *io;
 375         struct bch_io_opts      opts;
 376         struct bch_folio_sector *tmp;
 377         unsigned                tmp_sectors;
 378 };
 379
 380 static inline struct bch_writepage_state bch_writepage_state_init(struct bch_fs *c,
 381                                                                   struct bch_inode_info *inode)
 382 {
 383         struct bch_writepage_state ret = { 0 };
 384
 385         bch2_inode_opts_get(&ret.opts, c, &inode->ei_inode);
 386         return ret;
 387 }
 388
 389 /*
 390  * Determine when a writepage io is full. We have to limit writepage bios to a
 391  * single page per bvec (i.e. 1MB with 4k pages) because that is the limit to
 392  * what the bounce path in bch2_write_extent() can handle. In theory we could
 393  * loosen this restriction for non-bounce I/O, but we don't have that context
 394  * here. Ideally, we can up this limit and make it configurable in the future
 395  * when the bounce path can be enhanced to accommodate larger source bios.
 396  */
 397 static inline bool bch_io_full(struct bch_writepage_io *io, unsigned len)
 398 {
 399         struct bio *bio = &io->op.wbio.bio;
 400         return bio_full(bio, len) ||
 401                 (bio->bi_iter.bi_size + len > BIO_MAX_VECS * PAGE_SIZE);
 402 }
 403
 404 static void bch2_writepage_io_done(struct bch_write_op *op)
 405 {
 406         struct bch_writepage_io *io =
 407                 container_of(op, struct bch_writepage_io, op);
 408         struct bch_fs *c = io->op.c;
 409         struct bio *bio = &io->op.wbio.bio;
 410         struct folio_iter fi;
 411         unsigned i;
 412
 413         if (io->op.error) {
 414                 set_bit(EI_INODE_ERROR, &io->inode->ei_flags);
 415
 416                 bio_for_each_folio_all(fi, bio) {
 417                         struct bch_folio *s;
 418
 419                         folio_set_error(fi.folio);
 420                         mapping_set_error(fi.folio->mapping, -EIO);
 421
 422                         s = __bch2_folio(fi.folio);
 423                         spin_lock(&s->lock);
 424                         for (i = 0; i < folio_sectors(fi.folio); i++)
 425                                 s->s[i].nr_replicas = 0;
 426                         spin_unlock(&s->lock);
 427                 }
 428         }
 429
 430         if (io->op.flags & BCH_WRITE_WROTE_DATA_INLINE) {
 431                 bio_for_each_folio_all(fi, bio) {
 432                         struct bch_folio *s;
 433
 434                         s = __bch2_folio(fi.folio);
 435                         spin_lock(&s->lock);
 436                         for (i = 0; i < folio_sectors(fi.folio); i++)
 437                                 s->s[i].nr_replicas = 0;
 438                         spin_unlock(&s->lock);
 439                 }
 440         }
 441
 442         /*
 443          * racing with fallocate can cause us to add fewer sectors than
 444          * expected - but we shouldn't add more sectors than expected:
 445          */
 446         WARN_ON_ONCE(io->op.i_sectors_delta > 0);
 447
 448         /*
 449          * (error (due to going RO) halfway through a page can screw that up
 450          * slightly)
 451          * XXX wtf?
 452            BUG_ON(io->op.op.i_sectors_delta >= PAGE_SECTORS);
 453          */
 454
 455         /*
 456          * PageWriteback is effectively our ref on the inode - fixup i_blocks
 457          * before calling end_page_writeback:
 458          */
 459         bch2_i_sectors_acct(c, io->inode, NULL, io->op.i_sectors_delta);
 460
 461         bio_for_each_folio_all(fi, bio) {
 462                 struct bch_folio *s = __bch2_folio(fi.folio);
 463
 464                 if (atomic_dec_and_test(&s->write_count))
 465                         folio_end_writeback(fi.folio);
 466         }
 467
 468         bio_put(&io->op.wbio.bio);
 469 }
 470
 471 static void bch2_writepage_do_io(struct bch_writepage_state *w)
 472 {
 473         struct bch_writepage_io *io = w->io;
 474
 475         w->io = NULL;
 476         closure_call(&io->op.cl, bch2_write, NULL, NULL);
 477 }
 478
 479 /*
 480  * Get a bch_writepage_io and add @page to it - appending to an existing one if
 481  * possible, else allocating a new one:
 482  */
 483 static void bch2_writepage_io_alloc(struct bch_fs *c,
 484                                     struct writeback_control *wbc,
 485                                     struct bch_writepage_state *w,
 486                                     struct bch_inode_info *inode,
 487                                     u64 sector,
 488                                     unsigned nr_replicas)
 489 {
 490         struct bch_write_op *op;
 491
 492         w->io = container_of(bio_alloc_bioset(NULL, BIO_MAX_VECS,
 493                                               REQ_OP_WRITE,
 494                                               GFP_KERNEL,
 495                                               &c->writepage_bioset),
 496                              struct bch_writepage_io, op.wbio.bio);
 497
 498         w->io->inode            = inode;
 499         op                      = &w->io->op;
 500         bch2_write_op_init(op, c, w->opts);
 501         op->target              = w->opts.foreground_target;
 502         op->nr_replicas         = nr_replicas;
 503         op->res.nr_replicas     = nr_replicas;
 504         op->write_point         = writepoint_hashed(inode->ei_last_dirtied);
 505         op->subvol              = inode->ei_subvol;
 506         op->pos                 = POS(inode->v.i_ino, sector);
 507         op->end_io              = bch2_writepage_io_done;
 508         op->devs_need_flush     = &inode->ei_devs_need_flush;
 509         op->wbio.bio.bi_iter.bi_sector = sector;
 510         op->wbio.bio.bi_opf     = wbc_to_write_flags(wbc);
 511 }
 512
 513 static int __bch2_writepage(struct folio *folio,
 514                             struct writeback_control *wbc,
 515                             void *data)
 516 {
 517         struct bch_inode_info *inode = to_bch_ei(folio->mapping->host);
 518         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 519         struct bch_writepage_state *w = data;
 520         struct bch_folio *s;
 521         unsigned i, offset, f_sectors, nr_replicas_this_write = U32_MAX;
 522         loff_t i_size = i_size_read(&inode->v);
 523         int ret;
 524
 525         EBUG_ON(!folio_test_uptodate(folio));
 526
 527         /* Is the folio fully inside i_size? */
 528         if (folio_end_pos(folio) <= i_size)
 529                 goto do_io;
 530
 531         /* Is the folio fully outside i_size? (truncate in progress) */
 532         if (folio_pos(folio) >= i_size) {
 533                 folio_unlock(folio);
 534                 return 0;
 535         }
 536
 537         /*
 538          * The folio straddles i_size.  It must be zeroed out on each and every
 539          * writepage invocation because it may be mmapped.  "A file is mapped
 540          * in multiples of the folio size.  For a file that is not a multiple of
 541          * the  folio size, the remaining memory is zeroed when mapped, and
 542          * writes to that region are not written out to the file."
 543          */
 544         folio_zero_segment(folio,
 545                            i_size - folio_pos(folio),
 546                            folio_size(folio));
 547 do_io:
 548         f_sectors = folio_sectors(folio);
 549         s = bch2_folio(folio);
 550
 551         if (f_sectors > w->tmp_sectors) {
 552                 kfree(w->tmp);
 553                 w->tmp = kcalloc(f_sectors, sizeof(struct bch_folio_sector), __GFP_NOFAIL);
 554                 w->tmp_sectors = f_sectors;
 555         }
 556
 557         /*
 558          * Things get really hairy with errors during writeback:
 559          */
 560         ret = bch2_get_folio_disk_reservation(c, inode, folio, false);
 561         BUG_ON(ret);
 562
 563         /* Before unlocking the page, get copy of reservations: */
 564         spin_lock(&s->lock);
 565         memcpy(w->tmp, s->s, sizeof(struct bch_folio_sector) * f_sectors);
 566
 567         for (i = 0; i < f_sectors; i++) {
 568                 if (s->s[i].state < SECTOR_dirty)
 569                         continue;
 570
 571                 nr_replicas_this_write =
 572                         min_t(unsigned, nr_replicas_this_write,
 573                               s->s[i].nr_replicas +
 574                               s->s[i].replicas_reserved);
 575         }
 576
 577         for (i = 0; i < f_sectors; i++) {
 578                 if (s->s[i].state < SECTOR_dirty)
 579                         continue;
 580
 581                 s->s[i].nr_replicas = w->opts.compression
 582                         ? 0 : nr_replicas_this_write;
 583
 584                 s->s[i].replicas_reserved = 0;
 585                 bch2_folio_sector_set(folio, s, i, SECTOR_allocated);
 586         }
 587         spin_unlock(&s->lock);
 588
 589         BUG_ON(atomic_read(&s->write_count));
 590         atomic_set(&s->write_count, 1);
 591
 592         BUG_ON(folio_test_writeback(folio));
 593         folio_start_writeback(folio);
 594
 595         folio_unlock(folio);
 596
 597         offset = 0;
 598         while (1) {
 599                 unsigned sectors = 0, dirty_sectors = 0, reserved_sectors = 0;
 600                 u64 sector;
 601
 602                 while (offset < f_sectors &&
 603                        w->tmp[offset].state < SECTOR_dirty)
 604                         offset++;
 605
 606                 if (offset == f_sectors)
 607                         break;
 608
 609                 while (offset + sectors < f_sectors &&
 610                        w->tmp[offset + sectors].state >= SECTOR_dirty) {
 611                         reserved_sectors += w->tmp[offset + sectors].replicas_reserved;
 612                         dirty_sectors += w->tmp[offset + sectors].state == SECTOR_dirty;
 613                         sectors++;
 614                 }
 615                 BUG_ON(!sectors);
 616
 617                 sector = folio_sector(folio) + offset;
 618
 619                 if (w->io &&
 620                     (w->io->op.res.nr_replicas != nr_replicas_this_write ||
 621                      bch_io_full(w->io, sectors << 9) ||
 622                      bio_end_sector(&w->io->op.wbio.bio) != sector))
 623                         bch2_writepage_do_io(w);
 624
 625                 if (!w->io)
 626                         bch2_writepage_io_alloc(c, wbc, w, inode, sector,
 627                                                 nr_replicas_this_write);
 628
 629                 atomic_inc(&s->write_count);
 630
 631                 BUG_ON(inode != w->io->inode);
 632                 BUG_ON(!bio_add_folio(&w->io->op.wbio.bio, folio,
 633                                      sectors << 9, offset << 9));
 634
 635                 /* Check for writing past i_size: */
 636                 WARN_ONCE((bio_end_sector(&w->io->op.wbio.bio) << 9) >
 637                           round_up(i_size, block_bytes(c)) &&
 638                           !test_bit(BCH_FS_emergency_ro, &c->flags),
 639                           "writing past i_size: %llu > %llu (unrounded %llu)\n",
 640                           bio_end_sector(&w->io->op.wbio.bio) << 9,
 641                           round_up(i_size, block_bytes(c)),
 642                           i_size);
 643
 644                 w->io->op.res.sectors += reserved_sectors;
 645                 w->io->op.i_sectors_delta -= dirty_sectors;
 646                 w->io->op.new_i_size = i_size;
 647
 648                 offset += sectors;
 649         }
 650
 651         if (atomic_dec_and_test(&s->write_count))
 652                 folio_end_writeback(folio);
 653
 654         return 0;
 655 }
 656
 657 int bch2_writepages(struct address_space *mapping, struct writeback_control *wbc)
 658 {
 659         struct bch_fs *c = mapping->host->i_sb->s_fs_info;
 660         struct bch_writepage_state w =
 661                 bch_writepage_state_init(c, to_bch_ei(mapping->host));
 662         struct blk_plug plug;
 663         int ret;
 664
 665         blk_start_plug(&plug);
 666         ret = write_cache_pages(mapping, wbc, __bch2_writepage, &w);
 667         if (w.io)
 668                 bch2_writepage_do_io(&w);
 669         blk_finish_plug(&plug);
 670         kfree(w.tmp);
 671         return bch2_err_class(ret);
 672 }
 673
 674 /* buffered writes: */
 675
 676 int bch2_write_begin(struct file *file, struct address_space *mapping,
 677                      loff_t pos, unsigned len,
 678                      struct page **pagep, void **fsdata)
 679 {
 680         struct bch_inode_info *inode = to_bch_ei(mapping->host);
 681         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 682         struct bch2_folio_reservation *res;
 683         struct folio *folio;
 684         unsigned offset;
 685         int ret = -ENOMEM;
 686
 687         res = kmalloc(sizeof(*res), GFP_KERNEL);
 688         if (!res)
 689                 return -ENOMEM;
 690
 691         bch2_folio_reservation_init(c, inode, res);
 692         *fsdata = res;
 693
 694         bch2_pagecache_add_get(inode);
 695
 696         folio = __filemap_get_folio(mapping, pos >> PAGE_SHIFT,
 697                                 FGP_LOCK|FGP_WRITE|FGP_CREAT|FGP_STABLE,
 698                                 mapping_gfp_mask(mapping));
 699         if (IS_ERR_OR_NULL(folio))
 700                 goto err_unlock;
 701
 702         offset = pos - folio_pos(folio);
 703         len = min_t(size_t, len, folio_end_pos(folio) - pos);
 704
 705         if (folio_test_uptodate(folio))
 706                 goto out;
 707
 708         /* If we're writing entire folio, don't need to read it in first: */
 709         if (!offset && len == folio_size(folio))
 710                 goto out;
 711
 712         if (!offset && pos + len >= inode->v.i_size) {
 713                 folio_zero_segment(folio, len, folio_size(folio));
 714                 flush_dcache_folio(folio);
 715                 goto out;
 716         }
 717
 718         if (folio_pos(folio) >= inode->v.i_size) {
 719                 folio_zero_segments(folio, 0, offset, offset + len, folio_size(folio));
 720                 flush_dcache_folio(folio);
 721                 goto out;
 722         }
 723 readpage:
 724         ret = bch2_read_single_folio(folio, mapping);
 725         if (ret)
 726                 goto err;
 727 out:
 728         ret = bch2_folio_set(c, inode_inum(inode), &folio, 1);
 729         if (ret)
 730                 goto err;
 731
 732         ret = bch2_folio_reservation_get(c, inode, folio, res, offset, len);
 733         if (ret) {
 734                 if (!folio_test_uptodate(folio)) {
 735                         /*
 736                          * If the folio hasn't been read in, we won't know if we
 737                          * actually need a reservation - we don't actually need
 738                          * to read here, we just need to check if the folio is
 739                          * fully backed by uncompressed data:
 740                          */
 741                         goto readpage;
 742                 }
 743
 744                 goto err;
 745         }
 746
 747         *pagep = &folio->page;
 748         return 0;
 749 err:
 750         folio_unlock(folio);
 751         folio_put(folio);
 752         *pagep = NULL;
 753 err_unlock:
 754         bch2_pagecache_add_put(inode);
 755         kfree(res);
 756         *fsdata = NULL;
 757         return bch2_err_class(ret);
 758 }
 759
 760 int bch2_write_end(struct file *file, struct address_space *mapping,
 761                    loff_t pos, unsigned len, unsigned copied,
 762                    struct page *page, void *fsdata)
 763 {
 764         struct bch_inode_info *inode = to_bch_ei(mapping->host);
 765         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 766         struct bch2_folio_reservation *res = fsdata;
 767         struct folio *folio = page_folio(page);
 768         unsigned offset = pos - folio_pos(folio);
 769
 770         lockdep_assert_held(&inode->v.i_rwsem);
 771         BUG_ON(offset + copied > folio_size(folio));
 772
 773         if (unlikely(copied < len && !folio_test_uptodate(folio))) {
 774                 /*
 775                  * The folio needs to be read in, but that would destroy
 776                  * our partial write - simplest thing is to just force
 777                  * userspace to redo the write:
 778                  */
 779                 folio_zero_range(folio, 0, folio_size(folio));
 780                 flush_dcache_folio(folio);
 781                 copied = 0;
 782         }
 783
 784         spin_lock(&inode->v.i_lock);
 785         if (pos + copied > inode->v.i_size)
 786                 i_size_write(&inode->v, pos + copied);
 787         spin_unlock(&inode->v.i_lock);
 788
 789         if (copied) {
 790                 if (!folio_test_uptodate(folio))
 791                         folio_mark_uptodate(folio);
 792
 793                 bch2_set_folio_dirty(c, inode, folio, res, offset, copied);
 794
 795                 inode->ei_last_dirtied = (unsigned long) current;
 796         }
 797
 798         folio_unlock(folio);
 799         folio_put(folio);
 800         bch2_pagecache_add_put(inode);
 801
 802         bch2_folio_reservation_put(c, inode, res);
 803         kfree(res);
 804
 805         return copied;
 806 }
 807
 808 static noinline void folios_trunc(folios *fs, struct folio **fi)
 809 {
 810         while (fs->data + fs->nr > fi) {
 811                 struct folio *f = darray_pop(fs);
 812
 813                 folio_unlock(f);
 814                 folio_put(f);
 815         }
 816 }
 817
 818 static int __bch2_buffered_write(struct bch_inode_info *inode,
 819                                  struct address_space *mapping,
 820                                  struct iov_iter *iter,
 821                                  loff_t pos, unsigned len)
 822 {
 823         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 824         struct bch2_folio_reservation res;
 825         folios fs;
 826         struct folio *f;
 827         unsigned copied = 0, f_offset, f_copied;
 828         u64 end = pos + len, f_pos, f_len;
 829         loff_t last_folio_pos = inode->v.i_size;
 830         int ret = 0;
 831
 832         BUG_ON(!len);
 833
 834         bch2_folio_reservation_init(c, inode, &res);
 835         darray_init(&fs);
 836
 837         ret = bch2_filemap_get_contig_folios_d(mapping, pos, end,
 838                                    FGP_LOCK|FGP_WRITE|FGP_STABLE|FGP_CREAT,
 839                                    mapping_gfp_mask(mapping),
 840                                    &fs);
 841         if (ret)
 842                 goto out;
 843
 844         BUG_ON(!fs.nr);
 845
 846         f = darray_first(fs);
 847         if (pos != folio_pos(f) && !folio_test_uptodate(f)) {
 848                 ret = bch2_read_single_folio(f, mapping);
 849                 if (ret)
 850                         goto out;
 851         }
 852
 853         f = darray_last(fs);
 854         end = min(end, folio_end_pos(f));
 855         last_folio_pos = folio_pos(f);
 856         if (end != folio_end_pos(f) && !folio_test_uptodate(f)) {
 857                 if (end >= inode->v.i_size) {
 858                         folio_zero_range(f, 0, folio_size(f));
 859                 } else {
 860                         ret = bch2_read_single_folio(f, mapping);
 861                         if (ret)
 862                                 goto out;
 863                 }
 864         }
 865
 866         ret = bch2_folio_set(c, inode_inum(inode), fs.data, fs.nr);
 867         if (ret)
 868                 goto out;
 869
 870         f_pos = pos;
 871         f_offset = pos - folio_pos(darray_first(fs));
 872         darray_for_each(fs, fi) {
 873                 f = *fi;
 874                 f_len = min(end, folio_end_pos(f)) - f_pos;
 875
 876                 /*
 877                  * XXX: per POSIX and fstests generic/275, on -ENOSPC we're
 878                  * supposed to write as much as we have disk space for.
 879                  *
 880                  * On failure here we should still write out a partial page if
 881                  * we aren't completely out of disk space - we don't do that
 882                  * yet:
 883                  */
 884                 ret = bch2_folio_reservation_get(c, inode, f, &res, f_offset, f_len);
 885                 if (unlikely(ret)) {
 886                         folios_trunc(&fs, fi);
 887                         if (!fs.nr)
 888                                 goto out;
 889
 890                         end = min(end, folio_end_pos(darray_last(fs)));
 891                         break;
 892                 }
 893
 894                 f_pos = folio_end_pos(f);
 895                 f_offset = 0;
 896         }
 897
 898         if (mapping_writably_mapped(mapping))
 899                 darray_for_each(fs, fi)
 900                         flush_dcache_folio(*fi);
 901
 902         f_pos = pos;
 903         f_offset = pos - folio_pos(darray_first(fs));
 904         darray_for_each(fs, fi) {
 905                 f = *fi;
 906                 f_len = min(end, folio_end_pos(f)) - f_pos;
 907                 f_copied = copy_page_from_iter_atomic(&f->page, f_offset, f_len, iter);
 908                 if (!f_copied) {
 909                         folios_trunc(&fs, fi);
 910                         break;
 911                 }
 912
 913                 if (!folio_test_uptodate(f) &&
 914                     f_copied != folio_size(f) &&
 915                     pos + copied + f_copied < inode->v.i_size) {
 916                         iov_iter_revert(iter, f_copied);
 917                         folio_zero_range(f, 0, folio_size(f));
 918                         folios_trunc(&fs, fi);
 919                         break;
 920                 }
 921
 922                 flush_dcache_folio(f);
 923                 copied += f_copied;
 924
 925                 if (f_copied != f_len) {
 926                         folios_trunc(&fs, fi + 1);
 927                         break;
 928                 }
 929
 930                 f_pos = folio_end_pos(f);
 931                 f_offset = 0;
 932         }
 933
 934         if (!copied)
 935                 goto out;
 936
 937         end = pos + copied;
 938
 939         spin_lock(&inode->v.i_lock);
 940         if (end > inode->v.i_size)
 941                 i_size_write(&inode->v, end);
 942         spin_unlock(&inode->v.i_lock);
 943
 944         f_pos = pos;
 945         f_offset = pos - folio_pos(darray_first(fs));
 946         darray_for_each(fs, fi) {
 947                 f = *fi;
 948                 f_len = min(end, folio_end_pos(f)) - f_pos;
 949
 950                 if (!folio_test_uptodate(f))
 951                         folio_mark_uptodate(f);
 952
 953                 bch2_set_folio_dirty(c, inode, f, &res, f_offset, f_len);
 954
 955                 f_pos = folio_end_pos(f);
 956                 f_offset = 0;
 957         }
 958
 959         inode->ei_last_dirtied = (unsigned long) current;
 960 out:
 961         darray_for_each(fs, fi) {
 962                 folio_unlock(*fi);
 963                 folio_put(*fi);
 964         }
 965
 966         /*
 967          * If the last folio added to the mapping starts beyond current EOF, we
 968          * performed a short write but left around at least one post-EOF folio.
 969          * Clean up the mapping before we return.
 970          */
 971         if (last_folio_pos >= inode->v.i_size)
 972                 truncate_pagecache(&inode->v, inode->v.i_size);
 973
 974         darray_exit(&fs);
 975         bch2_folio_reservation_put(c, inode, &res);
 976
 977         return copied ?: ret;
 978 }
 979
 980 static ssize_t bch2_buffered_write(struct kiocb *iocb, struct iov_iter *iter)
 981 {
 982         struct file *file = iocb->ki_filp;
 983         struct address_space *mapping = file->f_mapping;
 984         struct bch_inode_info *inode = file_bch_inode(file);
 985         loff_t pos = iocb->ki_pos;
 986         ssize_t written = 0;
 987         int ret = 0;
 988
 989         bch2_pagecache_add_get(inode);
 990
 991         do {
 992                 unsigned offset = pos & (PAGE_SIZE - 1);
 993                 unsigned bytes = iov_iter_count(iter);
 994 again:
 995                 /*
 996                  * Bring in the user page that we will copy from _first_.
 997                  * Otherwise there's a nasty deadlock on copying from the
 998                  * same page as we're writing to, without it being marked
 999                  * up-to-date.
1000                  *
1001                  * Not only is this an optimisation, but it is also required
1002                  * to check that the address is actually valid, when atomic
1003                  * usercopies are used, below.
1004                  */
1005                 if (unlikely(fault_in_iov_iter_readable(iter, bytes))) {
1006                         bytes = min_t(unsigned long, iov_iter_count(iter),
1007                                       PAGE_SIZE - offset);
1008
1009                         if (unlikely(fault_in_iov_iter_readable(iter, bytes))) {
1010                                 ret = -EFAULT;
1011                                 break;
1012                         }
1013                 }
1014
1015                 if (unlikely(fatal_signal_pending(current))) {
1016                         ret = -EINTR;
1017                         break;
1018                 }
1019
1020                 ret = __bch2_buffered_write(inode, mapping, iter, pos, bytes);
1021                 if (unlikely(ret < 0))
1022                         break;
1023
1024                 cond_resched();
1025
1026                 if (unlikely(ret == 0)) {
1027                         /*
1028                          * If we were unable to copy any data at all, we must
1029                          * fall back to a single segment length write.
1030                          *
1031                          * If we didn't fallback here, we could livelock
1032                          * because not all segments in the iov can be copied at
1033                          * once without a pagefault.
1034                          */
1035                         bytes = min_t(unsigned long, PAGE_SIZE - offset,
1036                                       iov_iter_single_seg_count(iter));
1037                         goto again;
1038                 }
1039                 pos += ret;
1040                 written += ret;
1041                 ret = 0;
1042
1043                 balance_dirty_pages_ratelimited(mapping);
1044         } while (iov_iter_count(iter));
1045
1046         bch2_pagecache_add_put(inode);
1047
1048         return written ? written : ret;
1049 }
1050
1051 ssize_t bch2_write_iter(struct kiocb *iocb, struct iov_iter *from)
1052 {
1053         struct file *file = iocb->ki_filp;
1054         struct bch_inode_info *inode = file_bch_inode(file);
1055         ssize_t ret;
1056
1057         if (iocb->ki_flags & IOCB_DIRECT) {
1058                 ret = bch2_direct_write(iocb, from);
1059                 goto out;
1060         }
1061
1062         inode_lock(&inode->v);
1063
1064         ret = generic_write_checks(iocb, from);
1065         if (ret <= 0)
1066                 goto unlock;
1067
1068         ret = file_remove_privs(file);
1069         if (ret)
1070                 goto unlock;
1071
1072         ret = file_update_time(file);
1073         if (ret)
1074                 goto unlock;
1075
1076         ret = bch2_buffered_write(iocb, from);
1077         if (likely(ret > 0))
1078                 iocb->ki_pos += ret;
1079 unlock:
1080         inode_unlock(&inode->v);
1081
1082         if (ret > 0)
1083                 ret = generic_write_sync(iocb, ret);
1084 out:
1085         return bch2_err_class(ret);
1086 }
1087
1088 void bch2_fs_fs_io_buffered_exit(struct bch_fs *c)
1089 {
1090         bioset_exit(&c->writepage_bioset);
1091 }
1092
1093 int bch2_fs_fs_io_buffered_init(struct bch_fs *c)
1094 {
1095         if (bioset_init(&c->writepage_bioset,
1096                         4, offsetof(struct bch_writepage_io, op.wbio.bio),
1097                         BIOSET_NEED_BVECS))
1098                 return -BCH_ERR_ENOMEM_writepage_bioset_init;
1099
1100         return 0;
1101 }
1102
1103 #endif /* NO_BCACHEFS_FS */