git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/fs-io-buffered.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #ifndef NO_BCACHEFS_FS
   3
   4 #include "bcachefs.h"
   5 #include "alloc_foreground.h"
   6 #include "bkey_buf.h"
   7 #include "fs-io.h"
   8 #include "fs-io-buffered.h"
   9 #include "fs-io-direct.h"
  10 #include "fs-io-pagecache.h"
  11 #include "io.h"
  12
  13 #include <linux/backing-dev.h>
  14 #include <linux/pagemap.h>
  15 #include <linux/writeback.h>
  16
  17 static inline bool bio_full(struct bio *bio, unsigned len)
  18 {
  19         if (bio->bi_vcnt >= bio->bi_max_vecs)
  20                 return true;
  21         if (bio->bi_iter.bi_size > UINT_MAX - len)
  22                 return true;
  23         return false;
  24 }
  25
  26 /* readpage(s): */
  27
  28 static void bch2_readpages_end_io(struct bio *bio)
  29 {
  30         struct folio_iter fi;
  31
  32         bio_for_each_folio_all(fi, bio) {
  33                 if (!bio->bi_status) {
  34                         folio_mark_uptodate(fi.folio);
  35                 } else {
  36                         folio_clear_uptodate(fi.folio);
  37                         folio_set_error(fi.folio);
  38                 }
  39                 folio_unlock(fi.folio);
  40         }
  41
  42         bio_put(bio);
  43 }
  44
  45 struct readpages_iter {
  46         struct address_space    *mapping;
  47         unsigned                idx;
  48         folios                  folios;
  49 };
  50
  51 static int readpages_iter_init(struct readpages_iter *iter,
  52                                struct readahead_control *ractl)
  53 {
  54         struct folio **fi;
  55         int ret;
  56
  57         memset(iter, 0, sizeof(*iter));
  58
  59         iter->mapping = ractl->mapping;
  60
  61         ret = bch2_filemap_get_contig_folios_d(iter->mapping,
  62                                 ractl->_index << PAGE_SHIFT,
  63                                 (ractl->_index + ractl->_nr_pages) << PAGE_SHIFT,
  64                                 0, mapping_gfp_mask(iter->mapping),
  65                                 &iter->folios);
  66         if (ret)
  67                 return ret;
  68
  69         darray_for_each(iter->folios, fi) {
  70                 ractl->_nr_pages -= 1U << folio_order(*fi);
  71                 __bch2_folio_create(*fi, __GFP_NOFAIL|GFP_KERNEL);
  72                 folio_put(*fi);
  73                 folio_put(*fi);
  74         }
  75
  76         return 0;
  77 }
  78
  79 static inline struct folio *readpage_iter_peek(struct readpages_iter *iter)
  80 {
  81         if (iter->idx >= iter->folios.nr)
  82                 return NULL;
  83         return iter->folios.data[iter->idx];
  84 }
  85
  86 static inline void readpage_iter_advance(struct readpages_iter *iter)
  87 {
  88         iter->idx++;
  89 }
  90
  91 static bool extent_partial_reads_expensive(struct bkey_s_c k)
  92 {
  93         struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
  94         struct bch_extent_crc_unpacked crc;
  95         const union bch_extent_entry *i;
  96
  97         bkey_for_each_crc(k.k, ptrs, crc, i)
  98                 if (crc.csum_type || crc.compression_type)
  99                         return true;
 100         return false;
 101 }
 102
 103 static int readpage_bio_extend(struct btree_trans *trans,
 104                                struct readpages_iter *iter,
 105                                struct bio *bio,
 106                                unsigned sectors_this_extent,
 107                                bool get_more)
 108 {
 109         /* Don't hold btree locks while allocating memory: */
 110         bch2_trans_unlock(trans);
 111
 112         while (bio_sectors(bio) < sectors_this_extent &&
 113                bio->bi_vcnt < bio->bi_max_vecs) {
 114                 struct folio *folio = readpage_iter_peek(iter);
 115                 int ret;
 116
 117                 if (folio) {
 118                         readpage_iter_advance(iter);
 119                 } else {
 120                         pgoff_t folio_offset = bio_end_sector(bio) >> PAGE_SECTORS_SHIFT;
 121
 122                         if (!get_more)
 123                                 break;
 124
 125                         folio = xa_load(&iter->mapping->i_pages, folio_offset);
 126                         if (folio && !xa_is_value(folio))
 127                                 break;
 128
 129                         folio = filemap_alloc_folio(readahead_gfp_mask(iter->mapping), 0);
 130                         if (!folio)
 131                                 break;
 132
 133                         if (!__bch2_folio_create(folio, GFP_KERNEL)) {
 134                                 folio_put(folio);
 135                                 break;
 136                         }
 137
 138                         ret = filemap_add_folio(iter->mapping, folio, folio_offset, GFP_KERNEL);
 139                         if (ret) {
 140                                 __bch2_folio_release(folio);
 141                                 folio_put(folio);
 142                                 break;
 143                         }
 144
 145                         folio_put(folio);
 146                 }
 147
 148                 BUG_ON(folio_sector(folio) != bio_end_sector(bio));
 149
 150                 BUG_ON(!bio_add_folio(bio, folio, folio_size(folio), 0));
 151         }
 152
 153         return bch2_trans_relock(trans);
 154 }
 155
 156 static void bchfs_read(struct btree_trans *trans,
 157                        struct bch_read_bio *rbio,
 158                        subvol_inum inum,
 159                        struct readpages_iter *readpages_iter)
 160 {
 161         struct bch_fs *c = trans->c;
 162         struct btree_iter iter;
 163         struct bkey_buf sk;
 164         int flags = BCH_READ_RETRY_IF_STALE|
 165                 BCH_READ_MAY_PROMOTE;
 166         u32 snapshot;
 167         int ret = 0;
 168
 169         rbio->c = c;
 170         rbio->start_time = local_clock();
 171         rbio->subvol = inum.subvol;
 172
 173         bch2_bkey_buf_init(&sk);
 174 retry:
 175         bch2_trans_begin(trans);
 176         iter = (struct btree_iter) { NULL };
 177
 178         ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot);
 179         if (ret)
 180                 goto err;
 181
 182         bch2_trans_iter_init(trans, &iter, BTREE_ID_extents,
 183                              SPOS(inum.inum, rbio->bio.bi_iter.bi_sector, snapshot),
 184                              BTREE_ITER_SLOTS);
 185         while (1) {
 186                 struct bkey_s_c k;
 187                 unsigned bytes, sectors, offset_into_extent;
 188                 enum btree_id data_btree = BTREE_ID_extents;
 189
 190                 /*
 191                  * read_extent -> io_time_reset may cause a transaction restart
 192                  * without returning an error, we need to check for that here:
 193                  */
 194                 ret = bch2_trans_relock(trans);
 195                 if (ret)
 196                         break;
 197
 198                 bch2_btree_iter_set_pos(&iter,
 199                                 POS(inum.inum, rbio->bio.bi_iter.bi_sector));
 200
 201                 k = bch2_btree_iter_peek_slot(&iter);
 202                 ret = bkey_err(k);
 203                 if (ret)
 204                         break;
 205
 206                 offset_into_extent = iter.pos.offset -
 207                         bkey_start_offset(k.k);
 208                 sectors = k.k->size - offset_into_extent;
 209
 210                 bch2_bkey_buf_reassemble(&sk, c, k);
 211
 212                 ret = bch2_read_indirect_extent(trans, &data_btree,
 213                                         &offset_into_extent, &sk);
 214                 if (ret)
 215                         break;
 216
 217                 k = bkey_i_to_s_c(sk.k);
 218
 219                 sectors = min(sectors, k.k->size - offset_into_extent);
 220
 221                 if (readpages_iter) {
 222                         ret = readpage_bio_extend(trans, readpages_iter, &rbio->bio, sectors,
 223                                                   extent_partial_reads_expensive(k));
 224                         if (ret)
 225                                 break;
 226                 }
 227
 228                 bytes = min(sectors, bio_sectors(&rbio->bio)) << 9;
 229                 swap(rbio->bio.bi_iter.bi_size, bytes);
 230
 231                 if (rbio->bio.bi_iter.bi_size == bytes)
 232                         flags |= BCH_READ_LAST_FRAGMENT;
 233
 234                 bch2_bio_page_state_set(&rbio->bio, k);
 235
 236                 bch2_read_extent(trans, rbio, iter.pos,
 237                                  data_btree, k, offset_into_extent, flags);
 238
 239                 if (flags & BCH_READ_LAST_FRAGMENT)
 240                         break;
 241
 242                 swap(rbio->bio.bi_iter.bi_size, bytes);
 243                 bio_advance(&rbio->bio, bytes);
 244
 245                 ret = btree_trans_too_many_iters(trans);
 246                 if (ret)
 247                         break;
 248         }
 249 err:
 250         bch2_trans_iter_exit(trans, &iter);
 251
 252         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
 253                 goto retry;
 254
 255         if (ret) {
 256                 bch_err_inum_offset_ratelimited(c,
 257                                 iter.pos.inode,
 258                                 iter.pos.offset << 9,
 259                                 "read error %i from btree lookup", ret);
 260                 rbio->bio.bi_status = BLK_STS_IOERR;
 261                 bio_endio(&rbio->bio);
 262         }
 263
 264         bch2_bkey_buf_exit(&sk, c);
 265 }
 266
 267 void bch2_readahead(struct readahead_control *ractl)
 268 {
 269         struct bch_inode_info *inode = to_bch_ei(ractl->mapping->host);
 270         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 271         struct bch_io_opts opts;
 272         struct btree_trans trans;
 273         struct folio *folio;
 274         struct readpages_iter readpages_iter;
 275         int ret;
 276
 277         bch2_inode_opts_get(&opts, c, &inode->ei_inode);
 278
 279         ret = readpages_iter_init(&readpages_iter, ractl);
 280         BUG_ON(ret);
 281
 282         bch2_trans_init(&trans, c, 0, 0);
 283
 284         bch2_pagecache_add_get(inode);
 285
 286         while ((folio = readpage_iter_peek(&readpages_iter))) {
 287                 unsigned n = min_t(unsigned,
 288                                    readpages_iter.folios.nr -
 289                                    readpages_iter.idx,
 290                                    BIO_MAX_VECS);
 291                 struct bch_read_bio *rbio =
 292                         rbio_init(bio_alloc_bioset(NULL, n, REQ_OP_READ,
 293                                                    GFP_KERNEL, &c->bio_read),
 294                                   opts);
 295
 296                 readpage_iter_advance(&readpages_iter);
 297
 298                 rbio->bio.bi_iter.bi_sector = folio_sector(folio);
 299                 rbio->bio.bi_end_io = bch2_readpages_end_io;
 300                 BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0));
 301
 302                 bchfs_read(&trans, rbio, inode_inum(inode),
 303                            &readpages_iter);
 304                 bch2_trans_unlock(&trans);
 305         }
 306
 307         bch2_pagecache_add_put(inode);
 308
 309         bch2_trans_exit(&trans);
 310         darray_exit(&readpages_iter.folios);
 311 }
 312
 313 static void __bchfs_readfolio(struct bch_fs *c, struct bch_read_bio *rbio,
 314                              subvol_inum inum, struct folio *folio)
 315 {
 316         struct btree_trans trans;
 317
 318         bch2_folio_create(folio, __GFP_NOFAIL);
 319
 320         rbio->bio.bi_opf = REQ_OP_READ|REQ_SYNC;
 321         rbio->bio.bi_iter.bi_sector = folio_sector(folio);
 322         BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0));
 323
 324         bch2_trans_init(&trans, c, 0, 0);
 325         bchfs_read(&trans, rbio, inum, NULL);
 326         bch2_trans_exit(&trans);
 327 }
 328
 329 static void bch2_read_single_folio_end_io(struct bio *bio)
 330 {
 331         complete(bio->bi_private);
 332 }
 333
 334 int bch2_read_single_folio(struct folio *folio, struct address_space *mapping)
 335 {
 336         struct bch_inode_info *inode = to_bch_ei(mapping->host);
 337         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 338         struct bch_read_bio *rbio;
 339         struct bch_io_opts opts;
 340         int ret;
 341         DECLARE_COMPLETION_ONSTACK(done);
 342
 343         bch2_inode_opts_get(&opts, c, &inode->ei_inode);
 344
 345         rbio = rbio_init(bio_alloc_bioset(NULL, 1, REQ_OP_READ, GFP_KERNEL, &c->bio_read),
 346                          opts);
 347         rbio->bio.bi_private = &done;
 348         rbio->bio.bi_end_io = bch2_read_single_folio_end_io;
 349
 350         __bchfs_readfolio(c, rbio, inode_inum(inode), folio);
 351         wait_for_completion(&done);
 352
 353         ret = blk_status_to_errno(rbio->bio.bi_status);
 354         bio_put(&rbio->bio);
 355
 356         if (ret < 0)
 357                 return ret;
 358
 359         folio_mark_uptodate(folio);
 360         return 0;
 361 }
 362
 363 int bch2_read_folio(struct file *file, struct folio *folio)
 364 {
 365         int ret;
 366
 367         ret = bch2_read_single_folio(folio, folio->mapping);
 368         folio_unlock(folio);
 369         return bch2_err_class(ret);
 370 }
 371
 372 /* writepages: */
 373
 374 struct bch_writepage_io {
 375         struct bch_inode_info           *inode;
 376
 377         /* must be last: */
 378         struct bch_write_op             op;
 379 };
 380
 381 struct bch_writepage_state {
 382         struct bch_writepage_io *io;
 383         struct bch_io_opts      opts;
 384         struct bch_folio_sector *tmp;
 385         unsigned                tmp_sectors;
 386 };
 387
 388 static inline struct bch_writepage_state bch_writepage_state_init(struct bch_fs *c,
 389                                                                   struct bch_inode_info *inode)
 390 {
 391         struct bch_writepage_state ret = { 0 };
 392
 393         bch2_inode_opts_get(&ret.opts, c, &inode->ei_inode);
 394         return ret;
 395 }
 396
 397 static void bch2_writepage_io_done(struct bch_write_op *op)
 398 {
 399         struct bch_writepage_io *io =
 400                 container_of(op, struct bch_writepage_io, op);
 401         struct bch_fs *c = io->op.c;
 402         struct bio *bio = &io->op.wbio.bio;
 403         struct folio_iter fi;
 404         unsigned i;
 405
 406         if (io->op.error) {
 407                 set_bit(EI_INODE_ERROR, &io->inode->ei_flags);
 408
 409                 bio_for_each_folio_all(fi, bio) {
 410                         struct bch_folio *s;
 411
 412                         folio_set_error(fi.folio);
 413                         mapping_set_error(fi.folio->mapping, -EIO);
 414
 415                         s = __bch2_folio(fi.folio);
 416                         spin_lock(&s->lock);
 417                         for (i = 0; i < folio_sectors(fi.folio); i++)
 418                                 s->s[i].nr_replicas = 0;
 419                         spin_unlock(&s->lock);
 420                 }
 421         }
 422
 423         if (io->op.flags & BCH_WRITE_WROTE_DATA_INLINE) {
 424                 bio_for_each_folio_all(fi, bio) {
 425                         struct bch_folio *s;
 426
 427                         s = __bch2_folio(fi.folio);
 428                         spin_lock(&s->lock);
 429                         for (i = 0; i < folio_sectors(fi.folio); i++)
 430                                 s->s[i].nr_replicas = 0;
 431                         spin_unlock(&s->lock);
 432                 }
 433         }
 434
 435         /*
 436          * racing with fallocate can cause us to add fewer sectors than
 437          * expected - but we shouldn't add more sectors than expected:
 438          */
 439         WARN_ON_ONCE(io->op.i_sectors_delta > 0);
 440
 441         /*
 442          * (error (due to going RO) halfway through a page can screw that up
 443          * slightly)
 444          * XXX wtf?
 445            BUG_ON(io->op.op.i_sectors_delta >= PAGE_SECTORS);
 446          */
 447
 448         /*
 449          * PageWriteback is effectively our ref on the inode - fixup i_blocks
 450          * before calling end_page_writeback:
 451          */
 452         bch2_i_sectors_acct(c, io->inode, NULL, io->op.i_sectors_delta);
 453
 454         bio_for_each_folio_all(fi, bio) {
 455                 struct bch_folio *s = __bch2_folio(fi.folio);
 456
 457                 if (atomic_dec_and_test(&s->write_count))
 458                         folio_end_writeback(fi.folio);
 459         }
 460
 461         bio_put(&io->op.wbio.bio);
 462 }
 463
 464 static void bch2_writepage_do_io(struct bch_writepage_state *w)
 465 {
 466         struct bch_writepage_io *io = w->io;
 467
 468         w->io = NULL;
 469         closure_call(&io->op.cl, bch2_write, NULL, NULL);
 470 }
 471
 472 /*
 473  * Get a bch_writepage_io and add @page to it - appending to an existing one if
 474  * possible, else allocating a new one:
 475  */
 476 static void bch2_writepage_io_alloc(struct bch_fs *c,
 477                                     struct writeback_control *wbc,
 478                                     struct bch_writepage_state *w,
 479                                     struct bch_inode_info *inode,
 480                                     u64 sector,
 481                                     unsigned nr_replicas)
 482 {
 483         struct bch_write_op *op;
 484
 485         w->io = container_of(bio_alloc_bioset(NULL, BIO_MAX_VECS,
 486                                               REQ_OP_WRITE,
 487                                               GFP_KERNEL,
 488                                               &c->writepage_bioset),
 489                              struct bch_writepage_io, op.wbio.bio);
 490
 491         w->io->inode            = inode;
 492         op                      = &w->io->op;
 493         bch2_write_op_init(op, c, w->opts);
 494         op->target              = w->opts.foreground_target;
 495         op->nr_replicas         = nr_replicas;
 496         op->res.nr_replicas     = nr_replicas;
 497         op->write_point         = writepoint_hashed(inode->ei_last_dirtied);
 498         op->subvol              = inode->ei_subvol;
 499         op->pos                 = POS(inode->v.i_ino, sector);
 500         op->end_io              = bch2_writepage_io_done;
 501         op->devs_need_flush     = &inode->ei_devs_need_flush;
 502         op->wbio.bio.bi_iter.bi_sector = sector;
 503         op->wbio.bio.bi_opf     = wbc_to_write_flags(wbc);
 504 }
 505
 506 static int __bch2_writepage(struct folio *folio,
 507                             struct writeback_control *wbc,
 508                             void *data)
 509 {
 510         struct bch_inode_info *inode = to_bch_ei(folio->mapping->host);
 511         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 512         struct bch_writepage_state *w = data;
 513         struct bch_folio *s;
 514         unsigned i, offset, f_sectors, nr_replicas_this_write = U32_MAX;
 515         loff_t i_size = i_size_read(&inode->v);
 516         int ret;
 517
 518         EBUG_ON(!folio_test_uptodate(folio));
 519
 520         /* Is the folio fully inside i_size? */
 521         if (folio_end_pos(folio) <= i_size)
 522                 goto do_io;
 523
 524         /* Is the folio fully outside i_size? (truncate in progress) */
 525         if (folio_pos(folio) >= i_size) {
 526                 folio_unlock(folio);
 527                 return 0;
 528         }
 529
 530         /*
 531          * The folio straddles i_size.  It must be zeroed out on each and every
 532          * writepage invocation because it may be mmapped.  "A file is mapped
 533          * in multiples of the folio size.  For a file that is not a multiple of
 534          * the  folio size, the remaining memory is zeroed when mapped, and
 535          * writes to that region are not written out to the file."
 536          */
 537         folio_zero_segment(folio,
 538                            i_size - folio_pos(folio),
 539                            folio_size(folio));
 540 do_io:
 541         f_sectors = folio_sectors(folio);
 542         s = bch2_folio(folio);
 543
 544         if (f_sectors > w->tmp_sectors) {
 545                 kfree(w->tmp);
 546                 w->tmp = kcalloc(f_sectors, sizeof(struct bch_folio_sector), __GFP_NOFAIL);
 547                 w->tmp_sectors = f_sectors;
 548         }
 549
 550         /*
 551          * Things get really hairy with errors during writeback:
 552          */
 553         ret = bch2_get_folio_disk_reservation(c, inode, folio, false);
 554         BUG_ON(ret);
 555
 556         /* Before unlocking the page, get copy of reservations: */
 557         spin_lock(&s->lock);
 558         memcpy(w->tmp, s->s, sizeof(struct bch_folio_sector) * f_sectors);
 559
 560         for (i = 0; i < f_sectors; i++) {
 561                 if (s->s[i].state < SECTOR_dirty)
 562                         continue;
 563
 564                 nr_replicas_this_write =
 565                         min_t(unsigned, nr_replicas_this_write,
 566                               s->s[i].nr_replicas +
 567                               s->s[i].replicas_reserved);
 568         }
 569
 570         for (i = 0; i < f_sectors; i++) {
 571                 if (s->s[i].state < SECTOR_dirty)
 572                         continue;
 573
 574                 s->s[i].nr_replicas = w->opts.compression
 575                         ? 0 : nr_replicas_this_write;
 576
 577                 s->s[i].replicas_reserved = 0;
 578                 bch2_folio_sector_set(folio, s, i, SECTOR_allocated);
 579         }
 580         spin_unlock(&s->lock);
 581
 582         BUG_ON(atomic_read(&s->write_count));
 583         atomic_set(&s->write_count, 1);
 584
 585         BUG_ON(folio_test_writeback(folio));
 586         folio_start_writeback(folio);
 587
 588         folio_unlock(folio);
 589
 590         offset = 0;
 591         while (1) {
 592                 unsigned sectors = 0, dirty_sectors = 0, reserved_sectors = 0;
 593                 u64 sector;
 594
 595                 while (offset < f_sectors &&
 596                        w->tmp[offset].state < SECTOR_dirty)
 597                         offset++;
 598
 599                 if (offset == f_sectors)
 600                         break;
 601
 602                 while (offset + sectors < f_sectors &&
 603                        w->tmp[offset + sectors].state >= SECTOR_dirty) {
 604                         reserved_sectors += w->tmp[offset + sectors].replicas_reserved;
 605                         dirty_sectors += w->tmp[offset + sectors].state == SECTOR_dirty;
 606                         sectors++;
 607                 }
 608                 BUG_ON(!sectors);
 609
 610                 sector = folio_sector(folio) + offset;
 611
 612                 if (w->io &&
 613                     (w->io->op.res.nr_replicas != nr_replicas_this_write ||
 614                      bio_full(&w->io->op.wbio.bio, sectors << 9) ||
 615                      w->io->op.wbio.bio.bi_iter.bi_size + (sectors << 9) >=
 616                      (BIO_MAX_VECS * PAGE_SIZE) ||
 617                      bio_end_sector(&w->io->op.wbio.bio) != sector))
 618                         bch2_writepage_do_io(w);
 619
 620                 if (!w->io)
 621                         bch2_writepage_io_alloc(c, wbc, w, inode, sector,
 622                                                 nr_replicas_this_write);
 623
 624                 atomic_inc(&s->write_count);
 625
 626                 BUG_ON(inode != w->io->inode);
 627                 BUG_ON(!bio_add_folio(&w->io->op.wbio.bio, folio,
 628                                      sectors << 9, offset << 9));
 629
 630                 /* Check for writing past i_size: */
 631                 WARN_ONCE((bio_end_sector(&w->io->op.wbio.bio) << 9) >
 632                           round_up(i_size, block_bytes(c)) &&
 633                           !test_bit(BCH_FS_EMERGENCY_RO, &c->flags),
 634                           "writing past i_size: %llu > %llu (unrounded %llu)\n",
 635                           bio_end_sector(&w->io->op.wbio.bio) << 9,
 636                           round_up(i_size, block_bytes(c)),
 637                           i_size);
 638
 639                 w->io->op.res.sectors += reserved_sectors;
 640                 w->io->op.i_sectors_delta -= dirty_sectors;
 641                 w->io->op.new_i_size = i_size;
 642
 643                 offset += sectors;
 644         }
 645
 646         if (atomic_dec_and_test(&s->write_count))
 647                 folio_end_writeback(folio);
 648
 649         return 0;
 650 }
 651
 652 int bch2_writepages(struct address_space *mapping, struct writeback_control *wbc)
 653 {
 654         struct bch_fs *c = mapping->host->i_sb->s_fs_info;
 655         struct bch_writepage_state w =
 656                 bch_writepage_state_init(c, to_bch_ei(mapping->host));
 657         struct blk_plug plug;
 658         int ret;
 659
 660         blk_start_plug(&plug);
 661         ret = write_cache_pages(mapping, wbc, __bch2_writepage, &w);
 662         if (w.io)
 663                 bch2_writepage_do_io(&w);
 664         blk_finish_plug(&plug);
 665         kfree(w.tmp);
 666         return bch2_err_class(ret);
 667 }
 668
 669 /* buffered writes: */
 670
 671 int bch2_write_begin(struct file *file, struct address_space *mapping,
 672                      loff_t pos, unsigned len,
 673                      struct page **pagep, void **fsdata)
 674 {
 675         struct bch_inode_info *inode = to_bch_ei(mapping->host);
 676         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 677         struct bch2_folio_reservation *res;
 678         struct folio *folio;
 679         unsigned offset;
 680         int ret = -ENOMEM;
 681
 682         res = kmalloc(sizeof(*res), GFP_KERNEL);
 683         if (!res)
 684                 return -ENOMEM;
 685
 686         bch2_folio_reservation_init(c, inode, res);
 687         *fsdata = res;
 688
 689         bch2_pagecache_add_get(inode);
 690
 691         folio = __filemap_get_folio(mapping, pos >> PAGE_SHIFT,
 692                                 FGP_LOCK|FGP_WRITE|FGP_CREAT|FGP_STABLE,
 693                                 mapping_gfp_mask(mapping));
 694         if (IS_ERR_OR_NULL(folio))
 695                 goto err_unlock;
 696
 697         if (folio_test_uptodate(folio))
 698                 goto out;
 699
 700         offset = pos - folio_pos(folio);
 701         len = min_t(size_t, len, folio_end_pos(folio) - pos);
 702
 703         /* If we're writing entire folio, don't need to read it in first: */
 704         if (!offset && len == folio_size(folio))
 705                 goto out;
 706
 707         if (!offset && pos + len >= inode->v.i_size) {
 708                 folio_zero_segment(folio, len, folio_size(folio));
 709                 flush_dcache_folio(folio);
 710                 goto out;
 711         }
 712
 713         if (folio_pos(folio) >= inode->v.i_size) {
 714                 folio_zero_segments(folio, 0, offset, offset + len, folio_size(folio));
 715                 flush_dcache_folio(folio);
 716                 goto out;
 717         }
 718 readpage:
 719         ret = bch2_read_single_folio(folio, mapping);
 720         if (ret)
 721                 goto err;
 722 out:
 723         ret = bch2_folio_set(c, inode_inum(inode), &folio, 1);
 724         if (ret)
 725                 goto err;
 726
 727         ret = bch2_folio_reservation_get(c, inode, folio, res, offset, len);
 728         if (ret) {
 729                 if (!folio_test_uptodate(folio)) {
 730                         /*
 731                          * If the folio hasn't been read in, we won't know if we
 732                          * actually need a reservation - we don't actually need
 733                          * to read here, we just need to check if the folio is
 734                          * fully backed by uncompressed data:
 735                          */
 736                         goto readpage;
 737                 }
 738
 739                 goto err;
 740         }
 741
 742         *pagep = &folio->page;
 743         return 0;
 744 err:
 745         folio_unlock(folio);
 746         folio_put(folio);
 747         *pagep = NULL;
 748 err_unlock:
 749         bch2_pagecache_add_put(inode);
 750         kfree(res);
 751         *fsdata = NULL;
 752         return bch2_err_class(ret);
 753 }
 754
 755 int bch2_write_end(struct file *file, struct address_space *mapping,
 756                    loff_t pos, unsigned len, unsigned copied,
 757                    struct page *page, void *fsdata)
 758 {
 759         struct bch_inode_info *inode = to_bch_ei(mapping->host);
 760         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 761         struct bch2_folio_reservation *res = fsdata;
 762         struct folio *folio = page_folio(page);
 763         unsigned offset = pos - folio_pos(folio);
 764
 765         lockdep_assert_held(&inode->v.i_rwsem);
 766         BUG_ON(offset + copied > folio_size(folio));
 767
 768         if (unlikely(copied < len && !folio_test_uptodate(folio))) {
 769                 /*
 770                  * The folio needs to be read in, but that would destroy
 771                  * our partial write - simplest thing is to just force
 772                  * userspace to redo the write:
 773                  */
 774                 folio_zero_range(folio, 0, folio_size(folio));
 775                 flush_dcache_folio(folio);
 776                 copied = 0;
 777         }
 778
 779         spin_lock(&inode->v.i_lock);
 780         if (pos + copied > inode->v.i_size)
 781                 i_size_write(&inode->v, pos + copied);
 782         spin_unlock(&inode->v.i_lock);
 783
 784         if (copied) {
 785                 if (!folio_test_uptodate(folio))
 786                         folio_mark_uptodate(folio);
 787
 788                 bch2_set_folio_dirty(c, inode, folio, res, offset, copied);
 789
 790                 inode->ei_last_dirtied = (unsigned long) current;
 791         }
 792
 793         folio_unlock(folio);
 794         folio_put(folio);
 795         bch2_pagecache_add_put(inode);
 796
 797         bch2_folio_reservation_put(c, inode, res);
 798         kfree(res);
 799
 800         return copied;
 801 }
 802
 803 static noinline void folios_trunc(folios *folios, struct folio **fi)
 804 {
 805         while (folios->data + folios->nr > fi) {
 806                 struct folio *f = darray_pop(folios);
 807
 808                 folio_unlock(f);
 809                 folio_put(f);
 810         }
 811 }
 812
 813 static int __bch2_buffered_write(struct bch_inode_info *inode,
 814                                  struct address_space *mapping,
 815                                  struct iov_iter *iter,
 816                                  loff_t pos, unsigned len)
 817 {
 818         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 819         struct bch2_folio_reservation res;
 820         folios folios;
 821         struct folio **fi, *f;
 822         unsigned copied = 0, f_offset;
 823         u64 end = pos + len, f_pos;
 824         loff_t last_folio_pos = inode->v.i_size;
 825         int ret = 0;
 826
 827         BUG_ON(!len);
 828
 829         bch2_folio_reservation_init(c, inode, &res);
 830         darray_init(&folios);
 831
 832         ret = bch2_filemap_get_contig_folios_d(mapping, pos, end,
 833                                    FGP_LOCK|FGP_WRITE|FGP_STABLE|FGP_CREAT,
 834                                    mapping_gfp_mask(mapping),
 835                                    &folios);
 836         if (ret)
 837                 goto out;
 838
 839         BUG_ON(!folios.nr);
 840
 841         f = darray_first(folios);
 842         if (pos != folio_pos(f) && !folio_test_uptodate(f)) {
 843                 ret = bch2_read_single_folio(f, mapping);
 844                 if (ret)
 845                         goto out;
 846         }
 847
 848         f = darray_last(folios);
 849         end = min(end, folio_end_pos(f));
 850         last_folio_pos = folio_pos(f);
 851         if (end != folio_end_pos(f) && !folio_test_uptodate(f)) {
 852                 if (end >= inode->v.i_size) {
 853                         folio_zero_range(f, 0, folio_size(f));
 854                 } else {
 855                         ret = bch2_read_single_folio(f, mapping);
 856                         if (ret)
 857                                 goto out;
 858                 }
 859         }
 860
 861         ret = bch2_folio_set(c, inode_inum(inode), folios.data, folios.nr);
 862         if (ret)
 863                 goto out;
 864
 865         f_pos = pos;
 866         f_offset = pos - folio_pos(darray_first(folios));
 867         darray_for_each(folios, fi) {
 868                 struct folio *f = *fi;
 869                 u64 f_len = min(end, folio_end_pos(f)) - f_pos;
 870
 871                 /*
 872                  * XXX: per POSIX and fstests generic/275, on -ENOSPC we're
 873                  * supposed to write as much as we have disk space for.
 874                  *
 875                  * On failure here we should still write out a partial page if
 876                  * we aren't completely out of disk space - we don't do that
 877                  * yet:
 878                  */
 879                 ret = bch2_folio_reservation_get(c, inode, f, &res, f_offset, f_len);
 880                 if (unlikely(ret)) {
 881                         folios_trunc(&folios, fi);
 882                         if (!folios.nr)
 883                                 goto out;
 884
 885                         end = min(end, folio_end_pos(darray_last(folios)));
 886                         break;
 887                 }
 888
 889                 f_pos = folio_end_pos(f);
 890                 f_offset = 0;
 891         }
 892
 893         if (mapping_writably_mapped(mapping))
 894                 darray_for_each(folios, fi)
 895                         flush_dcache_folio(*fi);
 896
 897         f_pos = pos;
 898         f_offset = pos - folio_pos(darray_first(folios));
 899         darray_for_each(folios, fi) {
 900                 struct folio *f = *fi;
 901                 u64 f_len = min(end, folio_end_pos(f)) - f_pos;
 902                 unsigned f_copied = copy_page_from_iter_atomic(&f->page, f_offset, f_len, iter);
 903
 904                 if (!f_copied) {
 905                         folios_trunc(&folios, fi);
 906                         break;
 907                 }
 908
 909                 if (!folio_test_uptodate(f) &&
 910                     f_copied != folio_size(f) &&
 911                     pos + copied + f_copied < inode->v.i_size) {
 912                         iov_iter_revert(iter, f_copied);
 913                         folio_zero_range(f, 0, folio_size(f));
 914                         folios_trunc(&folios, fi);
 915                         break;
 916                 }
 917
 918                 flush_dcache_folio(f);
 919                 copied += f_copied;
 920
 921                 if (f_copied != f_len) {
 922                         folios_trunc(&folios, fi + 1);
 923                         break;
 924                 }
 925
 926                 f_pos = folio_end_pos(f);
 927                 f_offset = 0;
 928         }
 929
 930         if (!copied)
 931                 goto out;
 932
 933         end = pos + copied;
 934
 935         spin_lock(&inode->v.i_lock);
 936         if (end > inode->v.i_size)
 937                 i_size_write(&inode->v, end);
 938         spin_unlock(&inode->v.i_lock);
 939
 940         f_pos = pos;
 941         f_offset = pos - folio_pos(darray_first(folios));
 942         darray_for_each(folios, fi) {
 943                 struct folio *f = *fi;
 944                 u64 f_len = min(end, folio_end_pos(f)) - f_pos;
 945
 946                 if (!folio_test_uptodate(f))
 947                         folio_mark_uptodate(f);
 948
 949                 bch2_set_folio_dirty(c, inode, f, &res, f_offset, f_len);
 950
 951                 f_pos = folio_end_pos(f);
 952                 f_offset = 0;
 953         }
 954
 955         inode->ei_last_dirtied = (unsigned long) current;
 956 out:
 957         darray_for_each(folios, fi) {
 958                 folio_unlock(*fi);
 959                 folio_put(*fi);
 960         }
 961
 962         /*
 963          * If the last folio added to the mapping starts beyond current EOF, we
 964          * performed a short write but left around at least one post-EOF folio.
 965          * Clean up the mapping before we return.
 966          */
 967         if (last_folio_pos >= inode->v.i_size)
 968                 truncate_pagecache(&inode->v, inode->v.i_size);
 969
 970         darray_exit(&folios);
 971         bch2_folio_reservation_put(c, inode, &res);
 972
 973         return copied ?: ret;
 974 }
 975
 976 static ssize_t bch2_buffered_write(struct kiocb *iocb, struct iov_iter *iter)
 977 {
 978         struct file *file = iocb->ki_filp;
 979         struct address_space *mapping = file->f_mapping;
 980         struct bch_inode_info *inode = file_bch_inode(file);
 981         loff_t pos = iocb->ki_pos;
 982         ssize_t written = 0;
 983         int ret = 0;
 984
 985         bch2_pagecache_add_get(inode);
 986
 987         do {
 988                 unsigned offset = pos & (PAGE_SIZE - 1);
 989                 unsigned bytes = iov_iter_count(iter);
 990 again:
 991                 /*
 992                  * Bring in the user page that we will copy from _first_.
 993                  * Otherwise there's a nasty deadlock on copying from the
 994                  * same page as we're writing to, without it being marked
 995                  * up-to-date.
 996                  *
 997                  * Not only is this an optimisation, but it is also required
 998                  * to check that the address is actually valid, when atomic
 999                  * usercopies are used, below.
1000                  */
1001                 if (unlikely(fault_in_iov_iter_readable(iter, bytes))) {
1002                         bytes = min_t(unsigned long, iov_iter_count(iter),
1003                                       PAGE_SIZE - offset);
1004
1005                         if (unlikely(fault_in_iov_iter_readable(iter, bytes))) {
1006                                 ret = -EFAULT;
1007                                 break;
1008                         }
1009                 }
1010
1011                 if (unlikely(fatal_signal_pending(current))) {
1012                         ret = -EINTR;
1013                         break;
1014                 }
1015
1016                 ret = __bch2_buffered_write(inode, mapping, iter, pos, bytes);
1017                 if (unlikely(ret < 0))
1018                         break;
1019
1020                 cond_resched();
1021
1022                 if (unlikely(ret == 0)) {
1023                         /*
1024                          * If we were unable to copy any data at all, we must
1025                          * fall back to a single segment length write.
1026                          *
1027                          * If we didn't fallback here, we could livelock
1028                          * because not all segments in the iov can be copied at
1029                          * once without a pagefault.
1030                          */
1031                         bytes = min_t(unsigned long, PAGE_SIZE - offset,
1032                                       iov_iter_single_seg_count(iter));
1033                         goto again;
1034                 }
1035                 pos += ret;
1036                 written += ret;
1037                 ret = 0;
1038
1039                 balance_dirty_pages_ratelimited(mapping);
1040         } while (iov_iter_count(iter));
1041
1042         bch2_pagecache_add_put(inode);
1043
1044         return written ? written : ret;
1045 }
1046
1047 ssize_t bch2_write_iter(struct kiocb *iocb, struct iov_iter *from)
1048 {
1049         struct file *file = iocb->ki_filp;
1050         struct bch_inode_info *inode = file_bch_inode(file);
1051         ssize_t ret;
1052
1053         if (iocb->ki_flags & IOCB_DIRECT) {
1054                 ret = bch2_direct_write(iocb, from);
1055                 goto out;
1056         }
1057
1058         /* We can write back this queue in page reclaim */
1059         current->backing_dev_info = inode_to_bdi(&inode->v);
1060         inode_lock(&inode->v);
1061
1062         ret = generic_write_checks(iocb, from);
1063         if (ret <= 0)
1064                 goto unlock;
1065
1066         ret = file_remove_privs(file);
1067         if (ret)
1068                 goto unlock;
1069
1070         ret = file_update_time(file);
1071         if (ret)
1072                 goto unlock;
1073
1074         ret = bch2_buffered_write(iocb, from);
1075         if (likely(ret > 0))
1076                 iocb->ki_pos += ret;
1077 unlock:
1078         inode_unlock(&inode->v);
1079         current->backing_dev_info = NULL;
1080
1081         if (ret > 0)
1082                 ret = generic_write_sync(iocb, ret);
1083 out:
1084         return bch2_err_class(ret);
1085 }
1086
1087 void bch2_fs_fs_io_buffered_exit(struct bch_fs *c)
1088 {
1089         bioset_exit(&c->writepage_bioset);
1090 }
1091
1092 int bch2_fs_fs_io_buffered_init(struct bch_fs *c)
1093 {
1094         if (bioset_init(&c->writepage_bioset,
1095                         4, offsetof(struct bch_writepage_io, op.wbio.bio),
1096                         BIOSET_NEED_BVECS))
1097                 return -BCH_ERR_ENOMEM_writepage_bioset_init;
1098
1099         return 0;
1100 }
1101
1102 #endif /* NO_BCACHEFS_FS */