git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/fs-io.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #ifndef NO_BCACHEFS_FS
   3
   4 #include "bcachefs.h"
   5 #include "alloc_foreground.h"
   6 #include "bkey_buf.h"
   7 #include "btree_update.h"
   8 #include "buckets.h"
   9 #include "clock.h"
  10 #include "error.h"
  11 #include "extents.h"
  12 #include "extent_update.h"
  13 #include "fs.h"
  14 #include "fs-io.h"
  15 #include "fsck.h"
  16 #include "inode.h"
  17 #include "journal.h"
  18 #include "io.h"
  19 #include "keylist.h"
  20 #include "quota.h"
  21 #include "reflink.h"
  22 #include "trace.h"
  23
  24 #include <linux/aio.h>
  25 #include <linux/backing-dev.h>
  26 #include <linux/falloc.h>
  27 #include <linux/migrate.h>
  28 #include <linux/mmu_context.h>
  29 #include <linux/pagevec.h>
  30 #include <linux/rmap.h>
  31 #include <linux/sched/signal.h>
  32 #include <linux/task_io_accounting_ops.h>
  33 #include <linux/uio.h>
  34 #include <linux/writeback.h>
  35
  36 #include <trace/events/writeback.h>
  37
  38 static int bch2_clamp_data_hole(struct inode *, u64 *, u64 *, unsigned, bool);
  39
  40 struct folio_vec {
  41         struct folio    *fv_folio;
  42         size_t          fv_offset;
  43         size_t          fv_len;
  44 };
  45
  46 static inline struct folio_vec biovec_to_foliovec(struct bio_vec bv)
  47 {
  48
  49         struct folio *folio     = page_folio(bv.bv_page);
  50         size_t offset           = (folio_page_idx(folio, bv.bv_page) << PAGE_SHIFT) +
  51                 bv.bv_offset;
  52         size_t len = min_t(size_t, folio_size(folio) - offset, bv.bv_len);
  53
  54         return (struct folio_vec) {
  55                 .fv_folio       = folio,
  56                 .fv_offset      = offset,
  57                 .fv_len         = len,
  58         };
  59 }
  60
  61 static inline struct folio_vec bio_iter_iovec_folio(struct bio *bio,
  62                                                     struct bvec_iter iter)
  63 {
  64         return biovec_to_foliovec(bio_iter_iovec(bio, iter));
  65 }
  66
  67 #define __bio_for_each_folio(bvl, bio, iter, start)                     \
  68         for (iter = (start);                                            \
  69              (iter).bi_size &&                                          \
  70                 ((bvl = bio_iter_iovec_folio((bio), (iter))), 1);       \
  71              bio_advance_iter_single((bio), &(iter), (bvl).fv_len))
  72
  73 /**
  74  * bio_for_each_folio - iterate over folios within a bio
  75  *
  76  * Like other non-_all versions, this iterates over what bio->bi_iter currently
  77  * points to. This version is for drivers, where the bio may have previously
  78  * been split or cloned.
  79  */
  80 #define bio_for_each_folio(bvl, bio, iter)                              \
  81         __bio_for_each_folio(bvl, bio, iter, (bio)->bi_iter)
  82
  83 /*
  84  * Use u64 for the end pos and sector helpers because if the folio covers the
  85  * max supported range of the mapping, the start offset of the next folio
  86  * overflows loff_t. This breaks much of the range based processing in the
  87  * buffered write path.
  88  */
  89 static inline u64 folio_end_pos(struct folio *folio)
  90 {
  91         return folio_pos(folio) + folio_size(folio);
  92 }
  93
  94 static inline size_t folio_sectors(struct folio *folio)
  95 {
  96         return PAGE_SECTORS << folio_order(folio);
  97 }
  98
  99 static inline loff_t folio_sector(struct folio *folio)
 100 {
 101         return folio_pos(folio) >> 9;
 102 }
 103
 104 static inline u64 folio_end_sector(struct folio *folio)
 105 {
 106         return folio_end_pos(folio) >> 9;
 107 }
 108
 109 typedef DARRAY(struct folio *) folios;
 110
 111 static int filemap_get_contig_folios_d(struct address_space *mapping,
 112                                        loff_t start, u64 end,
 113                                        int fgp_flags, gfp_t gfp,
 114                                        folios *folios)
 115 {
 116         struct folio *f;
 117         u64 pos = start;
 118         int ret = 0;
 119
 120         while (pos < end) {
 121                 if ((u64) pos >= (u64) start + (1ULL << 20))
 122                         fgp_flags &= ~FGP_CREAT;
 123
 124                 ret = darray_make_room_gfp(folios, 1, gfp & GFP_KERNEL);
 125                 if (ret)
 126                         break;
 127
 128                 f = __filemap_get_folio(mapping, pos >> PAGE_SHIFT, fgp_flags, gfp);
 129                 if (IS_ERR_OR_NULL(f))
 130                         break;
 131
 132                 BUG_ON(folios->nr && folio_pos(f) != pos);
 133
 134                 pos = folio_end_pos(f);
 135                 darray_push(folios, f);
 136         }
 137
 138         if (!folios->nr && !ret && (fgp_flags & FGP_CREAT))
 139                 ret = -ENOMEM;
 140
 141         return folios->nr ? 0 : ret;
 142 }
 143
 144 struct nocow_flush {
 145         struct closure  *cl;
 146         struct bch_dev  *ca;
 147         struct bio      bio;
 148 };
 149
 150 static void nocow_flush_endio(struct bio *_bio)
 151 {
 152
 153         struct nocow_flush *bio = container_of(_bio, struct nocow_flush, bio);
 154
 155         closure_put(bio->cl);
 156         percpu_ref_put(&bio->ca->io_ref);
 157         bio_put(&bio->bio);
 158 }
 159
 160 static void bch2_inode_flush_nocow_writes_async(struct bch_fs *c,
 161                                                 struct bch_inode_info *inode,
 162                                                 struct closure *cl)
 163 {
 164         struct nocow_flush *bio;
 165         struct bch_dev *ca;
 166         struct bch_devs_mask devs;
 167         unsigned dev;
 168
 169         dev = find_first_bit(inode->ei_devs_need_flush.d, BCH_SB_MEMBERS_MAX);
 170         if (dev == BCH_SB_MEMBERS_MAX)
 171                 return;
 172
 173         devs = inode->ei_devs_need_flush;
 174         memset(&inode->ei_devs_need_flush, 0, sizeof(inode->ei_devs_need_flush));
 175
 176         for_each_set_bit(dev, devs.d, BCH_SB_MEMBERS_MAX) {
 177                 rcu_read_lock();
 178                 ca = rcu_dereference(c->devs[dev]);
 179                 if (ca && !percpu_ref_tryget(&ca->io_ref))
 180                         ca = NULL;
 181                 rcu_read_unlock();
 182
 183                 if (!ca)
 184                         continue;
 185
 186                 bio = container_of(bio_alloc_bioset(ca->disk_sb.bdev, 0,
 187                                                     REQ_OP_FLUSH,
 188                                                     GFP_KERNEL,
 189                                                     &c->nocow_flush_bioset),
 190                                    struct nocow_flush, bio);
 191                 bio->cl                 = cl;
 192                 bio->ca                 = ca;
 193                 bio->bio.bi_end_io      = nocow_flush_endio;
 194                 closure_bio_submit(&bio->bio, cl);
 195         }
 196 }
 197
 198 static int bch2_inode_flush_nocow_writes(struct bch_fs *c,
 199                                          struct bch_inode_info *inode)
 200 {
 201         struct closure cl;
 202
 203         closure_init_stack(&cl);
 204         bch2_inode_flush_nocow_writes_async(c, inode, &cl);
 205         closure_sync(&cl);
 206
 207         return 0;
 208 }
 209
 210 static inline bool bio_full(struct bio *bio, unsigned len)
 211 {
 212         if (bio->bi_vcnt >= bio->bi_max_vecs)
 213                 return true;
 214         if (bio->bi_iter.bi_size > UINT_MAX - len)
 215                 return true;
 216         return false;
 217 }
 218
 219 static inline struct address_space *faults_disabled_mapping(void)
 220 {
 221         return (void *) (((unsigned long) current->faults_disabled_mapping) & ~1UL);
 222 }
 223
 224 static inline void set_fdm_dropped_locks(void)
 225 {
 226         current->faults_disabled_mapping =
 227                 (void *) (((unsigned long) current->faults_disabled_mapping)|1);
 228 }
 229
 230 static inline bool fdm_dropped_locks(void)
 231 {
 232         return ((unsigned long) current->faults_disabled_mapping) & 1;
 233 }
 234
 235 struct quota_res {
 236         u64                             sectors;
 237 };
 238
 239 struct bch_writepage_io {
 240         struct bch_inode_info           *inode;
 241
 242         /* must be last: */
 243         struct bch_write_op             op;
 244 };
 245
 246 struct dio_write {
 247         struct kiocb                    *req;
 248         struct address_space            *mapping;
 249         struct bch_inode_info           *inode;
 250         struct mm_struct                *mm;
 251         unsigned                        loop:1,
 252                                         extending:1,
 253                                         sync:1,
 254                                         flush:1,
 255                                         free_iov:1;
 256         struct quota_res                quota_res;
 257         u64                             written;
 258
 259         struct iov_iter                 iter;
 260         struct iovec                    inline_vecs[2];
 261
 262         /* must be last: */
 263         struct bch_write_op             op;
 264 };
 265
 266 struct dio_read {
 267         struct closure                  cl;
 268         struct kiocb                    *req;
 269         long                            ret;
 270         bool                            should_dirty;
 271         struct bch_read_bio             rbio;
 272 };
 273
 274 /* pagecache_block must be held */
 275 static noinline int write_invalidate_inode_pages_range(struct address_space *mapping,
 276                                               loff_t start, loff_t end)
 277 {
 278         int ret;
 279
 280         /*
 281          * XXX: the way this is currently implemented, we can spin if a process
 282          * is continually redirtying a specific page
 283          */
 284         do {
 285                 if (!mapping->nrpages)
 286                         return 0;
 287
 288                 ret = filemap_write_and_wait_range(mapping, start, end);
 289                 if (ret)
 290                         break;
 291
 292                 if (!mapping->nrpages)
 293                         return 0;
 294
 295                 ret = invalidate_inode_pages2_range(mapping,
 296                                 start >> PAGE_SHIFT,
 297                                 end >> PAGE_SHIFT);
 298         } while (ret == -EBUSY);
 299
 300         return ret;
 301 }
 302
 303 /* quotas */
 304
 305 #ifdef CONFIG_BCACHEFS_QUOTA
 306
 307 static void __bch2_quota_reservation_put(struct bch_fs *c,
 308                                          struct bch_inode_info *inode,
 309                                          struct quota_res *res)
 310 {
 311         BUG_ON(res->sectors > inode->ei_quota_reserved);
 312
 313         bch2_quota_acct(c, inode->ei_qid, Q_SPC,
 314                         -((s64) res->sectors), KEY_TYPE_QUOTA_PREALLOC);
 315         inode->ei_quota_reserved -= res->sectors;
 316         res->sectors = 0;
 317 }
 318
 319 static void bch2_quota_reservation_put(struct bch_fs *c,
 320                                        struct bch_inode_info *inode,
 321                                        struct quota_res *res)
 322 {
 323         if (res->sectors) {
 324                 mutex_lock(&inode->ei_quota_lock);
 325                 __bch2_quota_reservation_put(c, inode, res);
 326                 mutex_unlock(&inode->ei_quota_lock);
 327         }
 328 }
 329
 330 static int bch2_quota_reservation_add(struct bch_fs *c,
 331                                       struct bch_inode_info *inode,
 332                                       struct quota_res *res,
 333                                       u64 sectors,
 334                                       bool check_enospc)
 335 {
 336         int ret;
 337
 338         if (test_bit(EI_INODE_SNAPSHOT, &inode->ei_flags))
 339                 return 0;
 340
 341         mutex_lock(&inode->ei_quota_lock);
 342         ret = bch2_quota_acct(c, inode->ei_qid, Q_SPC, sectors,
 343                               check_enospc ? KEY_TYPE_QUOTA_PREALLOC : KEY_TYPE_QUOTA_NOCHECK);
 344         if (likely(!ret)) {
 345                 inode->ei_quota_reserved += sectors;
 346                 res->sectors += sectors;
 347         }
 348         mutex_unlock(&inode->ei_quota_lock);
 349
 350         return ret;
 351 }
 352
 353 #else
 354
 355 static void __bch2_quota_reservation_put(struct bch_fs *c,
 356                                          struct bch_inode_info *inode,
 357                                          struct quota_res *res) {}
 358
 359 static void bch2_quota_reservation_put(struct bch_fs *c,
 360                                        struct bch_inode_info *inode,
 361                                        struct quota_res *res) {}
 362
 363 static int bch2_quota_reservation_add(struct bch_fs *c,
 364                                       struct bch_inode_info *inode,
 365                                       struct quota_res *res,
 366                                       unsigned sectors,
 367                                       bool check_enospc)
 368 {
 369         return 0;
 370 }
 371
 372 #endif
 373
 374 /* i_size updates: */
 375
 376 struct inode_new_size {
 377         loff_t          new_size;
 378         u64             now;
 379         unsigned        fields;
 380 };
 381
 382 static int inode_set_size(struct bch_inode_info *inode,
 383                           struct bch_inode_unpacked *bi,
 384                           void *p)
 385 {
 386         struct inode_new_size *s = p;
 387
 388         bi->bi_size = s->new_size;
 389         if (s->fields & ATTR_ATIME)
 390                 bi->bi_atime = s->now;
 391         if (s->fields & ATTR_MTIME)
 392                 bi->bi_mtime = s->now;
 393         if (s->fields & ATTR_CTIME)
 394                 bi->bi_ctime = s->now;
 395
 396         return 0;
 397 }
 398
 399 int __must_check bch2_write_inode_size(struct bch_fs *c,
 400                                        struct bch_inode_info *inode,
 401                                        loff_t new_size, unsigned fields)
 402 {
 403         struct inode_new_size s = {
 404                 .new_size       = new_size,
 405                 .now            = bch2_current_time(c),
 406                 .fields         = fields,
 407         };
 408
 409         return bch2_write_inode(c, inode, inode_set_size, &s, fields);
 410 }
 411
 412 static void __i_sectors_acct(struct bch_fs *c, struct bch_inode_info *inode,
 413                            struct quota_res *quota_res, s64 sectors)
 414 {
 415         bch2_fs_inconsistent_on((s64) inode->v.i_blocks + sectors < 0, c,
 416                                 "inode %lu i_blocks underflow: %llu + %lli < 0 (ondisk %lli)",
 417                                 inode->v.i_ino, (u64) inode->v.i_blocks, sectors,
 418                                 inode->ei_inode.bi_sectors);
 419         inode->v.i_blocks += sectors;
 420
 421 #ifdef CONFIG_BCACHEFS_QUOTA
 422         if (quota_res &&
 423             !test_bit(EI_INODE_SNAPSHOT, &inode->ei_flags) &&
 424             sectors > 0) {
 425                 BUG_ON(sectors > quota_res->sectors);
 426                 BUG_ON(sectors > inode->ei_quota_reserved);
 427
 428                 quota_res->sectors -= sectors;
 429                 inode->ei_quota_reserved -= sectors;
 430         } else {
 431                 bch2_quota_acct(c, inode->ei_qid, Q_SPC, sectors, KEY_TYPE_QUOTA_WARN);
 432         }
 433 #endif
 434 }
 435
 436 static void i_sectors_acct(struct bch_fs *c, struct bch_inode_info *inode,
 437                            struct quota_res *quota_res, s64 sectors)
 438 {
 439         if (sectors) {
 440                 mutex_lock(&inode->ei_quota_lock);
 441                 __i_sectors_acct(c, inode, quota_res, sectors);
 442                 mutex_unlock(&inode->ei_quota_lock);
 443         }
 444 }
 445
 446 /* page state: */
 447
 448 /* stored in page->private: */
 449
 450 #define BCH_FOLIO_SECTOR_STATE()        \
 451         x(unallocated)                  \
 452         x(reserved)                     \
 453         x(dirty)                        \
 454         x(dirty_reserved)               \
 455         x(allocated)
 456
 457 enum bch_folio_sector_state {
 458 #define x(n)    SECTOR_##n,
 459         BCH_FOLIO_SECTOR_STATE()
 460 #undef x
 461 };
 462
 463 static const char * const bch2_folio_sector_states[] = {
 464 #define x(n)    #n,
 465         BCH_FOLIO_SECTOR_STATE()
 466 #undef x
 467         NULL
 468 };
 469
 470 static inline enum bch_folio_sector_state
 471 folio_sector_dirty(enum bch_folio_sector_state state)
 472 {
 473         switch (state) {
 474         case SECTOR_unallocated:
 475                 return SECTOR_dirty;
 476         case SECTOR_reserved:
 477                 return SECTOR_dirty_reserved;
 478         default:
 479                 return state;
 480         }
 481 }
 482
 483 static inline enum bch_folio_sector_state
 484 folio_sector_undirty(enum bch_folio_sector_state state)
 485 {
 486         switch (state) {
 487         case SECTOR_dirty:
 488                 return SECTOR_unallocated;
 489         case SECTOR_dirty_reserved:
 490                 return SECTOR_reserved;
 491         default:
 492                 return state;
 493         }
 494 }
 495
 496 static inline enum bch_folio_sector_state
 497 folio_sector_reserve(enum bch_folio_sector_state state)
 498 {
 499         switch (state) {
 500         case SECTOR_unallocated:
 501                 return SECTOR_reserved;
 502         case SECTOR_dirty:
 503                 return SECTOR_dirty_reserved;
 504         default:
 505                 return state;
 506         }
 507 }
 508
 509 struct bch_folio_sector {
 510         /* Uncompressed, fully allocated replicas (or on disk reservation): */
 511         unsigned                nr_replicas:4;
 512
 513         /* Owns PAGE_SECTORS * replicas_reserved sized in memory reservation: */
 514         unsigned                replicas_reserved:4;
 515
 516         /* i_sectors: */
 517         enum bch_folio_sector_state state:8;
 518 };
 519
 520 struct bch_folio {
 521         spinlock_t              lock;
 522         atomic_t                write_count;
 523         /*
 524          * Is the sector state up to date with the btree?
 525          * (Not the data itself)
 526          */
 527         bool                    uptodate;
 528         struct bch_folio_sector s[];
 529 };
 530
 531 static inline void folio_sector_set(struct folio *folio,
 532                              struct bch_folio *s,
 533                              unsigned i, unsigned n)
 534 {
 535         s->s[i].state = n;
 536 }
 537
 538 /* file offset (to folio offset) to bch_folio_sector index */
 539 static inline int folio_pos_to_s(struct folio *folio, loff_t pos)
 540 {
 541         u64 f_offset = pos - folio_pos(folio);
 542         BUG_ON(pos < folio_pos(folio) || pos >= folio_end_pos(folio));
 543         return f_offset >> SECTOR_SHIFT;
 544 }
 545
 546 static inline struct bch_folio *__bch2_folio(struct folio *folio)
 547 {
 548         return folio_has_private(folio)
 549                 ? (struct bch_folio *) folio_get_private(folio)
 550                 : NULL;
 551 }
 552
 553 static inline struct bch_folio *bch2_folio(struct folio *folio)
 554 {
 555         EBUG_ON(!folio_test_locked(folio));
 556
 557         return __bch2_folio(folio);
 558 }
 559
 560 /* for newly allocated folios: */
 561 static void __bch2_folio_release(struct folio *folio)
 562 {
 563         kfree(folio_detach_private(folio));
 564 }
 565
 566 static void bch2_folio_release(struct folio *folio)
 567 {
 568         EBUG_ON(!folio_test_locked(folio));
 569         __bch2_folio_release(folio);
 570 }
 571
 572 /* for newly allocated folios: */
 573 static struct bch_folio *__bch2_folio_create(struct folio *folio, gfp_t gfp)
 574 {
 575         struct bch_folio *s;
 576
 577         s = kzalloc(sizeof(*s) +
 578                     sizeof(struct bch_folio_sector) *
 579                     folio_sectors(folio), gfp);
 580         if (!s)
 581                 return NULL;
 582
 583         spin_lock_init(&s->lock);
 584         folio_attach_private(folio, s);
 585         return s;
 586 }
 587
 588 static struct bch_folio *bch2_folio_create(struct folio *folio, gfp_t gfp)
 589 {
 590         return bch2_folio(folio) ?: __bch2_folio_create(folio, gfp);
 591 }
 592
 593 static unsigned bkey_to_sector_state(struct bkey_s_c k)
 594 {
 595         if (bkey_extent_is_reservation(k))
 596                 return SECTOR_reserved;
 597         if (bkey_extent_is_allocation(k.k))
 598                 return SECTOR_allocated;
 599         return SECTOR_unallocated;
 600 }
 601
 602 static void __bch2_folio_set(struct folio *folio,
 603                              unsigned pg_offset, unsigned pg_len,
 604                              unsigned nr_ptrs, unsigned state)
 605 {
 606         struct bch_folio *s = bch2_folio(folio);
 607         unsigned i, sectors = folio_sectors(folio);
 608
 609         BUG_ON(pg_offset >= sectors);
 610         BUG_ON(pg_offset + pg_len > sectors);
 611
 612         spin_lock(&s->lock);
 613
 614         for (i = pg_offset; i < pg_offset + pg_len; i++) {
 615                 s->s[i].nr_replicas     = nr_ptrs;
 616                 folio_sector_set(folio, s, i, state);
 617         }
 618
 619         if (i == sectors)
 620                 s->uptodate = true;
 621
 622         spin_unlock(&s->lock);
 623 }
 624
 625 /*
 626  * Initialize bch_folio state (allocated/unallocated, nr_replicas) from the
 627  * extents btree:
 628  */
 629 static int bch2_folio_set(struct bch_fs *c, subvol_inum inum,
 630                           struct folio **folios, unsigned nr_folios)
 631 {
 632         struct btree_trans trans;
 633         struct btree_iter iter;
 634         struct bkey_s_c k;
 635         struct bch_folio *s;
 636         u64 offset = folio_sector(folios[0]);
 637         unsigned folio_idx;
 638         u32 snapshot;
 639         bool need_set = false;
 640         int ret;
 641
 642         for (folio_idx = 0; folio_idx < nr_folios; folio_idx++) {
 643                 s = bch2_folio_create(folios[folio_idx], GFP_KERNEL);
 644                 if (!s)
 645                         return -ENOMEM;
 646
 647                 need_set |= !s->uptodate;
 648         }
 649
 650         if (!need_set)
 651                 return 0;
 652
 653         folio_idx = 0;
 654         bch2_trans_init(&trans, c, 0, 0);
 655 retry:
 656         bch2_trans_begin(&trans);
 657
 658         ret = bch2_subvolume_get_snapshot(&trans, inum.subvol, &snapshot);
 659         if (ret)
 660                 goto err;
 661
 662         for_each_btree_key_norestart(&trans, iter, BTREE_ID_extents,
 663                            SPOS(inum.inum, offset, snapshot),
 664                            BTREE_ITER_SLOTS, k, ret) {
 665                 unsigned nr_ptrs = bch2_bkey_nr_ptrs_fully_allocated(k);
 666                 unsigned state = bkey_to_sector_state(k);
 667
 668                 while (folio_idx < nr_folios) {
 669                         struct folio *folio = folios[folio_idx];
 670                         u64 folio_start = folio_sector(folio);
 671                         u64 folio_end   = folio_end_sector(folio);
 672                         unsigned folio_offset = max(bkey_start_offset(k.k), folio_start) - folio_start;
 673                         unsigned folio_len = min(k.k->p.offset, folio_end) - folio_offset - folio_start;
 674
 675                         BUG_ON(k.k->p.offset < folio_start);
 676                         BUG_ON(bkey_start_offset(k.k) > folio_end);
 677
 678                         if (!bch2_folio(folio)->uptodate)
 679                                 __bch2_folio_set(folio, folio_offset, folio_len, nr_ptrs, state);
 680
 681                         if (k.k->p.offset < folio_end)
 682                                 break;
 683                         folio_idx++;
 684                 }
 685
 686                 if (folio_idx == nr_folios)
 687                         break;
 688         }
 689
 690         offset = iter.pos.offset;
 691         bch2_trans_iter_exit(&trans, &iter);
 692 err:
 693         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
 694                 goto retry;
 695         bch2_trans_exit(&trans);
 696
 697         return ret;
 698 }
 699
 700 static void bch2_bio_page_state_set(struct bio *bio, struct bkey_s_c k)
 701 {
 702         struct bvec_iter iter;
 703         struct folio_vec fv;
 704         unsigned nr_ptrs = k.k->type == KEY_TYPE_reflink_v
 705                 ? 0 : bch2_bkey_nr_ptrs_fully_allocated(k);
 706         unsigned state = bkey_to_sector_state(k);
 707
 708         bio_for_each_folio(fv, bio, iter)
 709                 __bch2_folio_set(fv.fv_folio,
 710                                  fv.fv_offset >> 9,
 711                                  fv.fv_len >> 9,
 712                                  nr_ptrs, state);
 713 }
 714
 715 static void mark_pagecache_unallocated(struct bch_inode_info *inode,
 716                                        u64 start, u64 end)
 717 {
 718         pgoff_t index = start >> PAGE_SECTORS_SHIFT;
 719         pgoff_t end_index = (end - 1) >> PAGE_SECTORS_SHIFT;
 720         struct folio_batch fbatch;
 721         unsigned i, j;
 722
 723         if (end <= start)
 724                 return;
 725
 726         folio_batch_init(&fbatch);
 727
 728         while (filemap_get_folios(inode->v.i_mapping,
 729                                   &index, end_index, &fbatch)) {
 730                 for (i = 0; i < folio_batch_count(&fbatch); i++) {
 731                         struct folio *folio = fbatch.folios[i];
 732                         u64 folio_start = folio_sector(folio);
 733                         u64 folio_end = folio_end_sector(folio);
 734                         unsigned folio_offset = max(start, folio_start) - folio_start;
 735                         unsigned folio_len = min(end, folio_end) - folio_offset - folio_start;
 736                         struct bch_folio *s;
 737
 738                         BUG_ON(end <= folio_start);
 739
 740                         folio_lock(folio);
 741                         s = bch2_folio(folio);
 742
 743                         if (s) {
 744                                 spin_lock(&s->lock);
 745                                 for (j = folio_offset; j < folio_offset + folio_len; j++)
 746                                         s->s[j].nr_replicas = 0;
 747                                 spin_unlock(&s->lock);
 748                         }
 749
 750                         folio_unlock(folio);
 751                 }
 752                 folio_batch_release(&fbatch);
 753                 cond_resched();
 754         }
 755 }
 756
 757 static void mark_pagecache_reserved(struct bch_inode_info *inode,
 758                                     u64 start, u64 end)
 759 {
 760         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 761         pgoff_t index = start >> PAGE_SECTORS_SHIFT;
 762         pgoff_t end_index = (end - 1) >> PAGE_SECTORS_SHIFT;
 763         struct folio_batch fbatch;
 764         s64 i_sectors_delta = 0;
 765         unsigned i, j;
 766
 767         if (end <= start)
 768                 return;
 769
 770         folio_batch_init(&fbatch);
 771
 772         while (filemap_get_folios(inode->v.i_mapping,
 773                                   &index, end_index, &fbatch)) {
 774                 for (i = 0; i < folio_batch_count(&fbatch); i++) {
 775                         struct folio *folio = fbatch.folios[i];
 776                         u64 folio_start = folio_sector(folio);
 777                         u64 folio_end = folio_end_sector(folio);
 778                         unsigned folio_offset = max(start, folio_start) - folio_start;
 779                         unsigned folio_len = min(end, folio_end) - folio_offset - folio_start;
 780                         struct bch_folio *s;
 781
 782                         BUG_ON(end <= folio_start);
 783
 784                         folio_lock(folio);
 785                         s = bch2_folio(folio);
 786
 787                         if (s) {
 788                                 spin_lock(&s->lock);
 789                                 for (j = folio_offset; j < folio_offset + folio_len; j++) {
 790                                         i_sectors_delta -= s->s[j].state == SECTOR_dirty;
 791                                         folio_sector_set(folio, s, j, folio_sector_reserve(s->s[j].state));
 792                                 }
 793                                 spin_unlock(&s->lock);
 794                         }
 795
 796                         folio_unlock(folio);
 797                 }
 798                 folio_batch_release(&fbatch);
 799                 cond_resched();
 800         }
 801
 802         i_sectors_acct(c, inode, NULL, i_sectors_delta);
 803 }
 804
 805 static inline unsigned inode_nr_replicas(struct bch_fs *c, struct bch_inode_info *inode)
 806 {
 807         /* XXX: this should not be open coded */
 808         return inode->ei_inode.bi_data_replicas
 809                 ? inode->ei_inode.bi_data_replicas - 1
 810                 : c->opts.data_replicas;
 811 }
 812
 813 static inline unsigned sectors_to_reserve(struct bch_folio_sector *s,
 814                                           unsigned nr_replicas)
 815 {
 816         return max(0, (int) nr_replicas -
 817                    s->nr_replicas -
 818                    s->replicas_reserved);
 819 }
 820
 821 static int bch2_get_folio_disk_reservation(struct bch_fs *c,
 822                                 struct bch_inode_info *inode,
 823                                 struct folio *folio, bool check_enospc)
 824 {
 825         struct bch_folio *s = bch2_folio_create(folio, 0);
 826         unsigned nr_replicas = inode_nr_replicas(c, inode);
 827         struct disk_reservation disk_res = { 0 };
 828         unsigned i, sectors = folio_sectors(folio), disk_res_sectors = 0;
 829         int ret;
 830
 831         if (!s)
 832                 return -ENOMEM;
 833
 834         for (i = 0; i < sectors; i++)
 835                 disk_res_sectors += sectors_to_reserve(&s->s[i], nr_replicas);
 836
 837         if (!disk_res_sectors)
 838                 return 0;
 839
 840         ret = bch2_disk_reservation_get(c, &disk_res,
 841                                         disk_res_sectors, 1,
 842                                         !check_enospc
 843                                         ? BCH_DISK_RESERVATION_NOFAIL
 844                                         : 0);
 845         if (unlikely(ret))
 846                 return ret;
 847
 848         for (i = 0; i < sectors; i++)
 849                 s->s[i].replicas_reserved +=
 850                         sectors_to_reserve(&s->s[i], nr_replicas);
 851
 852         return 0;
 853 }
 854
 855 struct bch2_folio_reservation {
 856         struct disk_reservation disk;
 857         struct quota_res        quota;
 858 };
 859
 860 static void bch2_folio_reservation_init(struct bch_fs *c,
 861                         struct bch_inode_info *inode,
 862                         struct bch2_folio_reservation *res)
 863 {
 864         memset(res, 0, sizeof(*res));
 865
 866         res->disk.nr_replicas = inode_nr_replicas(c, inode);
 867 }
 868
 869 static void bch2_folio_reservation_put(struct bch_fs *c,
 870                         struct bch_inode_info *inode,
 871                         struct bch2_folio_reservation *res)
 872 {
 873         bch2_disk_reservation_put(c, &res->disk);
 874         bch2_quota_reservation_put(c, inode, &res->quota);
 875 }
 876
 877 static int bch2_folio_reservation_get(struct bch_fs *c,
 878                         struct bch_inode_info *inode,
 879                         struct folio *folio,
 880                         struct bch2_folio_reservation *res,
 881                         unsigned offset, unsigned len)
 882 {
 883         struct bch_folio *s = bch2_folio_create(folio, 0);
 884         unsigned i, disk_sectors = 0, quota_sectors = 0;
 885         int ret;
 886
 887         if (!s)
 888                 return -ENOMEM;
 889
 890         BUG_ON(!s->uptodate);
 891
 892         for (i = round_down(offset, block_bytes(c)) >> 9;
 893              i < round_up(offset + len, block_bytes(c)) >> 9;
 894              i++) {
 895                 disk_sectors += sectors_to_reserve(&s->s[i],
 896                                                 res->disk.nr_replicas);
 897                 quota_sectors += s->s[i].state == SECTOR_unallocated;
 898         }
 899
 900         if (disk_sectors) {
 901                 ret = bch2_disk_reservation_add(c, &res->disk, disk_sectors, 0);
 902                 if (unlikely(ret))
 903                         return ret;
 904         }
 905
 906         if (quota_sectors) {
 907                 ret = bch2_quota_reservation_add(c, inode, &res->quota,
 908                                                  quota_sectors, true);
 909                 if (unlikely(ret)) {
 910                         struct disk_reservation tmp = {
 911                                 .sectors = disk_sectors
 912                         };
 913
 914                         bch2_disk_reservation_put(c, &tmp);
 915                         res->disk.sectors -= disk_sectors;
 916                         return ret;
 917                 }
 918         }
 919
 920         return 0;
 921 }
 922
 923 static void bch2_clear_folio_bits(struct folio *folio)
 924 {
 925         struct bch_inode_info *inode = to_bch_ei(folio->mapping->host);
 926         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 927         struct bch_folio *s = bch2_folio(folio);
 928         struct disk_reservation disk_res = { 0 };
 929         int i, sectors = folio_sectors(folio), dirty_sectors = 0;
 930
 931         if (!s)
 932                 return;
 933
 934         EBUG_ON(!folio_test_locked(folio));
 935         EBUG_ON(folio_test_writeback(folio));
 936
 937         for (i = 0; i < sectors; i++) {
 938                 disk_res.sectors += s->s[i].replicas_reserved;
 939                 s->s[i].replicas_reserved = 0;
 940
 941                 dirty_sectors -= s->s[i].state == SECTOR_dirty;
 942                 folio_sector_set(folio, s, i, folio_sector_undirty(s->s[i].state));
 943         }
 944
 945         bch2_disk_reservation_put(c, &disk_res);
 946
 947         i_sectors_acct(c, inode, NULL, dirty_sectors);
 948
 949         bch2_folio_release(folio);
 950 }
 951
 952 static void bch2_set_folio_dirty(struct bch_fs *c,
 953                         struct bch_inode_info *inode,
 954                         struct folio *folio,
 955                         struct bch2_folio_reservation *res,
 956                         unsigned offset, unsigned len)
 957 {
 958         struct bch_folio *s = bch2_folio(folio);
 959         unsigned i, dirty_sectors = 0;
 960
 961         WARN_ON((u64) folio_pos(folio) + offset + len >
 962                 round_up((u64) i_size_read(&inode->v), block_bytes(c)));
 963
 964         BUG_ON(!s->uptodate);
 965
 966         spin_lock(&s->lock);
 967
 968         for (i = round_down(offset, block_bytes(c)) >> 9;
 969              i < round_up(offset + len, block_bytes(c)) >> 9;
 970              i++) {
 971                 unsigned sectors = sectors_to_reserve(&s->s[i],
 972                                                 res->disk.nr_replicas);
 973
 974                 /*
 975                  * This can happen if we race with the error path in
 976                  * bch2_writepage_io_done():
 977                  */
 978                 sectors = min_t(unsigned, sectors, res->disk.sectors);
 979
 980                 s->s[i].replicas_reserved += sectors;
 981                 res->disk.sectors -= sectors;
 982
 983                 dirty_sectors += s->s[i].state == SECTOR_unallocated;
 984
 985                 folio_sector_set(folio, s, i, folio_sector_dirty(s->s[i].state));
 986         }
 987
 988         spin_unlock(&s->lock);
 989
 990         i_sectors_acct(c, inode, &res->quota, dirty_sectors);
 991
 992         if (!folio_test_dirty(folio))
 993                 filemap_dirty_folio(inode->v.i_mapping, folio);
 994 }
 995
 996 vm_fault_t bch2_page_fault(struct vm_fault *vmf)
 997 {
 998         struct file *file = vmf->vma->vm_file;
 999         struct address_space *mapping = file->f_mapping;
1000         struct address_space *fdm = faults_disabled_mapping();
1001         struct bch_inode_info *inode = file_bch_inode(file);
1002         vm_fault_t ret;
1003
1004         if (fdm == mapping)
1005                 return VM_FAULT_SIGBUS;
1006
1007         /* Lock ordering: */
1008         if (fdm > mapping) {
1009                 struct bch_inode_info *fdm_host = to_bch_ei(fdm->host);
1010
1011                 if (bch2_pagecache_add_tryget(inode))
1012                         goto got_lock;
1013
1014                 bch2_pagecache_block_put(fdm_host);
1015
1016                 bch2_pagecache_add_get(inode);
1017                 bch2_pagecache_add_put(inode);
1018
1019                 bch2_pagecache_block_get(fdm_host);
1020
1021                 /* Signal that lock has been dropped: */
1022                 set_fdm_dropped_locks();
1023                 return VM_FAULT_SIGBUS;
1024         }
1025
1026         bch2_pagecache_add_get(inode);
1027 got_lock:
1028         ret = filemap_fault(vmf);
1029         bch2_pagecache_add_put(inode);
1030
1031         return ret;
1032 }
1033
1034 vm_fault_t bch2_page_mkwrite(struct vm_fault *vmf)
1035 {
1036         struct folio *folio = page_folio(vmf->page);
1037         struct file *file = vmf->vma->vm_file;
1038         struct bch_inode_info *inode = file_bch_inode(file);
1039         struct address_space *mapping = file->f_mapping;
1040         struct bch_fs *c = inode->v.i_sb->s_fs_info;
1041         struct bch2_folio_reservation res;
1042         unsigned len;
1043         loff_t isize;
1044         vm_fault_t ret;
1045
1046         bch2_folio_reservation_init(c, inode, &res);
1047
1048         sb_start_pagefault(inode->v.i_sb);
1049         file_update_time(file);
1050
1051         /*
1052          * Not strictly necessary, but helps avoid dio writes livelocking in
1053          * write_invalidate_inode_pages_range() - can drop this if/when we get
1054          * a write_invalidate_inode_pages_range() that works without dropping
1055          * page lock before invalidating page
1056          */
1057         bch2_pagecache_add_get(inode);
1058
1059         folio_lock(folio);
1060         isize = i_size_read(&inode->v);
1061
1062         if (folio->mapping != mapping || folio_pos(folio) >= isize) {
1063                 folio_unlock(folio);
1064                 ret = VM_FAULT_NOPAGE;
1065                 goto out;
1066         }
1067
1068         len = min_t(loff_t, folio_size(folio), isize - folio_pos(folio));
1069
1070         if (bch2_folio_set(c, inode_inum(inode), &folio, 1) ?:
1071             bch2_folio_reservation_get(c, inode, folio, &res, 0, len)) {
1072                 folio_unlock(folio);
1073                 ret = VM_FAULT_SIGBUS;
1074                 goto out;
1075         }
1076
1077         bch2_set_folio_dirty(c, inode, folio, &res, 0, len);
1078         bch2_folio_reservation_put(c, inode, &res);
1079
1080         folio_wait_stable(folio);
1081         ret = VM_FAULT_LOCKED;
1082 out:
1083         bch2_pagecache_add_put(inode);
1084         sb_end_pagefault(inode->v.i_sb);
1085
1086         return ret;
1087 }
1088
1089 void bch2_invalidate_folio(struct folio *folio, size_t offset, size_t length)
1090 {
1091         if (offset || length < folio_size(folio))
1092                 return;
1093
1094         bch2_clear_folio_bits(folio);
1095 }
1096
1097 bool bch2_release_folio(struct folio *folio, gfp_t gfp_mask)
1098 {
1099         if (folio_test_dirty(folio) || folio_test_writeback(folio))
1100                 return false;
1101
1102         bch2_clear_folio_bits(folio);
1103         return true;
1104 }
1105
1106 /* readpage(s): */
1107
1108 static void bch2_readpages_end_io(struct bio *bio)
1109 {
1110         struct folio_iter fi;
1111
1112         bio_for_each_folio_all(fi, bio) {
1113                 if (!bio->bi_status) {
1114                         folio_mark_uptodate(fi.folio);
1115                 } else {
1116                         folio_clear_uptodate(fi.folio);
1117                         folio_set_error(fi.folio);
1118                 }
1119                 folio_unlock(fi.folio);
1120         }
1121
1122         bio_put(bio);
1123 }
1124
1125 struct readpages_iter {
1126         struct address_space    *mapping;
1127         unsigned                idx;
1128         folios                  folios;
1129 };
1130
1131 static int readpages_iter_init(struct readpages_iter *iter,
1132                                struct readahead_control *ractl)
1133 {
1134         struct folio **fi;
1135         int ret;
1136
1137         memset(iter, 0, sizeof(*iter));
1138
1139         iter->mapping = ractl->mapping;
1140
1141         ret = filemap_get_contig_folios_d(iter->mapping,
1142                                 ractl->_index << PAGE_SHIFT,
1143                                 (ractl->_index + ractl->_nr_pages) << PAGE_SHIFT,
1144                                 0, mapping_gfp_mask(iter->mapping),
1145                                 &iter->folios);
1146         if (ret)
1147                 return ret;
1148
1149         darray_for_each(iter->folios, fi) {
1150                 ractl->_nr_pages -= 1U << folio_order(*fi);
1151                 __bch2_folio_create(*fi, __GFP_NOFAIL|GFP_KERNEL);
1152                 folio_put(*fi);
1153                 folio_put(*fi);
1154         }
1155
1156         return 0;
1157 }
1158
1159 static inline struct folio *readpage_iter_peek(struct readpages_iter *iter)
1160 {
1161         if (iter->idx >= iter->folios.nr)
1162                 return NULL;
1163         return iter->folios.data[iter->idx];
1164 }
1165
1166 static inline void readpage_iter_advance(struct readpages_iter *iter)
1167 {
1168         iter->idx++;
1169 }
1170
1171 static bool extent_partial_reads_expensive(struct bkey_s_c k)
1172 {
1173         struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
1174         struct bch_extent_crc_unpacked crc;
1175         const union bch_extent_entry *i;
1176
1177         bkey_for_each_crc(k.k, ptrs, crc, i)
1178                 if (crc.csum_type || crc.compression_type)
1179                         return true;
1180         return false;
1181 }
1182
1183 static int readpage_bio_extend(struct btree_trans *trans,
1184                                struct readpages_iter *iter,
1185                                struct bio *bio,
1186                                unsigned sectors_this_extent,
1187                                bool get_more)
1188 {
1189         /* Don't hold btree locks while allocating memory: */
1190         bch2_trans_unlock(trans);
1191
1192         while (bio_sectors(bio) < sectors_this_extent &&
1193                bio->bi_vcnt < bio->bi_max_vecs) {
1194                 struct folio *folio = readpage_iter_peek(iter);
1195                 int ret;
1196
1197                 if (folio) {
1198                         readpage_iter_advance(iter);
1199                 } else {
1200                         pgoff_t folio_offset = bio_end_sector(bio) >> PAGE_SECTORS_SHIFT;
1201
1202                         if (!get_more)
1203                                 break;
1204
1205                         folio = xa_load(&iter->mapping->i_pages, folio_offset);
1206                         if (folio && !xa_is_value(folio))
1207                                 break;
1208
1209                         folio = filemap_alloc_folio(readahead_gfp_mask(iter->mapping), 0);
1210                         if (!folio)
1211                                 break;
1212
1213                         if (!__bch2_folio_create(folio, GFP_KERNEL)) {
1214                                 folio_put(folio);
1215                                 break;
1216                         }
1217
1218                         ret = filemap_add_folio(iter->mapping, folio, folio_offset, GFP_KERNEL);
1219                         if (ret) {
1220                                 __bch2_folio_release(folio);
1221                                 folio_put(folio);
1222                                 break;
1223                         }
1224
1225                         folio_put(folio);
1226                 }
1227
1228                 BUG_ON(folio_sector(folio) != bio_end_sector(bio));
1229
1230                 BUG_ON(!bio_add_folio(bio, folio, folio_size(folio), 0));
1231         }
1232
1233         return bch2_trans_relock(trans);
1234 }
1235
1236 static void bchfs_read(struct btree_trans *trans,
1237                        struct bch_read_bio *rbio,
1238                        subvol_inum inum,
1239                        struct readpages_iter *readpages_iter)
1240 {
1241         struct bch_fs *c = trans->c;
1242         struct btree_iter iter;
1243         struct bkey_buf sk;
1244         int flags = BCH_READ_RETRY_IF_STALE|
1245                 BCH_READ_MAY_PROMOTE;
1246         u32 snapshot;
1247         int ret = 0;
1248
1249         rbio->c = c;
1250         rbio->start_time = local_clock();
1251         rbio->subvol = inum.subvol;
1252
1253         bch2_bkey_buf_init(&sk);
1254 retry:
1255         bch2_trans_begin(trans);
1256         iter = (struct btree_iter) { NULL };
1257
1258         ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot);
1259         if (ret)
1260                 goto err;
1261
1262         bch2_trans_iter_init(trans, &iter, BTREE_ID_extents,
1263                              SPOS(inum.inum, rbio->bio.bi_iter.bi_sector, snapshot),
1264                              BTREE_ITER_SLOTS);
1265         while (1) {
1266                 struct bkey_s_c k;
1267                 unsigned bytes, sectors, offset_into_extent;
1268                 enum btree_id data_btree = BTREE_ID_extents;
1269
1270                 /*
1271                  * read_extent -> io_time_reset may cause a transaction restart
1272                  * without returning an error, we need to check for that here:
1273                  */
1274                 ret = bch2_trans_relock(trans);
1275                 if (ret)
1276                         break;
1277
1278                 bch2_btree_iter_set_pos(&iter,
1279                                 POS(inum.inum, rbio->bio.bi_iter.bi_sector));
1280
1281                 k = bch2_btree_iter_peek_slot(&iter);
1282                 ret = bkey_err(k);
1283                 if (ret)
1284                         break;
1285
1286                 offset_into_extent = iter.pos.offset -
1287                         bkey_start_offset(k.k);
1288                 sectors = k.k->size - offset_into_extent;
1289
1290                 bch2_bkey_buf_reassemble(&sk, c, k);
1291
1292                 ret = bch2_read_indirect_extent(trans, &data_btree,
1293                                         &offset_into_extent, &sk);
1294                 if (ret)
1295                         break;
1296
1297                 k = bkey_i_to_s_c(sk.k);
1298
1299                 sectors = min(sectors, k.k->size - offset_into_extent);
1300
1301                 if (readpages_iter) {
1302                         ret = readpage_bio_extend(trans, readpages_iter, &rbio->bio, sectors,
1303                                                   extent_partial_reads_expensive(k));
1304                         if (ret)
1305                                 break;
1306                 }
1307
1308                 bytes = min(sectors, bio_sectors(&rbio->bio)) << 9;
1309                 swap(rbio->bio.bi_iter.bi_size, bytes);
1310
1311                 if (rbio->bio.bi_iter.bi_size == bytes)
1312                         flags |= BCH_READ_LAST_FRAGMENT;
1313
1314                 bch2_bio_page_state_set(&rbio->bio, k);
1315
1316                 bch2_read_extent(trans, rbio, iter.pos,
1317                                  data_btree, k, offset_into_extent, flags);
1318
1319                 if (flags & BCH_READ_LAST_FRAGMENT)
1320                         break;
1321
1322                 swap(rbio->bio.bi_iter.bi_size, bytes);
1323                 bio_advance(&rbio->bio, bytes);
1324
1325                 ret = btree_trans_too_many_iters(trans);
1326                 if (ret)
1327                         break;
1328         }
1329 err:
1330         bch2_trans_iter_exit(trans, &iter);
1331
1332         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
1333                 goto retry;
1334
1335         if (ret) {
1336                 bch_err_inum_offset_ratelimited(c,
1337                                 iter.pos.inode,
1338                                 iter.pos.offset << 9,
1339                                 "read error %i from btree lookup", ret);
1340                 rbio->bio.bi_status = BLK_STS_IOERR;
1341                 bio_endio(&rbio->bio);
1342         }
1343
1344         bch2_bkey_buf_exit(&sk, c);
1345 }
1346
1347 void bch2_readahead(struct readahead_control *ractl)
1348 {
1349         struct bch_inode_info *inode = to_bch_ei(ractl->mapping->host);
1350         struct bch_fs *c = inode->v.i_sb->s_fs_info;
1351         struct bch_io_opts opts;
1352         struct btree_trans trans;
1353         struct folio *folio;
1354         struct readpages_iter readpages_iter;
1355         int ret;
1356
1357         bch2_inode_opts_get(&opts, c, &inode->ei_inode);
1358
1359         ret = readpages_iter_init(&readpages_iter, ractl);
1360         BUG_ON(ret);
1361
1362         bch2_trans_init(&trans, c, 0, 0);
1363
1364         bch2_pagecache_add_get(inode);
1365
1366         while ((folio = readpage_iter_peek(&readpages_iter))) {
1367                 unsigned n = min_t(unsigned,
1368                                    readpages_iter.folios.nr -
1369                                    readpages_iter.idx,
1370                                    BIO_MAX_VECS);
1371                 struct bch_read_bio *rbio =
1372                         rbio_init(bio_alloc_bioset(NULL, n, REQ_OP_READ,
1373                                                    GFP_KERNEL, &c->bio_read),
1374                                   opts);
1375
1376                 readpage_iter_advance(&readpages_iter);
1377
1378                 rbio->bio.bi_iter.bi_sector = folio_sector(folio);
1379                 rbio->bio.bi_end_io = bch2_readpages_end_io;
1380                 BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0));
1381
1382                 bchfs_read(&trans, rbio, inode_inum(inode),
1383                            &readpages_iter);
1384                 bch2_trans_unlock(&trans);
1385         }
1386
1387         bch2_pagecache_add_put(inode);
1388
1389         bch2_trans_exit(&trans);
1390         darray_exit(&readpages_iter.folios);
1391 }
1392
1393 static void __bchfs_readfolio(struct bch_fs *c, struct bch_read_bio *rbio,
1394                              subvol_inum inum, struct folio *folio)
1395 {
1396         struct btree_trans trans;
1397
1398         bch2_folio_create(folio, __GFP_NOFAIL);
1399
1400         rbio->bio.bi_opf = REQ_OP_READ|REQ_SYNC;
1401         rbio->bio.bi_iter.bi_sector = folio_sector(folio);
1402         BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0));
1403
1404         bch2_trans_init(&trans, c, 0, 0);
1405         bchfs_read(&trans, rbio, inum, NULL);
1406         bch2_trans_exit(&trans);
1407 }
1408
1409 static void bch2_read_single_folio_end_io(struct bio *bio)
1410 {
1411         complete(bio->bi_private);
1412 }
1413
1414 static int bch2_read_single_folio(struct folio *folio,
1415                                   struct address_space *mapping)
1416 {
1417         struct bch_inode_info *inode = to_bch_ei(mapping->host);
1418         struct bch_fs *c = inode->v.i_sb->s_fs_info;
1419         struct bch_read_bio *rbio;
1420         struct bch_io_opts opts;
1421         int ret;
1422         DECLARE_COMPLETION_ONSTACK(done);
1423
1424         bch2_inode_opts_get(&opts, c, &inode->ei_inode);
1425
1426         rbio = rbio_init(bio_alloc_bioset(NULL, 1, REQ_OP_READ, GFP_KERNEL, &c->bio_read),
1427                          opts);
1428         rbio->bio.bi_private = &done;
1429         rbio->bio.bi_end_io = bch2_read_single_folio_end_io;
1430
1431         __bchfs_readfolio(c, rbio, inode_inum(inode), folio);
1432         wait_for_completion(&done);
1433
1434         ret = blk_status_to_errno(rbio->bio.bi_status);
1435         bio_put(&rbio->bio);
1436
1437         if (ret < 0)
1438                 return ret;
1439
1440         folio_mark_uptodate(folio);
1441         return 0;
1442 }
1443
1444 int bch2_read_folio(struct file *file, struct folio *folio)
1445 {
1446         int ret;
1447
1448         ret = bch2_read_single_folio(folio, folio->mapping);
1449         folio_unlock(folio);
1450         return bch2_err_class(ret);
1451 }
1452
1453 /* writepages: */
1454
1455 struct bch_writepage_state {
1456         struct bch_writepage_io *io;
1457         struct bch_io_opts      opts;
1458         struct bch_folio_sector *tmp;
1459         unsigned                tmp_sectors;
1460 };
1461
1462 static inline struct bch_writepage_state bch_writepage_state_init(struct bch_fs *c,
1463                                                                   struct bch_inode_info *inode)
1464 {
1465         struct bch_writepage_state ret = { 0 };
1466
1467         bch2_inode_opts_get(&ret.opts, c, &inode->ei_inode);
1468         return ret;
1469 }
1470
1471 static void bch2_writepage_io_done(struct bch_write_op *op)
1472 {
1473         struct bch_writepage_io *io =
1474                 container_of(op, struct bch_writepage_io, op);
1475         struct bch_fs *c = io->op.c;
1476         struct bio *bio = &io->op.wbio.bio;
1477         struct folio_iter fi;
1478         unsigned i;
1479
1480         if (io->op.error) {
1481                 set_bit(EI_INODE_ERROR, &io->inode->ei_flags);
1482
1483                 bio_for_each_folio_all(fi, bio) {
1484                         struct bch_folio *s;
1485
1486                         folio_set_error(fi.folio);
1487                         mapping_set_error(fi.folio->mapping, -EIO);
1488
1489                         s = __bch2_folio(fi.folio);
1490                         spin_lock(&s->lock);
1491                         for (i = 0; i < folio_sectors(fi.folio); i++)
1492                                 s->s[i].nr_replicas = 0;
1493                         spin_unlock(&s->lock);
1494                 }
1495         }
1496
1497         if (io->op.flags & BCH_WRITE_WROTE_DATA_INLINE) {
1498                 bio_for_each_folio_all(fi, bio) {
1499                         struct bch_folio *s;
1500
1501                         s = __bch2_folio(fi.folio);
1502                         spin_lock(&s->lock);
1503                         for (i = 0; i < folio_sectors(fi.folio); i++)
1504                                 s->s[i].nr_replicas = 0;
1505                         spin_unlock(&s->lock);
1506                 }
1507         }
1508
1509         /*
1510          * racing with fallocate can cause us to add fewer sectors than
1511          * expected - but we shouldn't add more sectors than expected:
1512          */
1513         WARN_ON_ONCE(io->op.i_sectors_delta > 0);
1514
1515         /*
1516          * (error (due to going RO) halfway through a page can screw that up
1517          * slightly)
1518          * XXX wtf?
1519            BUG_ON(io->op.op.i_sectors_delta >= PAGE_SECTORS);
1520          */
1521
1522         /*
1523          * PageWriteback is effectively our ref on the inode - fixup i_blocks
1524          * before calling end_page_writeback:
1525          */
1526         i_sectors_acct(c, io->inode, NULL, io->op.i_sectors_delta);
1527
1528         bio_for_each_folio_all(fi, bio) {
1529                 struct bch_folio *s = __bch2_folio(fi.folio);
1530
1531                 if (atomic_dec_and_test(&s->write_count))
1532                         folio_end_writeback(fi.folio);
1533         }
1534
1535         bio_put(&io->op.wbio.bio);
1536 }
1537
1538 static void bch2_writepage_do_io(struct bch_writepage_state *w)
1539 {
1540         struct bch_writepage_io *io = w->io;
1541
1542         w->io = NULL;
1543         closure_call(&io->op.cl, bch2_write, NULL, NULL);
1544 }
1545
1546 /*
1547  * Get a bch_writepage_io and add @page to it - appending to an existing one if
1548  * possible, else allocating a new one:
1549  */
1550 static void bch2_writepage_io_alloc(struct bch_fs *c,
1551                                     struct writeback_control *wbc,
1552                                     struct bch_writepage_state *w,
1553                                     struct bch_inode_info *inode,
1554                                     u64 sector,
1555                                     unsigned nr_replicas)
1556 {
1557         struct bch_write_op *op;
1558
1559         w->io = container_of(bio_alloc_bioset(NULL, BIO_MAX_VECS,
1560                                               REQ_OP_WRITE,
1561                                               GFP_KERNEL,
1562                                               &c->writepage_bioset),
1563                              struct bch_writepage_io, op.wbio.bio);
1564
1565         w->io->inode            = inode;
1566         op                      = &w->io->op;
1567         bch2_write_op_init(op, c, w->opts);
1568         op->target              = w->opts.foreground_target;
1569         op->nr_replicas         = nr_replicas;
1570         op->res.nr_replicas     = nr_replicas;
1571         op->write_point         = writepoint_hashed(inode->ei_last_dirtied);
1572         op->subvol              = inode->ei_subvol;
1573         op->pos                 = POS(inode->v.i_ino, sector);
1574         op->end_io              = bch2_writepage_io_done;
1575         op->devs_need_flush     = &inode->ei_devs_need_flush;
1576         op->wbio.bio.bi_iter.bi_sector = sector;
1577         op->wbio.bio.bi_opf     = wbc_to_write_flags(wbc);
1578 }
1579
1580 static int __bch2_writepage(struct folio *folio,
1581                             struct writeback_control *wbc,
1582                             void *data)
1583 {
1584         struct bch_inode_info *inode = to_bch_ei(folio->mapping->host);
1585         struct bch_fs *c = inode->v.i_sb->s_fs_info;
1586         struct bch_writepage_state *w = data;
1587         struct bch_folio *s;
1588         unsigned i, offset, f_sectors, nr_replicas_this_write = U32_MAX;
1589         loff_t i_size = i_size_read(&inode->v);
1590         int ret;
1591
1592         EBUG_ON(!folio_test_uptodate(folio));
1593
1594         /* Is the folio fully inside i_size? */
1595         if (folio_end_pos(folio) <= i_size)
1596                 goto do_io;
1597
1598         /* Is the folio fully outside i_size? (truncate in progress) */
1599         if (folio_pos(folio) >= i_size) {
1600                 folio_unlock(folio);
1601                 return 0;
1602         }
1603
1604         /*
1605          * The folio straddles i_size.  It must be zeroed out on each and every
1606          * writepage invocation because it may be mmapped.  "A file is mapped
1607          * in multiples of the folio size.  For a file that is not a multiple of
1608          * the  folio size, the remaining memory is zeroed when mapped, and
1609          * writes to that region are not written out to the file."
1610          */
1611         folio_zero_segment(folio,
1612                            i_size - folio_pos(folio),
1613                            folio_size(folio));
1614 do_io:
1615         f_sectors = folio_sectors(folio);
1616         s = bch2_folio(folio);
1617
1618         if (f_sectors > w->tmp_sectors) {
1619                 kfree(w->tmp);
1620                 w->tmp = kzalloc(sizeof(struct bch_folio_sector) *
1621                                  f_sectors, __GFP_NOFAIL);
1622                 w->tmp_sectors = f_sectors;
1623         }
1624
1625         /*
1626          * Things get really hairy with errors during writeback:
1627          */
1628         ret = bch2_get_folio_disk_reservation(c, inode, folio, false);
1629         BUG_ON(ret);
1630
1631         /* Before unlocking the page, get copy of reservations: */
1632         spin_lock(&s->lock);
1633         memcpy(w->tmp, s->s, sizeof(struct bch_folio_sector) * f_sectors);
1634
1635         for (i = 0; i < f_sectors; i++) {
1636                 if (s->s[i].state < SECTOR_dirty)
1637                         continue;
1638
1639                 nr_replicas_this_write =
1640                         min_t(unsigned, nr_replicas_this_write,
1641                               s->s[i].nr_replicas +
1642                               s->s[i].replicas_reserved);
1643         }
1644
1645         for (i = 0; i < f_sectors; i++) {
1646                 if (s->s[i].state < SECTOR_dirty)
1647                         continue;
1648
1649                 s->s[i].nr_replicas = w->opts.compression
1650                         ? 0 : nr_replicas_this_write;
1651
1652                 s->s[i].replicas_reserved = 0;
1653                 folio_sector_set(folio, s, i, SECTOR_allocated);
1654         }
1655         spin_unlock(&s->lock);
1656
1657         BUG_ON(atomic_read(&s->write_count));
1658         atomic_set(&s->write_count, 1);
1659
1660         BUG_ON(folio_test_writeback(folio));
1661         folio_start_writeback(folio);
1662
1663         folio_unlock(folio);
1664
1665         offset = 0;
1666         while (1) {
1667                 unsigned sectors = 0, dirty_sectors = 0, reserved_sectors = 0;
1668                 u64 sector;
1669
1670                 while (offset < f_sectors &&
1671                        w->tmp[offset].state < SECTOR_dirty)
1672                         offset++;
1673
1674                 if (offset == f_sectors)
1675                         break;
1676
1677                 while (offset + sectors < f_sectors &&
1678                        w->tmp[offset + sectors].state >= SECTOR_dirty) {
1679                         reserved_sectors += w->tmp[offset + sectors].replicas_reserved;
1680                         dirty_sectors += w->tmp[offset + sectors].state == SECTOR_dirty;
1681                         sectors++;
1682                 }
1683                 BUG_ON(!sectors);
1684
1685                 sector = folio_sector(folio) + offset;
1686
1687                 if (w->io &&
1688                     (w->io->op.res.nr_replicas != nr_replicas_this_write ||
1689                      bio_full(&w->io->op.wbio.bio, sectors << 9) ||
1690                      w->io->op.wbio.bio.bi_iter.bi_size + (sectors << 9) >=
1691                      (BIO_MAX_VECS * PAGE_SIZE) ||
1692                      bio_end_sector(&w->io->op.wbio.bio) != sector))
1693                         bch2_writepage_do_io(w);
1694
1695                 if (!w->io)
1696                         bch2_writepage_io_alloc(c, wbc, w, inode, sector,
1697                                                 nr_replicas_this_write);
1698
1699                 atomic_inc(&s->write_count);
1700
1701                 BUG_ON(inode != w->io->inode);
1702                 BUG_ON(!bio_add_folio(&w->io->op.wbio.bio, folio,
1703                                      sectors << 9, offset << 9));
1704
1705                 /* Check for writing past i_size: */
1706                 WARN_ONCE((bio_end_sector(&w->io->op.wbio.bio) << 9) >
1707                           round_up(i_size, block_bytes(c)) &&
1708                           !test_bit(BCH_FS_EMERGENCY_RO, &c->flags),
1709                           "writing past i_size: %llu > %llu (unrounded %llu)\n",
1710                           bio_end_sector(&w->io->op.wbio.bio) << 9,
1711                           round_up(i_size, block_bytes(c)),
1712                           i_size);
1713
1714                 w->io->op.res.sectors += reserved_sectors;
1715                 w->io->op.i_sectors_delta -= dirty_sectors;
1716                 w->io->op.new_i_size = i_size;
1717
1718                 offset += sectors;
1719         }
1720
1721         if (atomic_dec_and_test(&s->write_count))
1722                 folio_end_writeback(folio);
1723
1724         return 0;
1725 }
1726
1727 int bch2_writepages(struct address_space *mapping, struct writeback_control *wbc)
1728 {
1729         struct bch_fs *c = mapping->host->i_sb->s_fs_info;
1730         struct bch_writepage_state w =
1731                 bch_writepage_state_init(c, to_bch_ei(mapping->host));
1732         struct blk_plug plug;
1733         int ret;
1734
1735         blk_start_plug(&plug);
1736         ret = write_cache_pages(mapping, wbc, __bch2_writepage, &w);
1737         if (w.io)
1738                 bch2_writepage_do_io(&w);
1739         blk_finish_plug(&plug);
1740         kfree(w.tmp);
1741         return bch2_err_class(ret);
1742 }
1743
1744 /* buffered writes: */
1745
1746 int bch2_write_begin(struct file *file, struct address_space *mapping,
1747                      loff_t pos, unsigned len,
1748                      struct page **pagep, void **fsdata)
1749 {
1750         struct bch_inode_info *inode = to_bch_ei(mapping->host);
1751         struct bch_fs *c = inode->v.i_sb->s_fs_info;
1752         struct bch2_folio_reservation *res;
1753         struct folio *folio;
1754         unsigned offset;
1755         int ret = -ENOMEM;
1756
1757         res = kmalloc(sizeof(*res), GFP_KERNEL);
1758         if (!res)
1759                 return -ENOMEM;
1760
1761         bch2_folio_reservation_init(c, inode, res);
1762         *fsdata = res;
1763
1764         bch2_pagecache_add_get(inode);
1765
1766         folio = __filemap_get_folio(mapping, pos >> PAGE_SHIFT,
1767                                 FGP_LOCK|FGP_WRITE|FGP_CREAT|FGP_STABLE,
1768                                 mapping_gfp_mask(mapping));
1769         if (IS_ERR_OR_NULL(folio))
1770                 goto err_unlock;
1771
1772         if (folio_test_uptodate(folio))
1773                 goto out;
1774
1775         offset = pos - folio_pos(folio);
1776         len = min_t(size_t, len, folio_end_pos(folio) - pos);
1777
1778         /* If we're writing entire folio, don't need to read it in first: */
1779         if (!offset && len == folio_size(folio))
1780                 goto out;
1781
1782         if (!offset && pos + len >= inode->v.i_size) {
1783                 folio_zero_segment(folio, len, folio_size(folio));
1784                 flush_dcache_folio(folio);
1785                 goto out;
1786         }
1787
1788         if (folio_pos(folio) >= inode->v.i_size) {
1789                 folio_zero_segments(folio, 0, offset, offset + len, folio_size(folio));
1790                 flush_dcache_folio(folio);
1791                 goto out;
1792         }
1793 readpage:
1794         ret = bch2_read_single_folio(folio, mapping);
1795         if (ret)
1796                 goto err;
1797 out:
1798         ret = bch2_folio_set(c, inode_inum(inode), &folio, 1);
1799         if (ret)
1800                 goto err;
1801
1802         ret = bch2_folio_reservation_get(c, inode, folio, res, offset, len);
1803         if (ret) {
1804                 if (!folio_test_uptodate(folio)) {
1805                         /*
1806                          * If the folio hasn't been read in, we won't know if we
1807                          * actually need a reservation - we don't actually need
1808                          * to read here, we just need to check if the folio is
1809                          * fully backed by uncompressed data:
1810                          */
1811                         goto readpage;
1812                 }
1813
1814                 goto err;
1815         }
1816
1817         *pagep = &folio->page;
1818         return 0;
1819 err:
1820         folio_unlock(folio);
1821         folio_put(folio);
1822         *pagep = NULL;
1823 err_unlock:
1824         bch2_pagecache_add_put(inode);
1825         kfree(res);
1826         *fsdata = NULL;
1827         return bch2_err_class(ret);
1828 }
1829
1830 int bch2_write_end(struct file *file, struct address_space *mapping,
1831                    loff_t pos, unsigned len, unsigned copied,
1832                    struct page *page, void *fsdata)
1833 {
1834         struct bch_inode_info *inode = to_bch_ei(mapping->host);
1835         struct bch_fs *c = inode->v.i_sb->s_fs_info;
1836         struct bch2_folio_reservation *res = fsdata;
1837         struct folio *folio = page_folio(page);
1838         unsigned offset = pos - folio_pos(folio);
1839
1840         lockdep_assert_held(&inode->v.i_rwsem);
1841         BUG_ON(offset + copied > folio_size(folio));
1842
1843         if (unlikely(copied < len && !folio_test_uptodate(folio))) {
1844                 /*
1845                  * The folio needs to be read in, but that would destroy
1846                  * our partial write - simplest thing is to just force
1847                  * userspace to redo the write:
1848                  */
1849                 folio_zero_range(folio, 0, folio_size(folio));
1850                 flush_dcache_folio(folio);
1851                 copied = 0;
1852         }
1853
1854         spin_lock(&inode->v.i_lock);
1855         if (pos + copied > inode->v.i_size)
1856                 i_size_write(&inode->v, pos + copied);
1857         spin_unlock(&inode->v.i_lock);
1858
1859         if (copied) {
1860                 if (!folio_test_uptodate(folio))
1861                         folio_mark_uptodate(folio);
1862
1863                 bch2_set_folio_dirty(c, inode, folio, res, offset, copied);
1864
1865                 inode->ei_last_dirtied = (unsigned long) current;
1866         }
1867
1868         folio_unlock(folio);
1869         folio_put(folio);
1870         bch2_pagecache_add_put(inode);
1871
1872         bch2_folio_reservation_put(c, inode, res);
1873         kfree(res);
1874
1875         return copied;
1876 }
1877
1878 static noinline void folios_trunc(folios *folios, struct folio **fi)
1879 {
1880         while (folios->data + folios->nr > fi) {
1881                 struct folio *f = darray_pop(folios);
1882
1883                 folio_unlock(f);
1884                 folio_put(f);
1885         }
1886 }
1887
1888 static int __bch2_buffered_write(struct bch_inode_info *inode,
1889                                  struct address_space *mapping,
1890                                  struct iov_iter *iter,
1891                                  loff_t pos, unsigned len)
1892 {
1893         struct bch_fs *c = inode->v.i_sb->s_fs_info;
1894         struct bch2_folio_reservation res;
1895         folios folios;
1896         struct folio **fi, *f;
1897         unsigned copied = 0, f_offset;
1898         u64 end = pos + len, f_pos;
1899         loff_t last_folio_pos = inode->v.i_size;
1900         int ret = 0;
1901
1902         BUG_ON(!len);
1903
1904         bch2_folio_reservation_init(c, inode, &res);
1905         darray_init(&folios);
1906
1907         ret = filemap_get_contig_folios_d(mapping, pos, end,
1908                                    FGP_LOCK|FGP_WRITE|FGP_STABLE|FGP_CREAT,
1909                                    mapping_gfp_mask(mapping),
1910                                    &folios);
1911         if (ret)
1912                 goto out;
1913
1914         BUG_ON(!folios.nr);
1915
1916         f = darray_first(folios);
1917         if (pos != folio_pos(f) && !folio_test_uptodate(f)) {
1918                 ret = bch2_read_single_folio(f, mapping);
1919                 if (ret)
1920                         goto out;
1921         }
1922
1923         f = darray_last(folios);
1924         end = min(end, folio_end_pos(f));
1925         last_folio_pos = folio_pos(f);
1926         if (end != folio_end_pos(f) && !folio_test_uptodate(f)) {
1927                 if (end >= inode->v.i_size) {
1928                         folio_zero_range(f, 0, folio_size(f));
1929                 } else {
1930                         ret = bch2_read_single_folio(f, mapping);
1931                         if (ret)
1932                                 goto out;
1933                 }
1934         }
1935
1936         ret = bch2_folio_set(c, inode_inum(inode), folios.data, folios.nr);
1937         if (ret)
1938                 goto out;
1939
1940         f_pos = pos;
1941         f_offset = pos - folio_pos(darray_first(folios));
1942         darray_for_each(folios, fi) {
1943                 struct folio *f = *fi;
1944                 u64 f_len = min(end, folio_end_pos(f)) - f_pos;
1945
1946                 /*
1947                  * XXX: per POSIX and fstests generic/275, on -ENOSPC we're
1948                  * supposed to write as much as we have disk space for.
1949                  *
1950                  * On failure here we should still write out a partial page if
1951                  * we aren't completely out of disk space - we don't do that
1952                  * yet:
1953                  */
1954                 ret = bch2_folio_reservation_get(c, inode, f, &res, f_offset, f_len);
1955                 if (unlikely(ret)) {
1956                         folios_trunc(&folios, fi);
1957                         if (!folios.nr)
1958                                 goto out;
1959
1960                         end = min(end, folio_end_pos(darray_last(folios)));
1961                         break;
1962                 }
1963
1964                 f_pos = folio_end_pos(f);
1965                 f_offset = 0;
1966         }
1967
1968         if (mapping_writably_mapped(mapping))
1969                 darray_for_each(folios, fi)
1970                         flush_dcache_folio(*fi);
1971
1972         f_pos = pos;
1973         f_offset = pos - folio_pos(darray_first(folios));
1974         darray_for_each(folios, fi) {
1975                 struct folio *f = *fi;
1976                 u64 f_len = min(end, folio_end_pos(f)) - f_pos;
1977                 unsigned f_copied = copy_page_from_iter_atomic(&f->page, f_offset, f_len, iter);
1978
1979                 if (!f_copied) {
1980                         folios_trunc(&folios, fi);
1981                         break;
1982                 }
1983
1984                 if (!folio_test_uptodate(f) &&
1985                     f_copied != folio_size(f) &&
1986                     pos + copied + f_copied < inode->v.i_size) {
1987                         folio_zero_range(f, 0, folio_size(f));
1988                         folios_trunc(&folios, fi);
1989                         break;
1990                 }
1991
1992                 flush_dcache_folio(f);
1993                 copied += f_copied;
1994
1995                 if (f_copied != f_len) {
1996                         folios_trunc(&folios, fi + 1);
1997                         break;
1998                 }
1999
2000                 f_pos = folio_end_pos(f);
2001                 f_offset = 0;
2002         }
2003
2004         if (!copied)
2005                 goto out;
2006
2007         end = pos + copied;
2008
2009         spin_lock(&inode->v.i_lock);
2010         if (end > inode->v.i_size)
2011                 i_size_write(&inode->v, end);
2012         spin_unlock(&inode->v.i_lock);
2013
2014         f_pos = pos;
2015         f_offset = pos - folio_pos(darray_first(folios));
2016         darray_for_each(folios, fi) {
2017                 struct folio *f = *fi;
2018                 u64 f_len = min(end, folio_end_pos(f)) - f_pos;
2019
2020                 if (!folio_test_uptodate(f))
2021                         folio_mark_uptodate(f);
2022
2023                 bch2_set_folio_dirty(c, inode, f, &res, f_offset, f_len);
2024
2025                 f_pos = folio_end_pos(f);
2026                 f_offset = 0;
2027         }
2028
2029         inode->ei_last_dirtied = (unsigned long) current;
2030 out:
2031         darray_for_each(folios, fi) {
2032                 folio_unlock(*fi);
2033                 folio_put(*fi);
2034         }
2035
2036         /*
2037          * If the last folio added to the mapping starts beyond current EOF, we
2038          * performed a short write but left around at least one post-EOF folio.
2039          * Clean up the mapping before we return.
2040          */
2041         if (last_folio_pos >= inode->v.i_size)
2042                 truncate_pagecache(&inode->v, inode->v.i_size);
2043
2044         darray_exit(&folios);
2045         bch2_folio_reservation_put(c, inode, &res);
2046
2047         return copied ?: ret;
2048 }
2049
2050 static ssize_t bch2_buffered_write(struct kiocb *iocb, struct iov_iter *iter)
2051 {
2052         struct file *file = iocb->ki_filp;
2053         struct address_space *mapping = file->f_mapping;
2054         struct bch_inode_info *inode = file_bch_inode(file);
2055         loff_t pos = iocb->ki_pos;
2056         ssize_t written = 0;
2057         int ret = 0;
2058
2059         bch2_pagecache_add_get(inode);
2060
2061         do {
2062                 unsigned offset = pos & (PAGE_SIZE - 1);
2063                 unsigned bytes = iov_iter_count(iter);
2064 again:
2065                 /*
2066                  * Bring in the user page that we will copy from _first_.
2067                  * Otherwise there's a nasty deadlock on copying from the
2068                  * same page as we're writing to, without it being marked
2069                  * up-to-date.
2070                  *
2071                  * Not only is this an optimisation, but it is also required
2072                  * to check that the address is actually valid, when atomic
2073                  * usercopies are used, below.
2074                  */
2075                 if (unlikely(fault_in_iov_iter_readable(iter, bytes))) {
2076                         bytes = min_t(unsigned long, iov_iter_count(iter),
2077                                       PAGE_SIZE - offset);
2078
2079                         if (unlikely(fault_in_iov_iter_readable(iter, bytes))) {
2080                                 ret = -EFAULT;
2081                                 break;
2082                         }
2083                 }
2084
2085                 if (unlikely(fatal_signal_pending(current))) {
2086                         ret = -EINTR;
2087                         break;
2088                 }
2089
2090                 ret = __bch2_buffered_write(inode, mapping, iter, pos, bytes);
2091                 if (unlikely(ret < 0))
2092                         break;
2093
2094                 cond_resched();
2095
2096                 if (unlikely(ret == 0)) {
2097                         /*
2098                          * If we were unable to copy any data at all, we must
2099                          * fall back to a single segment length write.
2100                          *
2101                          * If we didn't fallback here, we could livelock
2102                          * because not all segments in the iov can be copied at
2103                          * once without a pagefault.
2104                          */
2105                         bytes = min_t(unsigned long, PAGE_SIZE - offset,
2106                                       iov_iter_single_seg_count(iter));
2107                         goto again;
2108                 }
2109                 pos += ret;
2110                 written += ret;
2111                 ret = 0;
2112
2113                 balance_dirty_pages_ratelimited(mapping);
2114         } while (iov_iter_count(iter));
2115
2116         bch2_pagecache_add_put(inode);
2117
2118         return written ? written : ret;
2119 }
2120
2121 /* O_DIRECT reads */
2122
2123 static void bio_check_or_release(struct bio *bio, bool check_dirty)
2124 {
2125         if (check_dirty) {
2126                 bio_check_pages_dirty(bio);
2127         } else {
2128                 bio_release_pages(bio, false);
2129                 bio_put(bio);
2130         }
2131 }
2132
2133 static void bch2_dio_read_complete(struct closure *cl)
2134 {
2135         struct dio_read *dio = container_of(cl, struct dio_read, cl);
2136
2137         dio->req->ki_complete(dio->req, dio->ret);
2138         bio_check_or_release(&dio->rbio.bio, dio->should_dirty);
2139 }
2140
2141 static void bch2_direct_IO_read_endio(struct bio *bio)
2142 {
2143         struct dio_read *dio = bio->bi_private;
2144
2145         if (bio->bi_status)
2146                 dio->ret = blk_status_to_errno(bio->bi_status);
2147
2148         closure_put(&dio->cl);
2149 }
2150
2151 static void bch2_direct_IO_read_split_endio(struct bio *bio)
2152 {
2153         struct dio_read *dio = bio->bi_private;
2154         bool should_dirty = dio->should_dirty;
2155
2156         bch2_direct_IO_read_endio(bio);
2157         bio_check_or_release(bio, should_dirty);
2158 }
2159
2160 static int bch2_direct_IO_read(struct kiocb *req, struct iov_iter *iter)
2161 {
2162         struct file *file = req->ki_filp;
2163         struct bch_inode_info *inode = file_bch_inode(file);
2164         struct bch_fs *c = inode->v.i_sb->s_fs_info;
2165         struct bch_io_opts opts;
2166         struct dio_read *dio;
2167         struct bio *bio;
2168         loff_t offset = req->ki_pos;
2169         bool sync = is_sync_kiocb(req);
2170         size_t shorten;
2171         ssize_t ret;
2172
2173         bch2_inode_opts_get(&opts, c, &inode->ei_inode);
2174
2175         if ((offset|iter->count) & (block_bytes(c) - 1))
2176                 return -EINVAL;
2177
2178         ret = min_t(loff_t, iter->count,
2179                     max_t(loff_t, 0, i_size_read(&inode->v) - offset));
2180
2181         if (!ret)
2182                 return ret;
2183
2184         shorten = iov_iter_count(iter) - round_up(ret, block_bytes(c));
2185         iter->count -= shorten;
2186
2187         bio = bio_alloc_bioset(NULL,
2188                                bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS),
2189                                REQ_OP_READ,
2190                                GFP_KERNEL,
2191                                &c->dio_read_bioset);
2192
2193         bio->bi_end_io = bch2_direct_IO_read_endio;
2194
2195         dio = container_of(bio, struct dio_read, rbio.bio);
2196         closure_init(&dio->cl, NULL);
2197
2198         /*
2199          * this is a _really_ horrible hack just to avoid an atomic sub at the
2200          * end:
2201          */
2202         if (!sync) {
2203                 set_closure_fn(&dio->cl, bch2_dio_read_complete, NULL);
2204                 atomic_set(&dio->cl.remaining,
2205                            CLOSURE_REMAINING_INITIALIZER -
2206                            CLOSURE_RUNNING +
2207                            CLOSURE_DESTRUCTOR);
2208         } else {
2209                 atomic_set(&dio->cl.remaining,
2210                            CLOSURE_REMAINING_INITIALIZER + 1);
2211         }
2212
2213         dio->req        = req;
2214         dio->ret        = ret;
2215         /*
2216          * This is one of the sketchier things I've encountered: we have to skip
2217          * the dirtying of requests that are internal from the kernel (i.e. from
2218          * loopback), because we'll deadlock on page_lock.
2219          */
2220         dio->should_dirty = iter_is_iovec(iter);
2221
2222         goto start;
2223         while (iter->count) {
2224                 bio = bio_alloc_bioset(NULL,
2225                                        bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS),
2226                                        REQ_OP_READ,
2227                                        GFP_KERNEL,
2228                                        &c->bio_read);
2229                 bio->bi_end_io          = bch2_direct_IO_read_split_endio;
2230 start:
2231                 bio->bi_opf             = REQ_OP_READ|REQ_SYNC;
2232                 bio->bi_iter.bi_sector  = offset >> 9;
2233                 bio->bi_private         = dio;
2234
2235                 ret = bio_iov_iter_get_pages(bio, iter);
2236                 if (ret < 0) {
2237                         /* XXX: fault inject this path */
2238                         bio->bi_status = BLK_STS_RESOURCE;
2239                         bio_endio(bio);
2240                         break;
2241                 }
2242
2243                 offset += bio->bi_iter.bi_size;
2244
2245                 if (dio->should_dirty)
2246                         bio_set_pages_dirty(bio);
2247
2248                 if (iter->count)
2249                         closure_get(&dio->cl);
2250
2251                 bch2_read(c, rbio_init(bio, opts), inode_inum(inode));
2252         }
2253
2254         iter->count += shorten;
2255
2256         if (sync) {
2257                 closure_sync(&dio->cl);
2258                 closure_debug_destroy(&dio->cl);
2259                 ret = dio->ret;
2260                 bio_check_or_release(&dio->rbio.bio, dio->should_dirty);
2261                 return ret;
2262         } else {
2263                 return -EIOCBQUEUED;
2264         }
2265 }
2266
2267 ssize_t bch2_read_iter(struct kiocb *iocb, struct iov_iter *iter)
2268 {
2269         struct file *file = iocb->ki_filp;
2270         struct bch_inode_info *inode = file_bch_inode(file);
2271         struct address_space *mapping = file->f_mapping;
2272         size_t count = iov_iter_count(iter);
2273         ssize_t ret;
2274
2275         if (!count)
2276                 return 0; /* skip atime */
2277
2278         if (iocb->ki_flags & IOCB_DIRECT) {
2279                 struct blk_plug plug;
2280
2281                 if (unlikely(mapping->nrpages)) {
2282                         ret = filemap_write_and_wait_range(mapping,
2283                                                 iocb->ki_pos,
2284                                                 iocb->ki_pos + count - 1);
2285                         if (ret < 0)
2286                                 goto out;
2287                 }
2288
2289                 file_accessed(file);
2290
2291                 blk_start_plug(&plug);
2292                 ret = bch2_direct_IO_read(iocb, iter);
2293                 blk_finish_plug(&plug);
2294
2295                 if (ret >= 0)
2296                         iocb->ki_pos += ret;
2297         } else {
2298                 bch2_pagecache_add_get(inode);
2299                 ret = generic_file_read_iter(iocb, iter);
2300                 bch2_pagecache_add_put(inode);
2301         }
2302 out:
2303         return bch2_err_class(ret);
2304 }
2305
2306 /* O_DIRECT writes */
2307
2308 static bool bch2_check_range_allocated(struct bch_fs *c, subvol_inum inum,
2309                                        u64 offset, u64 size,
2310                                        unsigned nr_replicas, bool compressed)
2311 {
2312         struct btree_trans trans;
2313         struct btree_iter iter;
2314         struct bkey_s_c k;
2315         u64 end = offset + size;
2316         u32 snapshot;
2317         bool ret = true;
2318         int err;
2319
2320         bch2_trans_init(&trans, c, 0, 0);
2321 retry:
2322         bch2_trans_begin(&trans);
2323
2324         err = bch2_subvolume_get_snapshot(&trans, inum.subvol, &snapshot);
2325         if (err)
2326                 goto err;
2327
2328         for_each_btree_key_norestart(&trans, iter, BTREE_ID_extents,
2329                            SPOS(inum.inum, offset, snapshot),
2330                            BTREE_ITER_SLOTS, k, err) {
2331                 if (bkey_ge(bkey_start_pos(k.k), POS(inum.inum, end)))
2332                         break;
2333
2334                 if (k.k->p.snapshot != snapshot ||
2335                     nr_replicas > bch2_bkey_replicas(c, k) ||
2336                     (!compressed && bch2_bkey_sectors_compressed(k))) {
2337                         ret = false;
2338                         break;
2339                 }
2340         }
2341
2342         offset = iter.pos.offset;
2343         bch2_trans_iter_exit(&trans, &iter);
2344 err:
2345         if (bch2_err_matches(err, BCH_ERR_transaction_restart))
2346                 goto retry;
2347         bch2_trans_exit(&trans);
2348
2349         return err ? false : ret;
2350 }
2351
2352 static noinline bool bch2_dio_write_check_allocated(struct dio_write *dio)
2353 {
2354         struct bch_fs *c = dio->op.c;
2355         struct bch_inode_info *inode = dio->inode;
2356         struct bio *bio = &dio->op.wbio.bio;
2357
2358         return bch2_check_range_allocated(c, inode_inum(inode),
2359                                 dio->op.pos.offset, bio_sectors(bio),
2360                                 dio->op.opts.data_replicas,
2361                                 dio->op.opts.compression != 0);
2362 }
2363
2364 static void bch2_dio_write_loop_async(struct bch_write_op *);
2365 static __always_inline long bch2_dio_write_done(struct dio_write *dio);
2366
2367 /*
2368  * We're going to return -EIOCBQUEUED, but we haven't finished consuming the
2369  * iov_iter yet, so we need to stash a copy of the iovec: it might be on the
2370  * caller's stack, we're not guaranteed that it will live for the duration of
2371  * the IO:
2372  */
2373 static noinline int bch2_dio_write_copy_iov(struct dio_write *dio)
2374 {
2375         struct iovec *iov = dio->inline_vecs;
2376
2377         /*
2378          * iov_iter has a single embedded iovec - nothing to do:
2379          */
2380         if (iter_is_ubuf(&dio->iter))
2381                 return 0;
2382
2383         /*
2384          * We don't currently handle non-iovec iov_iters here - return an error,
2385          * and we'll fall back to doing the IO synchronously:
2386          */
2387         if (!iter_is_iovec(&dio->iter))
2388                 return -1;
2389
2390         if (dio->iter.nr_segs > ARRAY_SIZE(dio->inline_vecs)) {
2391                 iov = kmalloc_array(dio->iter.nr_segs, sizeof(*iov),
2392                                     GFP_KERNEL);
2393                 if (unlikely(!iov))
2394                         return -ENOMEM;
2395
2396                 dio->free_iov = true;
2397         }
2398
2399         memcpy(iov, dio->iter.__iov, dio->iter.nr_segs * sizeof(*iov));
2400         dio->iter.__iov = iov;
2401         return 0;
2402 }
2403
2404 static void bch2_dio_write_flush_done(struct closure *cl)
2405 {
2406         struct dio_write *dio = container_of(cl, struct dio_write, op.cl);
2407         struct bch_fs *c = dio->op.c;
2408
2409         closure_debug_destroy(cl);
2410
2411         dio->op.error = bch2_journal_error(&c->journal);
2412
2413         bch2_dio_write_done(dio);
2414 }
2415
2416 static noinline void bch2_dio_write_flush(struct dio_write *dio)
2417 {
2418         struct bch_fs *c = dio->op.c;
2419         struct bch_inode_unpacked inode;
2420         int ret;
2421
2422         dio->flush = 0;
2423
2424         closure_init(&dio->op.cl, NULL);
2425
2426         if (!dio->op.error) {
2427                 ret = bch2_inode_find_by_inum(c, inode_inum(dio->inode), &inode);
2428                 if (ret) {
2429                         dio->op.error = ret;
2430                 } else {
2431                         bch2_journal_flush_seq_async(&c->journal, inode.bi_journal_seq, &dio->op.cl);
2432                         bch2_inode_flush_nocow_writes_async(c, dio->inode, &dio->op.cl);
2433                 }
2434         }
2435
2436         if (dio->sync) {
2437                 closure_sync(&dio->op.cl);
2438                 closure_debug_destroy(&dio->op.cl);
2439         } else {
2440                 continue_at(&dio->op.cl, bch2_dio_write_flush_done, NULL);
2441         }
2442 }
2443
2444 static __always_inline long bch2_dio_write_done(struct dio_write *dio)
2445 {
2446         struct kiocb *req = dio->req;
2447         struct bch_inode_info *inode = dio->inode;
2448         bool sync = dio->sync;
2449         long ret;
2450
2451         if (unlikely(dio->flush)) {
2452                 bch2_dio_write_flush(dio);
2453                 if (!sync)
2454                         return -EIOCBQUEUED;
2455         }
2456
2457         bch2_pagecache_block_put(inode);
2458
2459         if (dio->free_iov)
2460                 kfree(dio->iter.__iov);
2461
2462         ret = dio->op.error ?: ((long) dio->written << 9);
2463         bio_put(&dio->op.wbio.bio);
2464
2465         /* inode->i_dio_count is our ref on inode and thus bch_fs */
2466         inode_dio_end(&inode->v);
2467
2468         if (ret < 0)
2469                 ret = bch2_err_class(ret);
2470
2471         if (!sync) {
2472                 req->ki_complete(req, ret);
2473                 ret = -EIOCBQUEUED;
2474         }
2475         return ret;
2476 }
2477
2478 static __always_inline void bch2_dio_write_end(struct dio_write *dio)
2479 {
2480         struct bch_fs *c = dio->op.c;
2481         struct kiocb *req = dio->req;
2482         struct bch_inode_info *inode = dio->inode;
2483         struct bio *bio = &dio->op.wbio.bio;
2484
2485         req->ki_pos     += (u64) dio->op.written << 9;
2486         dio->written    += dio->op.written;
2487
2488         if (dio->extending) {
2489                 spin_lock(&inode->v.i_lock);
2490                 if (req->ki_pos > inode->v.i_size)
2491                         i_size_write(&inode->v, req->ki_pos);
2492                 spin_unlock(&inode->v.i_lock);
2493         }
2494
2495         if (dio->op.i_sectors_delta || dio->quota_res.sectors) {
2496                 mutex_lock(&inode->ei_quota_lock);
2497                 __i_sectors_acct(c, inode, &dio->quota_res, dio->op.i_sectors_delta);
2498                 __bch2_quota_reservation_put(c, inode, &dio->quota_res);
2499                 mutex_unlock(&inode->ei_quota_lock);
2500         }
2501
2502         bio_release_pages(bio, false);
2503
2504         if (unlikely(dio->op.error))
2505                 set_bit(EI_INODE_ERROR, &inode->ei_flags);
2506 }
2507
2508 static __always_inline long bch2_dio_write_loop(struct dio_write *dio)
2509 {
2510         struct bch_fs *c = dio->op.c;
2511         struct kiocb *req = dio->req;
2512         struct address_space *mapping = dio->mapping;
2513         struct bch_inode_info *inode = dio->inode;
2514         struct bch_io_opts opts;
2515         struct bio *bio = &dio->op.wbio.bio;
2516         unsigned unaligned, iter_count;
2517         bool sync = dio->sync, dropped_locks;
2518         long ret;
2519
2520         bch2_inode_opts_get(&opts, c, &inode->ei_inode);
2521
2522         while (1) {
2523                 iter_count = dio->iter.count;
2524
2525                 EBUG_ON(current->faults_disabled_mapping);
2526                 current->faults_disabled_mapping = mapping;
2527
2528                 ret = bio_iov_iter_get_pages(bio, &dio->iter);
2529
2530                 dropped_locks = fdm_dropped_locks();
2531
2532                 current->faults_disabled_mapping = NULL;
2533
2534                 /*
2535                  * If the fault handler returned an error but also signalled
2536                  * that it dropped & retook ei_pagecache_lock, we just need to
2537                  * re-shoot down the page cache and retry:
2538                  */
2539                 if (dropped_locks && ret)
2540                         ret = 0;
2541
2542                 if (unlikely(ret < 0))
2543                         goto err;
2544
2545                 if (unlikely(dropped_locks)) {
2546                         ret = write_invalidate_inode_pages_range(mapping,
2547                                         req->ki_pos,
2548                                         req->ki_pos + iter_count - 1);
2549                         if (unlikely(ret))
2550                                 goto err;
2551
2552                         if (!bio->bi_iter.bi_size)
2553                                 continue;
2554                 }
2555
2556                 unaligned = bio->bi_iter.bi_size & (block_bytes(c) - 1);
2557                 bio->bi_iter.bi_size -= unaligned;
2558                 iov_iter_revert(&dio->iter, unaligned);
2559
2560                 if (!bio->bi_iter.bi_size) {
2561                         /*
2562                          * bio_iov_iter_get_pages was only able to get <
2563                          * blocksize worth of pages:
2564                          */
2565                         ret = -EFAULT;
2566                         goto err;
2567                 }
2568
2569                 bch2_write_op_init(&dio->op, c, opts);
2570                 dio->op.end_io          = sync
2571                         ? NULL
2572                         : bch2_dio_write_loop_async;
2573                 dio->op.target          = dio->op.opts.foreground_target;
2574                 dio->op.write_point     = writepoint_hashed((unsigned long) current);
2575                 dio->op.nr_replicas     = dio->op.opts.data_replicas;
2576                 dio->op.subvol          = inode->ei_subvol;
2577                 dio->op.pos             = POS(inode->v.i_ino, (u64) req->ki_pos >> 9);
2578                 dio->op.devs_need_flush = &inode->ei_devs_need_flush;
2579
2580                 if (sync)
2581                         dio->op.flags |= BCH_WRITE_SYNC;
2582                 dio->op.flags |= BCH_WRITE_CHECK_ENOSPC;
2583
2584                 ret = bch2_quota_reservation_add(c, inode, &dio->quota_res,
2585                                                  bio_sectors(bio), true);
2586                 if (unlikely(ret))
2587                         goto err;
2588
2589                 ret = bch2_disk_reservation_get(c, &dio->op.res, bio_sectors(bio),
2590                                                 dio->op.opts.data_replicas, 0);
2591                 if (unlikely(ret) &&
2592                     !bch2_dio_write_check_allocated(dio))
2593                         goto err;
2594
2595                 task_io_account_write(bio->bi_iter.bi_size);
2596
2597                 if (unlikely(dio->iter.count) &&
2598                     !dio->sync &&
2599                     !dio->loop &&
2600                     bch2_dio_write_copy_iov(dio))
2601                         dio->sync = sync = true;
2602
2603                 dio->loop = true;
2604                 closure_call(&dio->op.cl, bch2_write, NULL, NULL);
2605
2606                 if (!sync)
2607                         return -EIOCBQUEUED;
2608
2609                 bch2_dio_write_end(dio);
2610
2611                 if (likely(!dio->iter.count) || dio->op.error)
2612                         break;
2613
2614                 bio_reset(bio, NULL, REQ_OP_WRITE);
2615         }
2616 out:
2617         return bch2_dio_write_done(dio);
2618 err:
2619         dio->op.error = ret;
2620
2621         bio_release_pages(bio, false);
2622
2623         bch2_quota_reservation_put(c, inode, &dio->quota_res);
2624         goto out;
2625 }
2626
2627 static noinline __cold void bch2_dio_write_continue(struct dio_write *dio)
2628 {
2629         struct mm_struct *mm = dio->mm;
2630
2631         bio_reset(&dio->op.wbio.bio, NULL, REQ_OP_WRITE);
2632
2633         if (mm)
2634                 kthread_use_mm(mm);
2635         bch2_dio_write_loop(dio);
2636         if (mm)
2637                 kthread_unuse_mm(mm);
2638 }
2639
2640 static void bch2_dio_write_loop_async(struct bch_write_op *op)
2641 {
2642         struct dio_write *dio = container_of(op, struct dio_write, op);
2643
2644         bch2_dio_write_end(dio);
2645
2646         if (likely(!dio->iter.count) || dio->op.error)
2647                 bch2_dio_write_done(dio);
2648         else
2649                 bch2_dio_write_continue(dio);
2650 }
2651
2652 static noinline
2653 ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter)
2654 {
2655         struct file *file = req->ki_filp;
2656         struct address_space *mapping = file->f_mapping;
2657         struct bch_inode_info *inode = file_bch_inode(file);
2658         struct bch_fs *c = inode->v.i_sb->s_fs_info;
2659         struct dio_write *dio;
2660         struct bio *bio;
2661         bool locked = true, extending;
2662         ssize_t ret;
2663
2664         prefetch(&c->opts);
2665         prefetch((void *) &c->opts + 64);
2666         prefetch(&inode->ei_inode);
2667         prefetch((void *) &inode->ei_inode + 64);
2668
2669         inode_lock(&inode->v);
2670
2671         ret = generic_write_checks(req, iter);
2672         if (unlikely(ret <= 0))
2673                 goto err;
2674
2675         ret = file_remove_privs(file);
2676         if (unlikely(ret))
2677                 goto err;
2678
2679         ret = file_update_time(file);
2680         if (unlikely(ret))
2681                 goto err;
2682
2683         if (unlikely((req->ki_pos|iter->count) & (block_bytes(c) - 1)))
2684                 goto err;
2685
2686         inode_dio_begin(&inode->v);
2687         bch2_pagecache_block_get(inode);
2688
2689         extending = req->ki_pos + iter->count > inode->v.i_size;
2690         if (!extending) {
2691                 inode_unlock(&inode->v);
2692                 locked = false;
2693         }
2694
2695         bio = bio_alloc_bioset(NULL,
2696                                bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS),
2697                                REQ_OP_WRITE,
2698                                GFP_KERNEL,
2699                                &c->dio_write_bioset);
2700         dio = container_of(bio, struct dio_write, op.wbio.bio);
2701         dio->req                = req;
2702         dio->mapping            = mapping;
2703         dio->inode              = inode;
2704         dio->mm                 = current->mm;
2705         dio->loop               = false;
2706         dio->extending          = extending;
2707         dio->sync               = is_sync_kiocb(req) || extending;
2708         dio->flush              = iocb_is_dsync(req) && !c->opts.journal_flush_disabled;
2709         dio->free_iov           = false;
2710         dio->quota_res.sectors  = 0;
2711         dio->written            = 0;
2712         dio->iter               = *iter;
2713         dio->op.c               = c;
2714
2715         if (unlikely(mapping->nrpages)) {
2716                 ret = write_invalidate_inode_pages_range(mapping,
2717                                                 req->ki_pos,
2718                                                 req->ki_pos + iter->count - 1);
2719                 if (unlikely(ret))
2720                         goto err_put_bio;
2721         }
2722
2723         ret = bch2_dio_write_loop(dio);
2724 err:
2725         if (locked)
2726                 inode_unlock(&inode->v);
2727         return ret;
2728 err_put_bio:
2729         bch2_pagecache_block_put(inode);
2730         bio_put(bio);
2731         inode_dio_end(&inode->v);
2732         goto err;
2733 }
2734
2735 ssize_t bch2_write_iter(struct kiocb *iocb, struct iov_iter *from)
2736 {
2737         struct file *file = iocb->ki_filp;
2738         struct bch_inode_info *inode = file_bch_inode(file);
2739         ssize_t ret;
2740
2741         if (iocb->ki_flags & IOCB_DIRECT) {
2742                 ret = bch2_direct_write(iocb, from);
2743                 goto out;
2744         }
2745
2746         /* We can write back this queue in page reclaim */
2747         current->backing_dev_info = inode_to_bdi(&inode->v);
2748         inode_lock(&inode->v);
2749
2750         ret = generic_write_checks(iocb, from);
2751         if (ret <= 0)
2752                 goto unlock;
2753
2754         ret = file_remove_privs(file);
2755         if (ret)
2756                 goto unlock;
2757
2758         ret = file_update_time(file);
2759         if (ret)
2760                 goto unlock;
2761
2762         ret = bch2_buffered_write(iocb, from);
2763         if (likely(ret > 0))
2764                 iocb->ki_pos += ret;
2765 unlock:
2766         inode_unlock(&inode->v);
2767         current->backing_dev_info = NULL;
2768
2769         if (ret > 0)
2770                 ret = generic_write_sync(iocb, ret);
2771 out:
2772         return bch2_err_class(ret);
2773 }
2774
2775 /* fsync: */
2776
2777 /*
2778  * inode->ei_inode.bi_journal_seq won't be up to date since it's set in an
2779  * insert trigger: look up the btree inode instead
2780  */
2781 static int bch2_flush_inode(struct bch_fs *c,
2782                             struct bch_inode_info *inode)
2783 {
2784         struct bch_inode_unpacked u;
2785         int ret;
2786
2787         if (c->opts.journal_flush_disabled)
2788                 return 0;
2789
2790         ret = bch2_inode_find_by_inum(c, inode_inum(inode), &u);
2791         if (ret)
2792                 return ret;
2793
2794         return bch2_journal_flush_seq(&c->journal, u.bi_journal_seq) ?:
2795                 bch2_inode_flush_nocow_writes(c, inode);
2796 }
2797
2798 int bch2_fsync(struct file *file, loff_t start, loff_t end, int datasync)
2799 {
2800         struct bch_inode_info *inode = file_bch_inode(file);
2801         struct bch_fs *c = inode->v.i_sb->s_fs_info;
2802         int ret, ret2, ret3;
2803
2804         ret = file_write_and_wait_range(file, start, end);
2805         ret2 = sync_inode_metadata(&inode->v, 1);
2806         ret3 = bch2_flush_inode(c, inode);
2807
2808         return bch2_err_class(ret ?: ret2 ?: ret3);
2809 }
2810
2811 /* truncate: */
2812
2813 static inline int range_has_data(struct bch_fs *c, u32 subvol,
2814                                  struct bpos start,
2815                                  struct bpos end)
2816 {
2817         struct btree_trans trans;
2818         struct btree_iter iter;
2819         struct bkey_s_c k;
2820         int ret = 0;
2821
2822         bch2_trans_init(&trans, c, 0, 0);
2823 retry:
2824         bch2_trans_begin(&trans);
2825
2826         ret = bch2_subvolume_get_snapshot(&trans, subvol, &start.snapshot);
2827         if (ret)
2828                 goto err;
2829
2830         for_each_btree_key_upto_norestart(&trans, iter, BTREE_ID_extents, start, end, 0, k, ret)
2831                 if (bkey_extent_is_data(k.k) && !bkey_extent_is_unwritten(k)) {
2832                         ret = 1;
2833                         break;
2834                 }
2835         start = iter.pos;
2836         bch2_trans_iter_exit(&trans, &iter);
2837 err:
2838         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
2839                 goto retry;
2840
2841         bch2_trans_exit(&trans);
2842         return ret;
2843 }
2844
2845 static int __bch2_truncate_folio(struct bch_inode_info *inode,
2846                                  pgoff_t index, loff_t start, loff_t end)
2847 {
2848         struct bch_fs *c = inode->v.i_sb->s_fs_info;
2849         struct address_space *mapping = inode->v.i_mapping;
2850         struct bch_folio *s;
2851         unsigned start_offset = start & (PAGE_SIZE - 1);
2852         unsigned end_offset = ((end - 1) & (PAGE_SIZE - 1)) + 1;
2853         unsigned i;
2854         struct folio *folio;
2855         s64 i_sectors_delta = 0;
2856         int ret = 0;
2857         u64 end_pos;
2858
2859         folio = filemap_lock_folio(mapping, index);
2860         if (IS_ERR_OR_NULL(folio)) {
2861                 /*
2862                  * XXX: we're doing two index lookups when we end up reading the
2863                  * folio
2864                  */
2865                 ret = range_has_data(c, inode->ei_subvol,
2866                                 POS(inode->v.i_ino, (index << PAGE_SECTORS_SHIFT)),
2867                                 POS(inode->v.i_ino, (index << PAGE_SECTORS_SHIFT) + PAGE_SECTORS));
2868                 if (ret <= 0)
2869                         return ret;
2870
2871                 folio = __filemap_get_folio(mapping, index,
2872                                             FGP_LOCK|FGP_CREAT, GFP_KERNEL);
2873                 if (unlikely(IS_ERR_OR_NULL(folio))) {
2874                         ret = -ENOMEM;
2875                         goto out;
2876                 }
2877         }
2878
2879         BUG_ON(start    >= folio_end_pos(folio));
2880         BUG_ON(end      <= folio_pos(folio));
2881
2882         start_offset    = max(start, folio_pos(folio)) - folio_pos(folio);
2883         end_offset      = min_t(u64, end, folio_end_pos(folio)) - folio_pos(folio);
2884
2885         /* Folio boundary? Nothing to do */
2886         if (start_offset == 0 &&
2887             end_offset == folio_size(folio)) {
2888                 ret = 0;
2889                 goto unlock;
2890         }
2891
2892         s = bch2_folio_create(folio, 0);
2893         if (!s) {
2894                 ret = -ENOMEM;
2895                 goto unlock;
2896         }
2897
2898         if (!folio_test_uptodate(folio)) {
2899                 ret = bch2_read_single_folio(folio, mapping);
2900                 if (ret)
2901                         goto unlock;
2902         }
2903
2904         ret = bch2_folio_set(c, inode_inum(inode), &folio, 1);
2905         if (ret)
2906                 goto unlock;
2907
2908         for (i = round_up(start_offset, block_bytes(c)) >> 9;
2909              i < round_down(end_offset, block_bytes(c)) >> 9;
2910              i++) {
2911                 s->s[i].nr_replicas     = 0;
2912
2913                 i_sectors_delta -= s->s[i].state == SECTOR_dirty;
2914                 folio_sector_set(folio, s, i, SECTOR_unallocated);
2915         }
2916
2917         i_sectors_acct(c, inode, NULL, i_sectors_delta);
2918
2919         /*
2920          * Caller needs to know whether this folio will be written out by
2921          * writeback - doing an i_size update if necessary - or whether it will
2922          * be responsible for the i_size update.
2923          *
2924          * Note that we shouldn't ever see a folio beyond EOF, but check and
2925          * warn if so. This has been observed by failure to clean up folios
2926          * after a short write and there's still a chance reclaim will fix
2927          * things up.
2928          */
2929         WARN_ON_ONCE(folio_pos(folio) >= inode->v.i_size);
2930         end_pos = folio_end_pos(folio);
2931         if (inode->v.i_size > folio_pos(folio))
2932                 end_pos = min_t(u64, inode->v.i_size, end_pos);
2933         ret = s->s[folio_pos_to_s(folio, end_pos - 1)].state >= SECTOR_dirty;
2934
2935         folio_zero_segment(folio, start_offset, end_offset);
2936
2937         /*
2938          * Bit of a hack - we don't want truncate to fail due to -ENOSPC.
2939          *
2940          * XXX: because we aren't currently tracking whether the folio has actual
2941          * data in it (vs. just 0s, or only partially written) this wrong. ick.
2942          */
2943         BUG_ON(bch2_get_folio_disk_reservation(c, inode, folio, false));
2944
2945         /*
2946          * This removes any writeable userspace mappings; we need to force
2947          * .page_mkwrite to be called again before any mmapped writes, to
2948          * redirty the full page:
2949          */
2950         folio_mkclean(folio);
2951         filemap_dirty_folio(mapping, folio);
2952 unlock:
2953         folio_unlock(folio);
2954         folio_put(folio);
2955 out:
2956         return ret;
2957 }
2958
2959 static int bch2_truncate_folio(struct bch_inode_info *inode, loff_t from)
2960 {
2961         return __bch2_truncate_folio(inode, from >> PAGE_SHIFT,
2962                                      from, ANYSINT_MAX(loff_t));
2963 }
2964
2965 static int bch2_truncate_folios(struct bch_inode_info *inode,
2966                                 loff_t start, loff_t end)
2967 {
2968         int ret = __bch2_truncate_folio(inode, start >> PAGE_SHIFT,
2969                                         start, end);
2970
2971         if (ret >= 0 &&
2972             start >> PAGE_SHIFT != end >> PAGE_SHIFT)
2973                 ret = __bch2_truncate_folio(inode,
2974                                         (end - 1) >> PAGE_SHIFT,
2975                                         start, end);
2976         return ret;
2977 }
2978
2979 static int bch2_extend(struct mnt_idmap *idmap,
2980                        struct bch_inode_info *inode,
2981                        struct bch_inode_unpacked *inode_u,
2982                        struct iattr *iattr)
2983 {
2984         struct address_space *mapping = inode->v.i_mapping;
2985         int ret;
2986
2987         /*
2988          * sync appends:
2989          *
2990          * this has to be done _before_ extending i_size:
2991          */
2992         ret = filemap_write_and_wait_range(mapping, inode_u->bi_size, S64_MAX);
2993         if (ret)
2994                 return ret;
2995
2996         truncate_setsize(&inode->v, iattr->ia_size);
2997
2998         return bch2_setattr_nonsize(idmap, inode, iattr);
2999 }
3000
3001 static int bch2_truncate_finish_fn(struct bch_inode_info *inode,
3002                                    struct bch_inode_unpacked *bi,
3003                                    void *p)
3004 {
3005         bi->bi_flags &= ~BCH_INODE_I_SIZE_DIRTY;
3006         return 0;
3007 }
3008
3009 static int bch2_truncate_start_fn(struct bch_inode_info *inode,
3010                                   struct bch_inode_unpacked *bi, void *p)
3011 {
3012         u64 *new_i_size = p;
3013
3014         bi->bi_flags |= BCH_INODE_I_SIZE_DIRTY;
3015         bi->bi_size = *new_i_size;
3016         return 0;
3017 }
3018
3019 int bch2_truncate(struct mnt_idmap *idmap,
3020                   struct bch_inode_info *inode, struct iattr *iattr)
3021 {
3022         struct bch_fs *c = inode->v.i_sb->s_fs_info;
3023         struct address_space *mapping = inode->v.i_mapping;
3024         struct bch_inode_unpacked inode_u;
3025         u64 new_i_size = iattr->ia_size;
3026         s64 i_sectors_delta = 0;
3027         int ret = 0;
3028
3029         /*
3030          * If the truncate call with change the size of the file, the
3031          * cmtimes should be updated. If the size will not change, we
3032          * do not need to update the cmtimes.
3033          */
3034         if (iattr->ia_size != inode->v.i_size) {
3035                 if (!(iattr->ia_valid & ATTR_MTIME))
3036                         ktime_get_coarse_real_ts64(&iattr->ia_mtime);
3037                 if (!(iattr->ia_valid & ATTR_CTIME))
3038                         ktime_get_coarse_real_ts64(&iattr->ia_ctime);
3039                 iattr->ia_valid |= ATTR_MTIME|ATTR_CTIME;
3040         }
3041
3042         inode_dio_wait(&inode->v);
3043         bch2_pagecache_block_get(inode);
3044
3045         ret = bch2_inode_find_by_inum(c, inode_inum(inode), &inode_u);
3046         if (ret)
3047                 goto err;
3048
3049         /*
3050          * check this before next assertion; on filesystem error our normal
3051          * invariants are a bit broken (truncate has to truncate the page cache
3052          * before the inode).
3053          */
3054         ret = bch2_journal_error(&c->journal);
3055         if (ret)
3056                 goto err;
3057
3058         WARN_ONCE(!test_bit(EI_INODE_ERROR, &inode->ei_flags) &&
3059                   inode->v.i_size < inode_u.bi_size,
3060                   "truncate spotted in mem i_size < btree i_size: %llu < %llu\n",
3061                   (u64) inode->v.i_size, inode_u.bi_size);
3062
3063         if (iattr->ia_size > inode->v.i_size) {
3064                 ret = bch2_extend(idmap, inode, &inode_u, iattr);
3065                 goto err;
3066         }
3067
3068         iattr->ia_valid &= ~ATTR_SIZE;
3069
3070         ret = bch2_truncate_folio(inode, iattr->ia_size);
3071         if (unlikely(ret < 0))
3072                 goto err;
3073
3074         /*
3075          * When extending, we're going to write the new i_size to disk
3076          * immediately so we need to flush anything above the current on disk
3077          * i_size first:
3078          *
3079          * Also, when extending we need to flush the page that i_size currently
3080          * straddles - if it's mapped to userspace, we need to ensure that
3081          * userspace has to redirty it and call .mkwrite -> set_page_dirty
3082          * again to allocate the part of the page that was extended.
3083          */
3084         if (iattr->ia_size > inode_u.bi_size)
3085                 ret = filemap_write_and_wait_range(mapping,
3086                                 inode_u.bi_size,
3087                                 iattr->ia_size - 1);
3088         else if (iattr->ia_size & (PAGE_SIZE - 1))
3089                 ret = filemap_write_and_wait_range(mapping,
3090                                 round_down(iattr->ia_size, PAGE_SIZE),
3091                                 iattr->ia_size - 1);
3092         if (ret)
3093                 goto err;
3094
3095         mutex_lock(&inode->ei_update_lock);
3096         ret = bch2_write_inode(c, inode, bch2_truncate_start_fn,
3097                                &new_i_size, 0);
3098         mutex_unlock(&inode->ei_update_lock);
3099
3100         if (unlikely(ret))
3101                 goto err;
3102
3103         truncate_setsize(&inode->v, iattr->ia_size);
3104
3105         ret = bch2_fpunch(c, inode_inum(inode),
3106                         round_up(iattr->ia_size, block_bytes(c)) >> 9,
3107                         U64_MAX, &i_sectors_delta);
3108         i_sectors_acct(c, inode, NULL, i_sectors_delta);
3109
3110         bch2_fs_inconsistent_on(!inode->v.i_size && inode->v.i_blocks &&
3111                                 !bch2_journal_error(&c->journal), c,
3112                                 "inode %lu truncated to 0 but i_blocks %llu (ondisk %lli)",
3113                                 inode->v.i_ino, (u64) inode->v.i_blocks,
3114                                 inode->ei_inode.bi_sectors);
3115         if (unlikely(ret))
3116                 goto err;
3117
3118         mutex_lock(&inode->ei_update_lock);
3119         ret = bch2_write_inode(c, inode, bch2_truncate_finish_fn, NULL, 0);
3120         mutex_unlock(&inode->ei_update_lock);
3121
3122         ret = bch2_setattr_nonsize(idmap, inode, iattr);
3123 err:
3124         bch2_pagecache_block_put(inode);
3125         return bch2_err_class(ret);
3126 }
3127
3128 /* fallocate: */
3129
3130 static int inode_update_times_fn(struct bch_inode_info *inode,
3131                                  struct bch_inode_unpacked *bi, void *p)
3132 {
3133         struct bch_fs *c = inode->v.i_sb->s_fs_info;
3134
3135         bi->bi_mtime = bi->bi_ctime = bch2_current_time(c);
3136         return 0;
3137 }
3138
3139 static long bchfs_fpunch(struct bch_inode_info *inode, loff_t offset, loff_t len)
3140 {
3141         struct bch_fs *c = inode->v.i_sb->s_fs_info;
3142         u64 end         = offset + len;
3143         u64 block_start = round_up(offset, block_bytes(c));
3144         u64 block_end   = round_down(end, block_bytes(c));
3145         bool truncated_last_page;
3146         int ret = 0;
3147
3148         ret = bch2_truncate_folios(inode, offset, end);
3149         if (unlikely(ret < 0))
3150                 goto err;
3151
3152         truncated_last_page = ret;
3153
3154         truncate_pagecache_range(&inode->v, offset, end - 1);
3155
3156         if (block_start < block_end) {
3157                 s64 i_sectors_delta = 0;
3158
3159                 ret = bch2_fpunch(c, inode_inum(inode),
3160                                   block_start >> 9, block_end >> 9,
3161                                   &i_sectors_delta);
3162                 i_sectors_acct(c, inode, NULL, i_sectors_delta);
3163         }
3164
3165         mutex_lock(&inode->ei_update_lock);
3166         if (end >= inode->v.i_size && !truncated_last_page) {
3167                 ret = bch2_write_inode_size(c, inode, inode->v.i_size,
3168                                             ATTR_MTIME|ATTR_CTIME);
3169         } else {
3170                 ret = bch2_write_inode(c, inode, inode_update_times_fn, NULL,
3171                                        ATTR_MTIME|ATTR_CTIME);
3172         }
3173         mutex_unlock(&inode->ei_update_lock);
3174 err:
3175         return ret;
3176 }
3177
3178 static long bchfs_fcollapse_finsert(struct bch_inode_info *inode,
3179                                    loff_t offset, loff_t len,
3180                                    bool insert)
3181 {
3182         struct bch_fs *c = inode->v.i_sb->s_fs_info;
3183         struct address_space *mapping = inode->v.i_mapping;
3184         struct bkey_buf copy;
3185         struct btree_trans trans;
3186         struct btree_iter src, dst, del;
3187         loff_t shift, new_size;
3188         u64 src_start;
3189         int ret = 0;
3190
3191         if ((offset | len) & (block_bytes(c) - 1))
3192                 return -EINVAL;
3193
3194         if (insert) {
3195                 if (inode->v.i_sb->s_maxbytes - inode->v.i_size < len)
3196                         return -EFBIG;
3197
3198                 if (offset >= inode->v.i_size)
3199                         return -EINVAL;
3200
3201                 src_start       = U64_MAX;
3202                 shift           = len;
3203         } else {
3204                 if (offset + len >= inode->v.i_size)
3205                         return -EINVAL;
3206
3207                 src_start       = offset + len;
3208                 shift           = -len;
3209         }
3210
3211         new_size = inode->v.i_size + shift;
3212
3213         ret = write_invalidate_inode_pages_range(mapping, offset, LLONG_MAX);
3214         if (ret)
3215                 return ret;
3216
3217         if (insert) {
3218                 i_size_write(&inode->v, new_size);
3219                 mutex_lock(&inode->ei_update_lock);
3220                 ret = bch2_write_inode_size(c, inode, new_size,
3221                                             ATTR_MTIME|ATTR_CTIME);
3222                 mutex_unlock(&inode->ei_update_lock);
3223         } else {
3224                 s64 i_sectors_delta = 0;
3225
3226                 ret = bch2_fpunch(c, inode_inum(inode),
3227                                   offset >> 9, (offset + len) >> 9,
3228                                   &i_sectors_delta);
3229                 i_sectors_acct(c, inode, NULL, i_sectors_delta);
3230
3231                 if (ret)
3232                         return ret;
3233         }
3234
3235         bch2_bkey_buf_init(&copy);
3236         bch2_trans_init(&trans, c, BTREE_ITER_MAX, 1024);
3237         bch2_trans_iter_init(&trans, &src, BTREE_ID_extents,
3238                         POS(inode->v.i_ino, src_start >> 9),
3239                         BTREE_ITER_INTENT);
3240         bch2_trans_copy_iter(&dst, &src);
3241         bch2_trans_copy_iter(&del, &src);
3242
3243         while (ret == 0 ||
3244                bch2_err_matches(ret, BCH_ERR_transaction_restart)) {
3245                 struct disk_reservation disk_res =
3246                         bch2_disk_reservation_init(c, 0);
3247                 struct bkey_i delete;
3248                 struct bkey_s_c k;
3249                 struct bpos next_pos;
3250                 struct bpos move_pos = POS(inode->v.i_ino, offset >> 9);
3251                 struct bpos atomic_end;
3252                 unsigned trigger_flags = 0;
3253                 u32 snapshot;
3254
3255                 bch2_trans_begin(&trans);
3256
3257                 ret = bch2_subvolume_get_snapshot(&trans,
3258                                         inode->ei_subvol, &snapshot);
3259                 if (ret)
3260                         continue;
3261
3262                 bch2_btree_iter_set_snapshot(&src, snapshot);
3263                 bch2_btree_iter_set_snapshot(&dst, snapshot);
3264                 bch2_btree_iter_set_snapshot(&del, snapshot);
3265
3266                 bch2_trans_begin(&trans);
3267
3268                 k = insert
3269                         ? bch2_btree_iter_peek_prev(&src)
3270                         : bch2_btree_iter_peek_upto(&src, POS(inode->v.i_ino, U64_MAX));
3271                 if ((ret = bkey_err(k)))
3272                         continue;
3273
3274                 if (!k.k || k.k->p.inode != inode->v.i_ino)
3275                         break;
3276
3277                 if (insert &&
3278                     bkey_le(k.k->p, POS(inode->v.i_ino, offset >> 9)))
3279                         break;
3280 reassemble:
3281                 bch2_bkey_buf_reassemble(&copy, c, k);
3282
3283                 if (insert &&
3284                     bkey_lt(bkey_start_pos(k.k), move_pos))
3285                         bch2_cut_front(move_pos, copy.k);
3286
3287                 copy.k->k.p.offset += shift >> 9;
3288                 bch2_btree_iter_set_pos(&dst, bkey_start_pos(&copy.k->k));
3289
3290                 ret = bch2_extent_atomic_end(&trans, &dst, copy.k, &atomic_end);
3291                 if (ret)
3292                         continue;
3293
3294                 if (!bkey_eq(atomic_end, copy.k->k.p)) {
3295                         if (insert) {
3296                                 move_pos = atomic_end;
3297                                 move_pos.offset -= shift >> 9;
3298                                 goto reassemble;
3299                         } else {
3300                                 bch2_cut_back(atomic_end, copy.k);
3301                         }
3302                 }
3303
3304                 bkey_init(&delete.k);
3305                 delete.k.p = copy.k->k.p;
3306                 delete.k.size = copy.k->k.size;
3307                 delete.k.p.offset -= shift >> 9;
3308                 bch2_btree_iter_set_pos(&del, bkey_start_pos(&delete.k));
3309
3310                 next_pos = insert ? bkey_start_pos(&delete.k) : delete.k.p;
3311
3312                 if (copy.k->k.size != k.k->size) {
3313                         /* We might end up splitting compressed extents: */
3314                         unsigned nr_ptrs =
3315                                 bch2_bkey_nr_ptrs_allocated(bkey_i_to_s_c(copy.k));
3316
3317                         ret = bch2_disk_reservation_get(c, &disk_res,
3318                                         copy.k->k.size, nr_ptrs,
3319                                         BCH_DISK_RESERVATION_NOFAIL);
3320                         BUG_ON(ret);
3321                 }
3322
3323                 ret =   bch2_btree_iter_traverse(&del) ?:
3324                         bch2_trans_update(&trans, &del, &delete, trigger_flags) ?:
3325                         bch2_trans_update(&trans, &dst, copy.k, trigger_flags) ?:
3326                         bch2_trans_commit(&trans, &disk_res, NULL,
3327                                           BTREE_INSERT_NOFAIL);
3328                 bch2_disk_reservation_put(c, &disk_res);
3329
3330                 if (!ret)
3331                         bch2_btree_iter_set_pos(&src, next_pos);
3332         }
3333         bch2_trans_iter_exit(&trans, &del);
3334         bch2_trans_iter_exit(&trans, &dst);
3335         bch2_trans_iter_exit(&trans, &src);
3336         bch2_trans_exit(&trans);
3337         bch2_bkey_buf_exit(&copy, c);
3338
3339         if (ret)
3340                 return ret;
3341
3342         mutex_lock(&inode->ei_update_lock);
3343         if (!insert) {
3344                 i_size_write(&inode->v, new_size);
3345                 ret = bch2_write_inode_size(c, inode, new_size,
3346                                             ATTR_MTIME|ATTR_CTIME);
3347         } else {
3348                 /* We need an inode update to update bi_journal_seq for fsync: */
3349                 ret = bch2_write_inode(c, inode, inode_update_times_fn, NULL,
3350                                        ATTR_MTIME|ATTR_CTIME);
3351         }
3352         mutex_unlock(&inode->ei_update_lock);
3353         return ret;
3354 }
3355
3356 static int __bchfs_fallocate(struct bch_inode_info *inode, int mode,
3357                              u64 start_sector, u64 end_sector)
3358 {
3359         struct bch_fs *c = inode->v.i_sb->s_fs_info;
3360         struct btree_trans trans;
3361         struct btree_iter iter;
3362         struct bpos end_pos = POS(inode->v.i_ino, end_sector);
3363         struct bch_io_opts opts;
3364         int ret = 0;
3365
3366         bch2_inode_opts_get(&opts, c, &inode->ei_inode);
3367         bch2_trans_init(&trans, c, BTREE_ITER_MAX, 512);
3368
3369         bch2_trans_iter_init(&trans, &iter, BTREE_ID_extents,
3370                         POS(inode->v.i_ino, start_sector),
3371                         BTREE_ITER_SLOTS|BTREE_ITER_INTENT);
3372
3373         while (!ret && bkey_lt(iter.pos, end_pos)) {
3374                 s64 i_sectors_delta = 0;
3375                 struct quota_res quota_res = { 0 };
3376                 struct bkey_s_c k;
3377                 unsigned sectors;
3378                 bool is_allocation;
3379                 u64 hole_start, hole_end;
3380                 u32 snapshot;
3381
3382                 bch2_trans_begin(&trans);
3383
3384                 ret = bch2_subvolume_get_snapshot(&trans,
3385                                         inode->ei_subvol, &snapshot);
3386                 if (ret)
3387                         goto bkey_err;
3388
3389                 bch2_btree_iter_set_snapshot(&iter, snapshot);
3390
3391                 k = bch2_btree_iter_peek_slot(&iter);
3392                 if ((ret = bkey_err(k)))
3393                         goto bkey_err;
3394
3395                 hole_start      = iter.pos.offset;
3396                 hole_end        = bpos_min(k.k->p, end_pos).offset;
3397                 is_allocation   = bkey_extent_is_allocation(k.k);
3398
3399                 /* already reserved */
3400                 if (bkey_extent_is_reservation(k) &&
3401                     bch2_bkey_nr_ptrs_fully_allocated(k) >= opts.data_replicas) {
3402                         bch2_btree_iter_advance(&iter);
3403                         continue;
3404                 }
3405
3406                 if (bkey_extent_is_data(k.k) &&
3407                     !(mode & FALLOC_FL_ZERO_RANGE)) {
3408                         bch2_btree_iter_advance(&iter);
3409                         continue;
3410                 }
3411
3412                 if (!(mode & FALLOC_FL_ZERO_RANGE)) {
3413                         if (bch2_clamp_data_hole(&inode->v,
3414                                                  &hole_start,
3415                                                  &hole_end,
3416                                                  opts.data_replicas, true))
3417                                 ret = drop_locks_do(&trans,
3418                                         (bch2_clamp_data_hole(&inode->v,
3419                                                               &hole_start,
3420                                                               &hole_end,
3421                                                               opts.data_replicas, false), 0));
3422                         bch2_btree_iter_set_pos(&iter, POS(iter.pos.inode, hole_start));
3423
3424                         if (ret)
3425                                 goto bkey_err;
3426
3427                         if (hole_start == hole_end)
3428                                 continue;
3429                 }
3430
3431                 sectors = hole_end - hole_start;
3432
3433                 if (!is_allocation) {
3434                         ret = bch2_quota_reservation_add(c, inode,
3435                                         &quota_res, sectors, true);
3436                         if (unlikely(ret))
3437                                 goto bkey_err;
3438                 }
3439
3440                 ret = bch2_extent_fallocate(&trans, inode_inum(inode), &iter,
3441                                             sectors, opts, &i_sectors_delta,
3442                                             writepoint_hashed((unsigned long) current));
3443                 if (ret)
3444                         goto bkey_err;
3445
3446                 i_sectors_acct(c, inode, &quota_res, i_sectors_delta);
3447
3448                 drop_locks_do(&trans,
3449                         (mark_pagecache_reserved(inode, hole_start, iter.pos.offset), 0));
3450 bkey_err:
3451                 bch2_quota_reservation_put(c, inode, &quota_res);
3452                 if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
3453                         ret = 0;
3454         }
3455
3456         if (bch2_err_matches(ret, ENOSPC) && (mode & FALLOC_FL_ZERO_RANGE)) {
3457                 struct quota_res quota_res = { 0 };
3458                 s64 i_sectors_delta = 0;
3459
3460                 bch2_fpunch_at(&trans, &iter, inode_inum(inode),
3461                                end_sector, &i_sectors_delta);
3462                 i_sectors_acct(c, inode, &quota_res, i_sectors_delta);
3463                 bch2_quota_reservation_put(c, inode, &quota_res);
3464         }
3465
3466         bch2_trans_iter_exit(&trans, &iter);
3467         bch2_trans_exit(&trans);
3468         return ret;
3469 }
3470
3471 static long bchfs_fallocate(struct bch_inode_info *inode, int mode,
3472                             loff_t offset, loff_t len)
3473 {
3474         struct bch_fs *c = inode->v.i_sb->s_fs_info;
3475         u64 end         = offset + len;
3476         u64 block_start = round_down(offset,    block_bytes(c));
3477         u64 block_end   = round_up(end,         block_bytes(c));
3478         bool truncated_last_page = false;
3479         int ret, ret2 = 0;
3480
3481         if (!(mode & FALLOC_FL_KEEP_SIZE) && end > inode->v.i_size) {
3482                 ret = inode_newsize_ok(&inode->v, end);
3483                 if (ret)
3484                         return ret;
3485         }
3486
3487         if (mode & FALLOC_FL_ZERO_RANGE) {
3488                 ret = bch2_truncate_folios(inode, offset, end);
3489                 if (unlikely(ret < 0))
3490                         return ret;
3491
3492                 truncated_last_page = ret;
3493
3494                 truncate_pagecache_range(&inode->v, offset, end - 1);
3495
3496                 block_start     = round_up(offset,      block_bytes(c));
3497                 block_end       = round_down(end,       block_bytes(c));
3498         }
3499
3500         ret = __bchfs_fallocate(inode, mode, block_start >> 9, block_end >> 9);
3501
3502         /*
3503          * On -ENOSPC in ZERO_RANGE mode, we still want to do the inode update,
3504          * so that the VFS cache i_size is consistent with the btree i_size:
3505          */
3506         if (ret &&
3507             !(bch2_err_matches(ret, ENOSPC) && (mode & FALLOC_FL_ZERO_RANGE)))
3508                 return ret;
3509
3510         if (mode & FALLOC_FL_KEEP_SIZE && end > inode->v.i_size)
3511                 end = inode->v.i_size;
3512
3513         if (end >= inode->v.i_size &&
3514             (((mode & FALLOC_FL_ZERO_RANGE) && !truncated_last_page) ||
3515              !(mode & FALLOC_FL_KEEP_SIZE))) {
3516                 spin_lock(&inode->v.i_lock);
3517                 i_size_write(&inode->v, end);
3518                 spin_unlock(&inode->v.i_lock);
3519
3520                 mutex_lock(&inode->ei_update_lock);
3521                 ret2 = bch2_write_inode_size(c, inode, end, 0);
3522                 mutex_unlock(&inode->ei_update_lock);
3523         }
3524
3525         return ret ?: ret2;
3526 }
3527
3528 long bch2_fallocate_dispatch(struct file *file, int mode,
3529                              loff_t offset, loff_t len)
3530 {
3531         struct bch_inode_info *inode = file_bch_inode(file);
3532         struct bch_fs *c = inode->v.i_sb->s_fs_info;
3533         long ret;
3534
3535         if (!bch2_write_ref_tryget(c, BCH_WRITE_REF_fallocate))
3536                 return -EROFS;
3537
3538         inode_lock(&inode->v);
3539         inode_dio_wait(&inode->v);
3540         bch2_pagecache_block_get(inode);
3541
3542         ret = file_modified(file);
3543         if (ret)
3544                 goto err;
3545
3546         if (!(mode & ~(FALLOC_FL_KEEP_SIZE|FALLOC_FL_ZERO_RANGE)))
3547                 ret = bchfs_fallocate(inode, mode, offset, len);
3548         else if (mode == (FALLOC_FL_PUNCH_HOLE|FALLOC_FL_KEEP_SIZE))
3549                 ret = bchfs_fpunch(inode, offset, len);
3550         else if (mode == FALLOC_FL_INSERT_RANGE)
3551                 ret = bchfs_fcollapse_finsert(inode, offset, len, true);
3552         else if (mode == FALLOC_FL_COLLAPSE_RANGE)
3553                 ret = bchfs_fcollapse_finsert(inode, offset, len, false);
3554         else
3555                 ret = -EOPNOTSUPP;
3556 err:
3557         bch2_pagecache_block_put(inode);
3558         inode_unlock(&inode->v);
3559         bch2_write_ref_put(c, BCH_WRITE_REF_fallocate);
3560
3561         return bch2_err_class(ret);
3562 }
3563
3564 /*
3565  * Take a quota reservation for unallocated blocks in a given file range
3566  * Does not check pagecache
3567  */
3568 static int quota_reserve_range(struct bch_inode_info *inode,
3569                                struct quota_res *res,
3570                                u64 start, u64 end)
3571 {
3572         struct bch_fs *c = inode->v.i_sb->s_fs_info;
3573         struct btree_trans trans;
3574         struct btree_iter iter;
3575         struct bkey_s_c k;
3576         u32 snapshot;
3577         u64 sectors = end - start;
3578         u64 pos = start;
3579         int ret;
3580
3581         bch2_trans_init(&trans, c, 0, 0);
3582 retry:
3583         bch2_trans_begin(&trans);
3584
3585         ret = bch2_subvolume_get_snapshot(&trans, inode->ei_subvol, &snapshot);
3586         if (ret)
3587                 goto err;
3588
3589         bch2_trans_iter_init(&trans, &iter, BTREE_ID_extents,
3590                              SPOS(inode->v.i_ino, pos, snapshot), 0);
3591
3592         while (!(ret = btree_trans_too_many_iters(&trans)) &&
3593                (k = bch2_btree_iter_peek_upto(&iter, POS(inode->v.i_ino, end - 1))).k &&
3594                !(ret = bkey_err(k))) {
3595                 if (bkey_extent_is_allocation(k.k)) {
3596                         u64 s = min(end, k.k->p.offset) -
3597                                 max(start, bkey_start_offset(k.k));
3598                         BUG_ON(s > sectors);
3599                         sectors -= s;
3600                 }
3601                 bch2_btree_iter_advance(&iter);
3602         }
3603         pos = iter.pos.offset;
3604         bch2_trans_iter_exit(&trans, &iter);
3605 err:
3606         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
3607                 goto retry;
3608
3609         bch2_trans_exit(&trans);
3610
3611         if (ret)
3612                 return ret;
3613
3614         return bch2_quota_reservation_add(c, inode, res, sectors, true);
3615 }
3616
3617 loff_t bch2_remap_file_range(struct file *file_src, loff_t pos_src,
3618                              struct file *file_dst, loff_t pos_dst,
3619                              loff_t len, unsigned remap_flags)
3620 {
3621         struct bch_inode_info *src = file_bch_inode(file_src);
3622         struct bch_inode_info *dst = file_bch_inode(file_dst);
3623         struct bch_fs *c = src->v.i_sb->s_fs_info;
3624         struct quota_res quota_res = { 0 };
3625         s64 i_sectors_delta = 0;
3626         u64 aligned_len;
3627         loff_t ret = 0;
3628
3629         if (remap_flags & ~(REMAP_FILE_DEDUP|REMAP_FILE_ADVISORY))
3630                 return -EINVAL;
3631
3632         if (remap_flags & REMAP_FILE_DEDUP)
3633                 return -EOPNOTSUPP;
3634
3635         if ((pos_src & (block_bytes(c) - 1)) ||
3636             (pos_dst & (block_bytes(c) - 1)))
3637                 return -EINVAL;
3638
3639         if (src == dst &&
3640             abs(pos_src - pos_dst) < len)
3641                 return -EINVAL;
3642
3643         bch2_lock_inodes(INODE_LOCK|INODE_PAGECACHE_BLOCK, src, dst);
3644
3645         inode_dio_wait(&src->v);
3646         inode_dio_wait(&dst->v);
3647
3648         ret = generic_remap_file_range_prep(file_src, pos_src,
3649                                             file_dst, pos_dst,
3650                                             &len, remap_flags);
3651         if (ret < 0 || len == 0)
3652                 goto err;
3653
3654         aligned_len = round_up((u64) len, block_bytes(c));
3655
3656         ret = write_invalidate_inode_pages_range(dst->v.i_mapping,
3657                                 pos_dst, pos_dst + len - 1);
3658         if (ret)
3659                 goto err;
3660
3661         ret = quota_reserve_range(dst, &quota_res, pos_dst >> 9,
3662                                   (pos_dst + aligned_len) >> 9);
3663         if (ret)
3664                 goto err;
3665
3666         file_update_time(file_dst);
3667
3668         mark_pagecache_unallocated(src, pos_src >> 9,
3669                                    (pos_src + aligned_len) >> 9);
3670
3671         ret = bch2_remap_range(c,
3672                                inode_inum(dst), pos_dst >> 9,
3673                                inode_inum(src), pos_src >> 9,
3674                                aligned_len >> 9,
3675                                pos_dst + len, &i_sectors_delta);
3676         if (ret < 0)
3677                 goto err;
3678
3679         /*
3680          * due to alignment, we might have remapped slightly more than requsted
3681          */
3682         ret = min((u64) ret << 9, (u64) len);
3683
3684         i_sectors_acct(c, dst, &quota_res, i_sectors_delta);
3685
3686         spin_lock(&dst->v.i_lock);
3687         if (pos_dst + ret > dst->v.i_size)
3688                 i_size_write(&dst->v, pos_dst + ret);
3689         spin_unlock(&dst->v.i_lock);
3690
3691         if ((file_dst->f_flags & (__O_SYNC | O_DSYNC)) ||
3692             IS_SYNC(file_inode(file_dst)))
3693                 ret = bch2_flush_inode(c, dst);
3694 err:
3695         bch2_quota_reservation_put(c, dst, &quota_res);
3696         bch2_unlock_inodes(INODE_LOCK|INODE_PAGECACHE_BLOCK, src, dst);
3697
3698         return bch2_err_class(ret);
3699 }
3700
3701 /* fseek: */
3702
3703 static int folio_data_offset(struct folio *folio, loff_t pos,
3704                              unsigned min_replicas)
3705 {
3706         struct bch_folio *s = bch2_folio(folio);
3707         unsigned i, sectors = folio_sectors(folio);
3708
3709         if (s)
3710                 for (i = folio_pos_to_s(folio, pos); i < sectors; i++)
3711                         if (s->s[i].state >= SECTOR_dirty &&
3712                             s->s[i].nr_replicas + s->s[i].replicas_reserved >= min_replicas)
3713                                 return i << SECTOR_SHIFT;
3714
3715         return -1;
3716 }
3717
3718 static loff_t bch2_seek_pagecache_data(struct inode *vinode,
3719                                        loff_t start_offset,
3720                                        loff_t end_offset,
3721                                        unsigned min_replicas,
3722                                        bool nonblock)
3723 {
3724         struct folio_batch fbatch;
3725         pgoff_t start_index     = start_offset >> PAGE_SHIFT;
3726         pgoff_t end_index       = end_offset >> PAGE_SHIFT;
3727         pgoff_t index           = start_index;
3728         unsigned i;
3729         loff_t ret;
3730         int offset;
3731
3732         folio_batch_init(&fbatch);
3733
3734         while (filemap_get_folios(vinode->i_mapping,
3735                                   &index, end_index, &fbatch)) {
3736                 for (i = 0; i < folio_batch_count(&fbatch); i++) {
3737                         struct folio *folio = fbatch.folios[i];
3738
3739                         if (!nonblock) {
3740                                 folio_lock(folio);
3741                         } else if (!folio_trylock(folio)) {
3742                                 folio_batch_release(&fbatch);
3743                                 return -EAGAIN;
3744                         }
3745
3746                         offset = folio_data_offset(folio,
3747                                         max(folio_pos(folio), start_offset),
3748                                         min_replicas);
3749                         if (offset >= 0) {
3750                                 ret = clamp(folio_pos(folio) + offset,
3751                                             start_offset, end_offset);
3752                                 folio_unlock(folio);
3753                                 folio_batch_release(&fbatch);
3754                                 return ret;
3755                         }
3756                         folio_unlock(folio);
3757                 }
3758                 folio_batch_release(&fbatch);
3759                 cond_resched();
3760         }
3761
3762         return end_offset;
3763 }
3764
3765 static loff_t bch2_seek_data(struct file *file, u64 offset)
3766 {
3767         struct bch_inode_info *inode = file_bch_inode(file);
3768         struct bch_fs *c = inode->v.i_sb->s_fs_info;
3769         struct btree_trans trans;
3770         struct btree_iter iter;
3771         struct bkey_s_c k;
3772         subvol_inum inum = inode_inum(inode);
3773         u64 isize, next_data = MAX_LFS_FILESIZE;
3774         u32 snapshot;
3775         int ret;
3776
3777         isize = i_size_read(&inode->v);
3778         if (offset >= isize)
3779                 return -ENXIO;
3780
3781         bch2_trans_init(&trans, c, 0, 0);
3782 retry:
3783         bch2_trans_begin(&trans);
3784
3785         ret = bch2_subvolume_get_snapshot(&trans, inum.subvol, &snapshot);
3786         if (ret)
3787                 goto err;
3788
3789         for_each_btree_key_upto_norestart(&trans, iter, BTREE_ID_extents,
3790                            SPOS(inode->v.i_ino, offset >> 9, snapshot),
3791                            POS(inode->v.i_ino, U64_MAX),
3792                            0, k, ret) {
3793                 if (bkey_extent_is_data(k.k)) {
3794                         next_data = max(offset, bkey_start_offset(k.k) << 9);
3795                         break;
3796                 } else if (k.k->p.offset >> 9 > isize)
3797                         break;
3798         }
3799         bch2_trans_iter_exit(&trans, &iter);
3800 err:
3801         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
3802                 goto retry;
3803
3804         bch2_trans_exit(&trans);
3805         if (ret)
3806                 return ret;
3807
3808         if (next_data > offset)
3809                 next_data = bch2_seek_pagecache_data(&inode->v,
3810                                         offset, next_data, 0, false);
3811
3812         if (next_data >= isize)
3813                 return -ENXIO;
3814
3815         return vfs_setpos(file, next_data, MAX_LFS_FILESIZE);
3816 }
3817
3818 static int folio_hole_offset(struct address_space *mapping, loff_t *offset,
3819                               unsigned min_replicas, bool nonblock)
3820 {
3821         struct folio *folio;
3822         struct bch_folio *s;
3823         unsigned i, sectors;
3824         bool ret = true;
3825
3826         folio = __filemap_get_folio(mapping, *offset >> PAGE_SHIFT,
3827                                     !nonblock ? FGP_LOCK : 0, 0);
3828         if (IS_ERR_OR_NULL(folio))
3829                 return true;
3830
3831         if (nonblock && !folio_trylock(folio)) {
3832                 folio_put(folio);
3833                 return -EAGAIN;
3834         }
3835
3836         s = bch2_folio(folio);
3837         if (!s)
3838                 goto unlock;
3839
3840         sectors = folio_sectors(folio);
3841         for (i = folio_pos_to_s(folio, *offset); i < sectors; i++)
3842                 if (s->s[i].state < SECTOR_dirty ||
3843                     s->s[i].nr_replicas + s->s[i].replicas_reserved < min_replicas) {
3844                         *offset = max(*offset,
3845                                       folio_pos(folio) + (i << SECTOR_SHIFT));
3846                         goto unlock;
3847                 }
3848
3849         *offset = folio_end_pos(folio);
3850         ret = false;
3851 unlock:
3852         folio_unlock(folio);
3853         folio_put(folio);
3854         return ret;
3855 }
3856
3857 static loff_t bch2_seek_pagecache_hole(struct inode *vinode,
3858                                        loff_t start_offset,
3859                                        loff_t end_offset,
3860                                        unsigned min_replicas,
3861                                        bool nonblock)
3862 {
3863         struct address_space *mapping = vinode->i_mapping;
3864         loff_t offset = start_offset;
3865
3866         while (offset < end_offset &&
3867                !folio_hole_offset(mapping, &offset, min_replicas, nonblock))
3868                 ;
3869
3870         return min(offset, end_offset);
3871 }
3872
3873 static int bch2_clamp_data_hole(struct inode *inode,
3874                                 u64 *hole_start,
3875                                 u64 *hole_end,
3876                                 unsigned min_replicas,
3877                                 bool nonblock)
3878 {
3879         loff_t ret;
3880
3881         ret = bch2_seek_pagecache_hole(inode,
3882                 *hole_start << 9, *hole_end << 9, min_replicas, nonblock) >> 9;
3883         if (ret < 0)
3884                 return ret;
3885
3886         *hole_start = ret;
3887
3888         if (*hole_start == *hole_end)
3889                 return 0;
3890
3891         ret = bch2_seek_pagecache_data(inode,
3892                 *hole_start << 9, *hole_end << 9, min_replicas, nonblock) >> 9;
3893         if (ret < 0)
3894                 return ret;
3895
3896         *hole_end = ret;
3897         return 0;
3898 }
3899
3900 static loff_t bch2_seek_hole(struct file *file, u64 offset)
3901 {
3902         struct bch_inode_info *inode = file_bch_inode(file);
3903         struct bch_fs *c = inode->v.i_sb->s_fs_info;
3904         struct btree_trans trans;
3905         struct btree_iter iter;
3906         struct bkey_s_c k;
3907         subvol_inum inum = inode_inum(inode);
3908         u64 isize, next_hole = MAX_LFS_FILESIZE;
3909         u32 snapshot;
3910         int ret;
3911
3912         isize = i_size_read(&inode->v);
3913         if (offset >= isize)
3914                 return -ENXIO;
3915
3916         bch2_trans_init(&trans, c, 0, 0);
3917 retry:
3918         bch2_trans_begin(&trans);
3919
3920         ret = bch2_subvolume_get_snapshot(&trans, inum.subvol, &snapshot);
3921         if (ret)
3922                 goto err;
3923
3924         for_each_btree_key_norestart(&trans, iter, BTREE_ID_extents,
3925                            SPOS(inode->v.i_ino, offset >> 9, snapshot),
3926                            BTREE_ITER_SLOTS, k, ret) {
3927                 if (k.k->p.inode != inode->v.i_ino) {
3928                         next_hole = bch2_seek_pagecache_hole(&inode->v,
3929                                         offset, MAX_LFS_FILESIZE, 0, false);
3930                         break;
3931                 } else if (!bkey_extent_is_data(k.k)) {
3932                         next_hole = bch2_seek_pagecache_hole(&inode->v,
3933                                         max(offset, bkey_start_offset(k.k) << 9),
3934                                         k.k->p.offset << 9, 0, false);
3935
3936                         if (next_hole < k.k->p.offset << 9)
3937                                 break;
3938                 } else {
3939                         offset = max(offset, bkey_start_offset(k.k) << 9);
3940                 }
3941         }
3942         bch2_trans_iter_exit(&trans, &iter);
3943 err:
3944         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
3945                 goto retry;
3946
3947         bch2_trans_exit(&trans);
3948         if (ret)
3949                 return ret;
3950
3951         if (next_hole > isize)
3952                 next_hole = isize;
3953
3954         return vfs_setpos(file, next_hole, MAX_LFS_FILESIZE);
3955 }
3956
3957 loff_t bch2_llseek(struct file *file, loff_t offset, int whence)
3958 {
3959         loff_t ret;
3960
3961         switch (whence) {
3962         case SEEK_SET:
3963         case SEEK_CUR:
3964         case SEEK_END:
3965                 ret = generic_file_llseek(file, offset, whence);
3966                 break;
3967         case SEEK_DATA:
3968                 ret = bch2_seek_data(file, offset);
3969                 break;
3970         case SEEK_HOLE:
3971                 ret = bch2_seek_hole(file, offset);
3972                 break;
3973         default:
3974                 ret = -EINVAL;
3975                 break;
3976         }
3977
3978         return bch2_err_class(ret);
3979 }
3980
3981 void bch2_fs_fsio_exit(struct bch_fs *c)
3982 {
3983         bioset_exit(&c->nocow_flush_bioset);
3984         bioset_exit(&c->dio_write_bioset);
3985         bioset_exit(&c->dio_read_bioset);
3986         bioset_exit(&c->writepage_bioset);
3987 }
3988
3989 int bch2_fs_fsio_init(struct bch_fs *c)
3990 {
3991         if (bioset_init(&c->writepage_bioset,
3992                         4, offsetof(struct bch_writepage_io, op.wbio.bio),
3993                         BIOSET_NEED_BVECS))
3994                 return -BCH_ERR_ENOMEM_writepage_bioset_init;
3995
3996         if (bioset_init(&c->dio_read_bioset,
3997                         4, offsetof(struct dio_read, rbio.bio),
3998                         BIOSET_NEED_BVECS))
3999                 return -BCH_ERR_ENOMEM_dio_read_bioset_init;
4000
4001         if (bioset_init(&c->dio_write_bioset,
4002                         4, offsetof(struct dio_write, op.wbio.bio),
4003                         BIOSET_NEED_BVECS))
4004                 return -BCH_ERR_ENOMEM_dio_write_bioset_init;
4005
4006         if (bioset_init(&c->nocow_flush_bioset,
4007                         1, offsetof(struct nocow_flush, bio), 0))
4008                 return -BCH_ERR_ENOMEM_nocow_flush_bioset_init;
4009
4010         return 0;
4011 }
4012
4013 #endif /* NO_BCACHEFS_FS */