git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/fs-io.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #ifndef NO_BCACHEFS_FS
   3
   4 #include "bcachefs.h"
   5 #include "alloc_foreground.h"
   6 #include "bkey_buf.h"
   7 #include "btree_update.h"
   8 #include "buckets.h"
   9 #include "clock.h"
  10 #include "error.h"
  11 #include "extents.h"
  12 #include "extent_update.h"
  13 #include "fs.h"
  14 #include "fs-io.h"
  15 #include "fsck.h"
  16 #include "inode.h"
  17 #include "journal.h"
  18 #include "io.h"
  19 #include "keylist.h"
  20 #include "quota.h"
  21 #include "reflink.h"
  22
  23 #include <linux/aio.h>
  24 #include <linux/backing-dev.h>
  25 #include <linux/falloc.h>
  26 #include <linux/migrate.h>
  27 #include <linux/mmu_context.h>
  28 #include <linux/pagevec.h>
  29 #include <linux/rmap.h>
  30 #include <linux/sched/signal.h>
  31 #include <linux/task_io_accounting_ops.h>
  32 #include <linux/uio.h>
  33 #include <linux/writeback.h>
  34
  35 #include <trace/events/bcachefs.h>
  36 #include <trace/events/writeback.h>
  37
  38 struct nocow_flush {
  39         struct closure  *cl;
  40         struct bch_dev  *ca;
  41         struct bio      bio;
  42 };
  43
  44 static void nocow_flush_endio(struct bio *_bio)
  45 {
  46
  47         struct nocow_flush *bio = container_of(_bio, struct nocow_flush, bio);
  48
  49         closure_put(bio->cl);
  50         percpu_ref_put(&bio->ca->io_ref);
  51         bio_put(&bio->bio);
  52 }
  53
  54 static void bch2_inode_flush_nocow_writes_async(struct bch_fs *c,
  55                                                 struct bch_inode_info *inode,
  56                                                 struct closure *cl)
  57 {
  58         struct nocow_flush *bio;
  59         struct bch_dev *ca;
  60         struct bch_devs_mask devs;
  61         unsigned dev;
  62
  63         dev = find_first_bit(inode->ei_devs_need_flush.d, BCH_SB_MEMBERS_MAX);
  64         if (dev == BCH_SB_MEMBERS_MAX)
  65                 return;
  66
  67         devs = inode->ei_devs_need_flush;
  68         memset(&inode->ei_devs_need_flush, 0, sizeof(inode->ei_devs_need_flush));
  69
  70         for_each_set_bit(dev, devs.d, BCH_SB_MEMBERS_MAX) {
  71                 rcu_read_lock();
  72                 ca = rcu_dereference(c->devs[dev]);
  73                 if (ca && !percpu_ref_tryget(&ca->io_ref))
  74                         ca = NULL;
  75                 rcu_read_unlock();
  76
  77                 if (!ca)
  78                         continue;
  79
  80                 bio = container_of(bio_alloc_bioset(ca->disk_sb.bdev, 0,
  81                                                     REQ_OP_FLUSH,
  82                                                     GFP_KERNEL,
  83                                                     &c->nocow_flush_bioset),
  84                                    struct nocow_flush, bio);
  85                 bio->cl                 = cl;
  86                 bio->ca                 = ca;
  87                 bio->bio.bi_end_io      = nocow_flush_endio;
  88                 closure_bio_submit(&bio->bio, cl);
  89         }
  90 }
  91
  92 static int bch2_inode_flush_nocow_writes(struct bch_fs *c,
  93                                          struct bch_inode_info *inode)
  94 {
  95         struct closure cl;
  96
  97         closure_init_stack(&cl);
  98         bch2_inode_flush_nocow_writes_async(c, inode, &cl);
  99         closure_sync(&cl);
 100
 101         return 0;
 102 }
 103
 104 static inline bool bio_full(struct bio *bio, unsigned len)
 105 {
 106         if (bio->bi_vcnt >= bio->bi_max_vecs)
 107                 return true;
 108         if (bio->bi_iter.bi_size > UINT_MAX - len)
 109                 return true;
 110         return false;
 111 }
 112
 113 static inline struct address_space *faults_disabled_mapping(void)
 114 {
 115         return (void *) (((unsigned long) current->faults_disabled_mapping) & ~1UL);
 116 }
 117
 118 static inline void set_fdm_dropped_locks(void)
 119 {
 120         current->faults_disabled_mapping =
 121                 (void *) (((unsigned long) current->faults_disabled_mapping)|1);
 122 }
 123
 124 static inline bool fdm_dropped_locks(void)
 125 {
 126         return ((unsigned long) current->faults_disabled_mapping) & 1;
 127 }
 128
 129 struct quota_res {
 130         u64                             sectors;
 131 };
 132
 133 struct bch_writepage_io {
 134         struct bch_inode_info           *inode;
 135
 136         /* must be last: */
 137         struct bch_write_op             op;
 138 };
 139
 140 struct dio_write {
 141         struct kiocb                    *req;
 142         struct address_space            *mapping;
 143         struct bch_inode_info           *inode;
 144         struct mm_struct                *mm;
 145         unsigned                        loop:1,
 146                                         extending:1,
 147                                         sync:1,
 148                                         flush:1,
 149                                         free_iov:1;
 150         struct quota_res                quota_res;
 151         u64                             written;
 152
 153         struct iov_iter                 iter;
 154         struct iovec                    inline_vecs[2];
 155
 156         /* must be last: */
 157         struct bch_write_op             op;
 158 };
 159
 160 struct dio_read {
 161         struct closure                  cl;
 162         struct kiocb                    *req;
 163         long                            ret;
 164         bool                            should_dirty;
 165         struct bch_read_bio             rbio;
 166 };
 167
 168 /* pagecache_block must be held */
 169 static noinline int write_invalidate_inode_pages_range(struct address_space *mapping,
 170                                               loff_t start, loff_t end)
 171 {
 172         int ret;
 173
 174         /*
 175          * XXX: the way this is currently implemented, we can spin if a process
 176          * is continually redirtying a specific page
 177          */
 178         do {
 179                 if (!mapping->nrpages)
 180                         return 0;
 181
 182                 ret = filemap_write_and_wait_range(mapping, start, end);
 183                 if (ret)
 184                         break;
 185
 186                 if (!mapping->nrpages)
 187                         return 0;
 188
 189                 ret = invalidate_inode_pages2_range(mapping,
 190                                 start >> PAGE_SHIFT,
 191                                 end >> PAGE_SHIFT);
 192         } while (ret == -EBUSY);
 193
 194         return ret;
 195 }
 196
 197 /* quotas */
 198
 199 #ifdef CONFIG_BCACHEFS_QUOTA
 200
 201 static void __bch2_quota_reservation_put(struct bch_fs *c,
 202                                          struct bch_inode_info *inode,
 203                                          struct quota_res *res)
 204 {
 205         BUG_ON(res->sectors > inode->ei_quota_reserved);
 206
 207         bch2_quota_acct(c, inode->ei_qid, Q_SPC,
 208                         -((s64) res->sectors), KEY_TYPE_QUOTA_PREALLOC);
 209         inode->ei_quota_reserved -= res->sectors;
 210         res->sectors = 0;
 211 }
 212
 213 static void bch2_quota_reservation_put(struct bch_fs *c,
 214                                        struct bch_inode_info *inode,
 215                                        struct quota_res *res)
 216 {
 217         if (res->sectors) {
 218                 mutex_lock(&inode->ei_quota_lock);
 219                 __bch2_quota_reservation_put(c, inode, res);
 220                 mutex_unlock(&inode->ei_quota_lock);
 221         }
 222 }
 223
 224 static int bch2_quota_reservation_add(struct bch_fs *c,
 225                                       struct bch_inode_info *inode,
 226                                       struct quota_res *res,
 227                                       u64 sectors,
 228                                       bool check_enospc)
 229 {
 230         int ret;
 231
 232         mutex_lock(&inode->ei_quota_lock);
 233         ret = bch2_quota_acct(c, inode->ei_qid, Q_SPC, sectors,
 234                               check_enospc ? KEY_TYPE_QUOTA_PREALLOC : KEY_TYPE_QUOTA_NOCHECK);
 235         if (likely(!ret)) {
 236                 inode->ei_quota_reserved += sectors;
 237                 res->sectors += sectors;
 238         }
 239         mutex_unlock(&inode->ei_quota_lock);
 240
 241         return ret;
 242 }
 243
 244 #else
 245
 246 static void __bch2_quota_reservation_put(struct bch_fs *c,
 247                                          struct bch_inode_info *inode,
 248                                          struct quota_res *res) {}
 249
 250 static void bch2_quota_reservation_put(struct bch_fs *c,
 251                                        struct bch_inode_info *inode,
 252                                        struct quota_res *res) {}
 253
 254 static int bch2_quota_reservation_add(struct bch_fs *c,
 255                                       struct bch_inode_info *inode,
 256                                       struct quota_res *res,
 257                                       unsigned sectors,
 258                                       bool check_enospc)
 259 {
 260         return 0;
 261 }
 262
 263 #endif
 264
 265 /* i_size updates: */
 266
 267 struct inode_new_size {
 268         loff_t          new_size;
 269         u64             now;
 270         unsigned        fields;
 271 };
 272
 273 static int inode_set_size(struct bch_inode_info *inode,
 274                           struct bch_inode_unpacked *bi,
 275                           void *p)
 276 {
 277         struct inode_new_size *s = p;
 278
 279         bi->bi_size = s->new_size;
 280         if (s->fields & ATTR_ATIME)
 281                 bi->bi_atime = s->now;
 282         if (s->fields & ATTR_MTIME)
 283                 bi->bi_mtime = s->now;
 284         if (s->fields & ATTR_CTIME)
 285                 bi->bi_ctime = s->now;
 286
 287         return 0;
 288 }
 289
 290 int __must_check bch2_write_inode_size(struct bch_fs *c,
 291                                        struct bch_inode_info *inode,
 292                                        loff_t new_size, unsigned fields)
 293 {
 294         struct inode_new_size s = {
 295                 .new_size       = new_size,
 296                 .now            = bch2_current_time(c),
 297                 .fields         = fields,
 298         };
 299
 300         return bch2_write_inode(c, inode, inode_set_size, &s, fields);
 301 }
 302
 303 static void __i_sectors_acct(struct bch_fs *c, struct bch_inode_info *inode,
 304                            struct quota_res *quota_res, s64 sectors)
 305 {
 306         bch2_fs_inconsistent_on((s64) inode->v.i_blocks + sectors < 0, c,
 307                                 "inode %lu i_blocks underflow: %llu + %lli < 0 (ondisk %lli)",
 308                                 inode->v.i_ino, (u64) inode->v.i_blocks, sectors,
 309                                 inode->ei_inode.bi_sectors);
 310         inode->v.i_blocks += sectors;
 311
 312 #ifdef CONFIG_BCACHEFS_QUOTA
 313         if (quota_res && sectors > 0) {
 314                 BUG_ON(sectors > quota_res->sectors);
 315                 BUG_ON(sectors > inode->ei_quota_reserved);
 316
 317                 quota_res->sectors -= sectors;
 318                 inode->ei_quota_reserved -= sectors;
 319         } else {
 320                 bch2_quota_acct(c, inode->ei_qid, Q_SPC, sectors, KEY_TYPE_QUOTA_WARN);
 321         }
 322 #endif
 323 }
 324
 325 static void i_sectors_acct(struct bch_fs *c, struct bch_inode_info *inode,
 326                            struct quota_res *quota_res, s64 sectors)
 327 {
 328         if (sectors) {
 329                 mutex_lock(&inode->ei_quota_lock);
 330                 __i_sectors_acct(c, inode, quota_res, sectors);
 331                 mutex_unlock(&inode->ei_quota_lock);
 332         }
 333 }
 334
 335 /* page state: */
 336
 337 /* stored in page->private: */
 338
 339 struct bch_page_sector {
 340         /* Uncompressed, fully allocated replicas (or on disk reservation): */
 341         unsigned                nr_replicas:4;
 342
 343         /* Owns PAGE_SECTORS * replicas_reserved sized in memory reservation: */
 344         unsigned                replicas_reserved:4;
 345
 346         /* i_sectors: */
 347         enum {
 348                 SECTOR_UNALLOCATED,
 349                 SECTOR_RESERVED,
 350                 SECTOR_DIRTY,
 351                 SECTOR_DIRTY_RESERVED,
 352                 SECTOR_ALLOCATED,
 353         }                       state:8;
 354 };
 355
 356 struct bch_page_state {
 357         spinlock_t              lock;
 358         atomic_t                write_count;
 359         bool                    uptodate;
 360         struct bch_page_sector  s[PAGE_SECTORS];
 361 };
 362
 363 static inline struct bch_page_state *__bch2_page_state(struct page *page)
 364 {
 365         return page_has_private(page)
 366                 ? (struct bch_page_state *) page_private(page)
 367                 : NULL;
 368 }
 369
 370 static inline struct bch_page_state *bch2_page_state(struct page *page)
 371 {
 372         EBUG_ON(!PageLocked(page));
 373
 374         return __bch2_page_state(page);
 375 }
 376
 377 /* for newly allocated pages: */
 378 static void __bch2_page_state_release(struct page *page)
 379 {
 380         kfree(detach_page_private(page));
 381 }
 382
 383 static void bch2_page_state_release(struct page *page)
 384 {
 385         EBUG_ON(!PageLocked(page));
 386         __bch2_page_state_release(page);
 387 }
 388
 389 /* for newly allocated pages: */
 390 static struct bch_page_state *__bch2_page_state_create(struct page *page,
 391                                                        gfp_t gfp)
 392 {
 393         struct bch_page_state *s;
 394
 395         s = kzalloc(sizeof(*s), GFP_NOFS|gfp);
 396         if (!s)
 397                 return NULL;
 398
 399         spin_lock_init(&s->lock);
 400         attach_page_private(page, s);
 401         return s;
 402 }
 403
 404 static struct bch_page_state *bch2_page_state_create(struct page *page,
 405                                                      gfp_t gfp)
 406 {
 407         return bch2_page_state(page) ?: __bch2_page_state_create(page, gfp);
 408 }
 409
 410 static unsigned bkey_to_sector_state(struct bkey_s_c k)
 411 {
 412         if (bkey_extent_is_reservation(k))
 413                 return SECTOR_RESERVED;
 414         if (bkey_extent_is_allocation(k.k))
 415                 return SECTOR_ALLOCATED;
 416         return SECTOR_UNALLOCATED;
 417 }
 418
 419 static void __bch2_page_state_set(struct page *page,
 420                                   unsigned pg_offset, unsigned pg_len,
 421                                   unsigned nr_ptrs, unsigned state)
 422 {
 423         struct bch_page_state *s = bch2_page_state_create(page, __GFP_NOFAIL);
 424         unsigned i;
 425
 426         BUG_ON(pg_offset >= PAGE_SECTORS);
 427         BUG_ON(pg_offset + pg_len > PAGE_SECTORS);
 428
 429         spin_lock(&s->lock);
 430
 431         for (i = pg_offset; i < pg_offset + pg_len; i++) {
 432                 s->s[i].nr_replicas = nr_ptrs;
 433                 s->s[i].state = state;
 434         }
 435
 436         if (i == PAGE_SECTORS)
 437                 s->uptodate = true;
 438
 439         spin_unlock(&s->lock);
 440 }
 441
 442 static int bch2_page_state_set(struct bch_fs *c, subvol_inum inum,
 443                                struct page **pages, unsigned nr_pages)
 444 {
 445         struct btree_trans trans;
 446         struct btree_iter iter;
 447         struct bkey_s_c k;
 448         u64 offset = pages[0]->index << PAGE_SECTORS_SHIFT;
 449         unsigned pg_idx = 0;
 450         u32 snapshot;
 451         int ret;
 452
 453         bch2_trans_init(&trans, c, 0, 0);
 454 retry:
 455         bch2_trans_begin(&trans);
 456
 457         ret = bch2_subvolume_get_snapshot(&trans, inum.subvol, &snapshot);
 458         if (ret)
 459                 goto err;
 460
 461         for_each_btree_key_norestart(&trans, iter, BTREE_ID_extents,
 462                            SPOS(inum.inum, offset, snapshot),
 463                            BTREE_ITER_SLOTS, k, ret) {
 464                 unsigned nr_ptrs = bch2_bkey_nr_ptrs_fully_allocated(k);
 465                 unsigned state = bkey_to_sector_state(k);
 466
 467                 while (pg_idx < nr_pages) {
 468                         struct page *page = pages[pg_idx];
 469                         u64 pg_start = page->index << PAGE_SECTORS_SHIFT;
 470                         u64 pg_end = (page->index + 1) << PAGE_SECTORS_SHIFT;
 471                         unsigned pg_offset = max(bkey_start_offset(k.k), pg_start) - pg_start;
 472                         unsigned pg_len = min(k.k->p.offset, pg_end) - pg_offset - pg_start;
 473
 474                         BUG_ON(k.k->p.offset < pg_start);
 475                         BUG_ON(bkey_start_offset(k.k) > pg_end);
 476
 477                         if (!bch2_page_state_create(page, __GFP_NOFAIL)->uptodate)
 478                                 __bch2_page_state_set(page, pg_offset, pg_len, nr_ptrs, state);
 479
 480                         if (k.k->p.offset < pg_end)
 481                                 break;
 482                         pg_idx++;
 483                 }
 484
 485                 if (pg_idx == nr_pages)
 486                         break;
 487         }
 488
 489         offset = iter.pos.offset;
 490         bch2_trans_iter_exit(&trans, &iter);
 491 err:
 492         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
 493                 goto retry;
 494         bch2_trans_exit(&trans);
 495
 496         return ret;
 497 }
 498
 499 static void bch2_bio_page_state_set(struct bio *bio, struct bkey_s_c k)
 500 {
 501         struct bvec_iter iter;
 502         struct bio_vec bv;
 503         unsigned nr_ptrs = k.k->type == KEY_TYPE_reflink_v
 504                 ? 0 : bch2_bkey_nr_ptrs_fully_allocated(k);
 505         unsigned state = bkey_to_sector_state(k);
 506
 507         bio_for_each_segment(bv, bio, iter)
 508                 __bch2_page_state_set(bv.bv_page, bv.bv_offset >> 9,
 509                                       bv.bv_len >> 9, nr_ptrs, state);
 510 }
 511
 512 static void mark_pagecache_unallocated(struct bch_inode_info *inode,
 513                                        u64 start, u64 end)
 514 {
 515         pgoff_t index = start >> PAGE_SECTORS_SHIFT;
 516         pgoff_t end_index = (end - 1) >> PAGE_SECTORS_SHIFT;
 517         struct folio_batch fbatch;
 518         unsigned i, j;
 519
 520         if (end <= start)
 521                 return;
 522
 523         folio_batch_init(&fbatch);
 524
 525         while (filemap_get_folios(inode->v.i_mapping,
 526                                   &index, end_index, &fbatch)) {
 527                 for (i = 0; i < folio_batch_count(&fbatch); i++) {
 528                         struct folio *folio = fbatch.folios[i];
 529                         u64 pg_start = folio->index << PAGE_SECTORS_SHIFT;
 530                         u64 pg_end = (folio->index + 1) << PAGE_SECTORS_SHIFT;
 531                         unsigned pg_offset = max(start, pg_start) - pg_start;
 532                         unsigned pg_len = min(end, pg_end) - pg_offset - pg_start;
 533                         struct bch_page_state *s;
 534
 535                         BUG_ON(end <= pg_start);
 536                         BUG_ON(pg_offset >= PAGE_SECTORS);
 537                         BUG_ON(pg_offset + pg_len > PAGE_SECTORS);
 538
 539                         folio_lock(folio);
 540                         s = bch2_page_state(&folio->page);
 541
 542                         if (s) {
 543                                 spin_lock(&s->lock);
 544                                 for (j = pg_offset; j < pg_offset + pg_len; j++)
 545                                         s->s[j].nr_replicas = 0;
 546                                 spin_unlock(&s->lock);
 547                         }
 548
 549                         folio_unlock(folio);
 550                 }
 551                 folio_batch_release(&fbatch);
 552                 cond_resched();
 553         }
 554 }
 555
 556 static void mark_pagecache_reserved(struct bch_inode_info *inode,
 557                                     u64 start, u64 end)
 558 {
 559         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 560         pgoff_t index = start >> PAGE_SECTORS_SHIFT;
 561         pgoff_t end_index = (end - 1) >> PAGE_SECTORS_SHIFT;
 562         struct folio_batch fbatch;
 563         s64 i_sectors_delta = 0;
 564         unsigned i, j;
 565
 566         if (end <= start)
 567                 return;
 568
 569         folio_batch_init(&fbatch);
 570
 571         while (filemap_get_folios(inode->v.i_mapping,
 572                                   &index, end_index, &fbatch)) {
 573                 for (i = 0; i < folio_batch_count(&fbatch); i++) {
 574                         struct folio *folio = fbatch.folios[i];
 575                         u64 pg_start = folio->index << PAGE_SECTORS_SHIFT;
 576                         u64 pg_end = (folio->index + 1) << PAGE_SECTORS_SHIFT;
 577                         unsigned pg_offset = max(start, pg_start) - pg_start;
 578                         unsigned pg_len = min(end, pg_end) - pg_offset - pg_start;
 579                         struct bch_page_state *s;
 580
 581                         BUG_ON(end <= pg_start);
 582                         BUG_ON(pg_offset >= PAGE_SECTORS);
 583                         BUG_ON(pg_offset + pg_len > PAGE_SECTORS);
 584
 585                         folio_lock(folio);
 586                         s = bch2_page_state(&folio->page);
 587
 588                         if (s) {
 589                                 spin_lock(&s->lock);
 590                                 for (j = pg_offset; j < pg_offset + pg_len; j++)
 591                                         switch (s->s[j].state) {
 592                                         case SECTOR_UNALLOCATED:
 593                                                 s->s[j].state = SECTOR_RESERVED;
 594                                                 break;
 595                                         case SECTOR_DIRTY:
 596                                                 s->s[j].state = SECTOR_DIRTY_RESERVED;
 597                                                 i_sectors_delta--;
 598                                                 break;
 599                                         default:
 600                                                 break;
 601                                         }
 602                                 spin_unlock(&s->lock);
 603                         }
 604
 605                         folio_unlock(folio);
 606                 }
 607                 folio_batch_release(&fbatch);
 608                 cond_resched();
 609         }
 610
 611         i_sectors_acct(c, inode, NULL, i_sectors_delta);
 612 }
 613
 614 static inline unsigned inode_nr_replicas(struct bch_fs *c, struct bch_inode_info *inode)
 615 {
 616         /* XXX: this should not be open coded */
 617         return inode->ei_inode.bi_data_replicas
 618                 ? inode->ei_inode.bi_data_replicas - 1
 619                 : c->opts.data_replicas;
 620 }
 621
 622 static inline unsigned sectors_to_reserve(struct bch_page_sector *s,
 623                                                   unsigned nr_replicas)
 624 {
 625         return max(0, (int) nr_replicas -
 626                    s->nr_replicas -
 627                    s->replicas_reserved);
 628 }
 629
 630 static int bch2_get_page_disk_reservation(struct bch_fs *c,
 631                                 struct bch_inode_info *inode,
 632                                 struct page *page, bool check_enospc)
 633 {
 634         struct bch_page_state *s = bch2_page_state_create(page, 0);
 635         unsigned nr_replicas = inode_nr_replicas(c, inode);
 636         struct disk_reservation disk_res = { 0 };
 637         unsigned i, disk_res_sectors = 0;
 638         int ret;
 639
 640         if (!s)
 641                 return -ENOMEM;
 642
 643         for (i = 0; i < ARRAY_SIZE(s->s); i++)
 644                 disk_res_sectors += sectors_to_reserve(&s->s[i], nr_replicas);
 645
 646         if (!disk_res_sectors)
 647                 return 0;
 648
 649         ret = bch2_disk_reservation_get(c, &disk_res,
 650                                         disk_res_sectors, 1,
 651                                         !check_enospc
 652                                         ? BCH_DISK_RESERVATION_NOFAIL
 653                                         : 0);
 654         if (unlikely(ret))
 655                 return ret;
 656
 657         for (i = 0; i < ARRAY_SIZE(s->s); i++)
 658                 s->s[i].replicas_reserved +=
 659                         sectors_to_reserve(&s->s[i], nr_replicas);
 660
 661         return 0;
 662 }
 663
 664 struct bch2_page_reservation {
 665         struct disk_reservation disk;
 666         struct quota_res        quota;
 667 };
 668
 669 static void bch2_page_reservation_init(struct bch_fs *c,
 670                         struct bch_inode_info *inode,
 671                         struct bch2_page_reservation *res)
 672 {
 673         memset(res, 0, sizeof(*res));
 674
 675         res->disk.nr_replicas = inode_nr_replicas(c, inode);
 676 }
 677
 678 static void bch2_page_reservation_put(struct bch_fs *c,
 679                         struct bch_inode_info *inode,
 680                         struct bch2_page_reservation *res)
 681 {
 682         bch2_disk_reservation_put(c, &res->disk);
 683         bch2_quota_reservation_put(c, inode, &res->quota);
 684 }
 685
 686 static int bch2_page_reservation_get(struct bch_fs *c,
 687                         struct bch_inode_info *inode, struct page *page,
 688                         struct bch2_page_reservation *res,
 689                         unsigned offset, unsigned len)
 690 {
 691         struct bch_page_state *s = bch2_page_state_create(page, 0);
 692         unsigned i, disk_sectors = 0, quota_sectors = 0;
 693         int ret;
 694
 695         if (!s)
 696                 return -ENOMEM;
 697
 698         BUG_ON(!s->uptodate);
 699
 700         for (i = round_down(offset, block_bytes(c)) >> 9;
 701              i < round_up(offset + len, block_bytes(c)) >> 9;
 702              i++) {
 703                 disk_sectors += sectors_to_reserve(&s->s[i],
 704                                                 res->disk.nr_replicas);
 705                 quota_sectors += s->s[i].state == SECTOR_UNALLOCATED;
 706         }
 707
 708         if (disk_sectors) {
 709                 ret = bch2_disk_reservation_add(c, &res->disk, disk_sectors, 0);
 710                 if (unlikely(ret))
 711                         return ret;
 712         }
 713
 714         if (quota_sectors) {
 715                 ret = bch2_quota_reservation_add(c, inode, &res->quota,
 716                                                  quota_sectors, true);
 717                 if (unlikely(ret)) {
 718                         struct disk_reservation tmp = {
 719                                 .sectors = disk_sectors
 720                         };
 721
 722                         bch2_disk_reservation_put(c, &tmp);
 723                         res->disk.sectors -= disk_sectors;
 724                         return ret;
 725                 }
 726         }
 727
 728         return 0;
 729 }
 730
 731 static void bch2_clear_page_bits(struct page *page)
 732 {
 733         struct bch_inode_info *inode = to_bch_ei(page->mapping->host);
 734         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 735         struct bch_page_state *s = bch2_page_state(page);
 736         struct disk_reservation disk_res = { 0 };
 737         int i, dirty_sectors = 0;
 738
 739         if (!s)
 740                 return;
 741
 742         EBUG_ON(!PageLocked(page));
 743         EBUG_ON(PageWriteback(page));
 744
 745         for (i = 0; i < ARRAY_SIZE(s->s); i++) {
 746                 disk_res.sectors += s->s[i].replicas_reserved;
 747                 s->s[i].replicas_reserved = 0;
 748
 749                 switch (s->s[i].state) {
 750                 case SECTOR_DIRTY:
 751                         s->s[i].state = SECTOR_UNALLOCATED;
 752                         --dirty_sectors;
 753                         break;
 754                 case SECTOR_DIRTY_RESERVED:
 755                         s->s[i].state = SECTOR_RESERVED;
 756                         break;
 757                 default:
 758                         break;
 759                 }
 760         }
 761
 762         bch2_disk_reservation_put(c, &disk_res);
 763
 764         i_sectors_acct(c, inode, NULL, dirty_sectors);
 765
 766         bch2_page_state_release(page);
 767 }
 768
 769 static void bch2_set_page_dirty(struct bch_fs *c,
 770                         struct bch_inode_info *inode, struct page *page,
 771                         struct bch2_page_reservation *res,
 772                         unsigned offset, unsigned len)
 773 {
 774         struct bch_page_state *s = bch2_page_state(page);
 775         unsigned i, dirty_sectors = 0;
 776
 777         WARN_ON((u64) page_offset(page) + offset + len >
 778                 round_up((u64) i_size_read(&inode->v), block_bytes(c)));
 779
 780         spin_lock(&s->lock);
 781
 782         for (i = round_down(offset, block_bytes(c)) >> 9;
 783              i < round_up(offset + len, block_bytes(c)) >> 9;
 784              i++) {
 785                 unsigned sectors = sectors_to_reserve(&s->s[i],
 786                                                 res->disk.nr_replicas);
 787
 788                 /*
 789                  * This can happen if we race with the error path in
 790                  * bch2_writepage_io_done():
 791                  */
 792                 sectors = min_t(unsigned, sectors, res->disk.sectors);
 793
 794                 s->s[i].replicas_reserved += sectors;
 795                 res->disk.sectors -= sectors;
 796
 797                 switch (s->s[i].state) {
 798                 case SECTOR_UNALLOCATED:
 799                         s->s[i].state = SECTOR_DIRTY;
 800                         dirty_sectors++;
 801                         break;
 802                 case SECTOR_RESERVED:
 803                         s->s[i].state = SECTOR_DIRTY_RESERVED;
 804                         break;
 805                 default:
 806                         break;
 807                 }
 808         }
 809
 810         spin_unlock(&s->lock);
 811
 812         i_sectors_acct(c, inode, &res->quota, dirty_sectors);
 813
 814         if (!PageDirty(page))
 815                 __set_page_dirty_nobuffers(page);
 816 }
 817
 818 vm_fault_t bch2_page_fault(struct vm_fault *vmf)
 819 {
 820         struct file *file = vmf->vma->vm_file;
 821         struct address_space *mapping = file->f_mapping;
 822         struct address_space *fdm = faults_disabled_mapping();
 823         struct bch_inode_info *inode = file_bch_inode(file);
 824         int ret;
 825
 826         if (fdm == mapping)
 827                 return VM_FAULT_SIGBUS;
 828
 829         /* Lock ordering: */
 830         if (fdm > mapping) {
 831                 struct bch_inode_info *fdm_host = to_bch_ei(fdm->host);
 832
 833                 if (bch2_pagecache_add_tryget(inode))
 834                         goto got_lock;
 835
 836                 bch2_pagecache_block_put(fdm_host);
 837
 838                 bch2_pagecache_add_get(inode);
 839                 bch2_pagecache_add_put(inode);
 840
 841                 bch2_pagecache_block_get(fdm_host);
 842
 843                 /* Signal that lock has been dropped: */
 844                 set_fdm_dropped_locks();
 845                 return VM_FAULT_SIGBUS;
 846         }
 847
 848         bch2_pagecache_add_get(inode);
 849 got_lock:
 850         ret = filemap_fault(vmf);
 851         bch2_pagecache_add_put(inode);
 852
 853         return ret;
 854 }
 855
 856 vm_fault_t bch2_page_mkwrite(struct vm_fault *vmf)
 857 {
 858         struct page *page = vmf->page;
 859         struct file *file = vmf->vma->vm_file;
 860         struct bch_inode_info *inode = file_bch_inode(file);
 861         struct address_space *mapping = file->f_mapping;
 862         struct bch_fs *c = inode->v.i_sb->s_fs_info;
 863         struct bch2_page_reservation res;
 864         unsigned len;
 865         loff_t isize;
 866         int ret;
 867
 868         bch2_page_reservation_init(c, inode, &res);
 869
 870         sb_start_pagefault(inode->v.i_sb);
 871         file_update_time(file);
 872
 873         /*
 874          * Not strictly necessary, but helps avoid dio writes livelocking in
 875          * write_invalidate_inode_pages_range() - can drop this if/when we get
 876          * a write_invalidate_inode_pages_range() that works without dropping
 877          * page lock before invalidating page
 878          */
 879         bch2_pagecache_add_get(inode);
 880
 881         lock_page(page);
 882         isize = i_size_read(&inode->v);
 883
 884         if (page->mapping != mapping || page_offset(page) >= isize) {
 885                 unlock_page(page);
 886                 ret = VM_FAULT_NOPAGE;
 887                 goto out;
 888         }
 889
 890         len = min_t(loff_t, PAGE_SIZE, isize - page_offset(page));
 891
 892         if (!bch2_page_state_create(page, __GFP_NOFAIL)->uptodate) {
 893                 if (bch2_page_state_set(c, inode_inum(inode), &page, 1)) {
 894                         unlock_page(page);
 895                         ret = VM_FAULT_SIGBUS;
 896                         goto out;
 897                 }
 898         }
 899
 900         if (bch2_page_reservation_get(c, inode, page, &res, 0, len)) {
 901                 unlock_page(page);
 902                 ret = VM_FAULT_SIGBUS;
 903                 goto out;
 904         }
 905
 906         bch2_set_page_dirty(c, inode, page, &res, 0, len);
 907         bch2_page_reservation_put(c, inode, &res);
 908
 909         wait_for_stable_page(page);
 910         ret = VM_FAULT_LOCKED;
 911 out:
 912         bch2_pagecache_add_put(inode);
 913         sb_end_pagefault(inode->v.i_sb);
 914
 915         return ret;
 916 }
 917
 918 void bch2_invalidate_folio(struct folio *folio, size_t offset, size_t length)
 919 {
 920         if (offset || length < folio_size(folio))
 921                 return;
 922
 923         bch2_clear_page_bits(&folio->page);
 924 }
 925
 926 bool bch2_release_folio(struct folio *folio, gfp_t gfp_mask)
 927 {
 928         if (folio_test_dirty(folio) || folio_test_writeback(folio))
 929                 return false;
 930
 931         bch2_clear_page_bits(&folio->page);
 932         return true;
 933 }
 934
 935 /* readpage(s): */
 936
 937 static void bch2_readpages_end_io(struct bio *bio)
 938 {
 939         struct bvec_iter_all iter;
 940         struct bio_vec *bv;
 941
 942         bio_for_each_segment_all(bv, bio, iter) {
 943                 struct page *page = bv->bv_page;
 944
 945                 if (!bio->bi_status) {
 946                         SetPageUptodate(page);
 947                 } else {
 948                         ClearPageUptodate(page);
 949                         SetPageError(page);
 950                 }
 951                 unlock_page(page);
 952         }
 953
 954         bio_put(bio);
 955 }
 956
 957 struct readpages_iter {
 958         struct address_space    *mapping;
 959         struct page             **pages;
 960         unsigned                nr_pages;
 961         unsigned                idx;
 962         pgoff_t                 offset;
 963 };
 964
 965 static int readpages_iter_init(struct readpages_iter *iter,
 966                                struct readahead_control *ractl)
 967 {
 968         unsigned i, nr_pages = readahead_count(ractl);
 969
 970         memset(iter, 0, sizeof(*iter));
 971
 972         iter->mapping   = ractl->mapping;
 973         iter->offset    = readahead_index(ractl);
 974         iter->nr_pages  = nr_pages;
 975
 976         iter->pages = kmalloc_array(nr_pages, sizeof(struct page *), GFP_NOFS);
 977         if (!iter->pages)
 978                 return -ENOMEM;
 979
 980         nr_pages = __readahead_batch(ractl, iter->pages, nr_pages);
 981         for (i = 0; i < nr_pages; i++) {
 982                 __bch2_page_state_create(iter->pages[i], __GFP_NOFAIL);
 983                 put_page(iter->pages[i]);
 984         }
 985
 986         return 0;
 987 }
 988
 989 static inline struct page *readpage_iter_next(struct readpages_iter *iter)
 990 {
 991         if (iter->idx >= iter->nr_pages)
 992                 return NULL;
 993
 994         EBUG_ON(iter->pages[iter->idx]->index != iter->offset + iter->idx);
 995
 996         return iter->pages[iter->idx];
 997 }
 998
 999 static bool extent_partial_reads_expensive(struct bkey_s_c k)
1000 {
1001         struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
1002         struct bch_extent_crc_unpacked crc;
1003         const union bch_extent_entry *i;
1004
1005         bkey_for_each_crc(k.k, ptrs, crc, i)
1006                 if (crc.csum_type || crc.compression_type)
1007                         return true;
1008         return false;
1009 }
1010
1011 static void readpage_bio_extend(struct readpages_iter *iter,
1012                                 struct bio *bio,
1013                                 unsigned sectors_this_extent,
1014                                 bool get_more)
1015 {
1016         while (bio_sectors(bio) < sectors_this_extent &&
1017                bio->bi_vcnt < bio->bi_max_vecs) {
1018                 pgoff_t page_offset = bio_end_sector(bio) >> PAGE_SECTORS_SHIFT;
1019                 struct page *page = readpage_iter_next(iter);
1020                 int ret;
1021
1022                 if (page) {
1023                         if (iter->offset + iter->idx != page_offset)
1024                                 break;
1025
1026                         iter->idx++;
1027                 } else {
1028                         if (!get_more)
1029                                 break;
1030
1031                         page = xa_load(&iter->mapping->i_pages, page_offset);
1032                         if (page && !xa_is_value(page))
1033                                 break;
1034
1035                         page = __page_cache_alloc(readahead_gfp_mask(iter->mapping));
1036                         if (!page)
1037                                 break;
1038
1039                         if (!__bch2_page_state_create(page, 0)) {
1040                                 put_page(page);
1041                                 break;
1042                         }
1043
1044                         ret = add_to_page_cache_lru(page, iter->mapping,
1045                                                     page_offset, GFP_NOFS);
1046                         if (ret) {
1047                                 __bch2_page_state_release(page);
1048                                 put_page(page);
1049                                 break;
1050                         }
1051
1052                         put_page(page);
1053                 }
1054
1055                 BUG_ON(!bio_add_page(bio, page, PAGE_SIZE, 0));
1056         }
1057 }
1058
1059 static void bchfs_read(struct btree_trans *trans,
1060                        struct bch_read_bio *rbio,
1061                        subvol_inum inum,
1062                        struct readpages_iter *readpages_iter)
1063 {
1064         struct bch_fs *c = trans->c;
1065         struct btree_iter iter;
1066         struct bkey_buf sk;
1067         int flags = BCH_READ_RETRY_IF_STALE|
1068                 BCH_READ_MAY_PROMOTE;
1069         u32 snapshot;
1070         int ret = 0;
1071
1072         rbio->c = c;
1073         rbio->start_time = local_clock();
1074         rbio->subvol = inum.subvol;
1075
1076         bch2_bkey_buf_init(&sk);
1077 retry:
1078         bch2_trans_begin(trans);
1079         iter = (struct btree_iter) { NULL };
1080
1081         ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot);
1082         if (ret)
1083                 goto err;
1084
1085         bch2_trans_iter_init(trans, &iter, BTREE_ID_extents,
1086                              SPOS(inum.inum, rbio->bio.bi_iter.bi_sector, snapshot),
1087                              BTREE_ITER_SLOTS);
1088         while (1) {
1089                 struct bkey_s_c k;
1090                 unsigned bytes, sectors, offset_into_extent;
1091                 enum btree_id data_btree = BTREE_ID_extents;
1092
1093                 /*
1094                  * read_extent -> io_time_reset may cause a transaction restart
1095                  * without returning an error, we need to check for that here:
1096                  */
1097                 ret = bch2_trans_relock(trans);
1098                 if (ret)
1099                         break;
1100
1101                 bch2_btree_iter_set_pos(&iter,
1102                                 POS(inum.inum, rbio->bio.bi_iter.bi_sector));
1103
1104                 k = bch2_btree_iter_peek_slot(&iter);
1105                 ret = bkey_err(k);
1106                 if (ret)
1107                         break;
1108
1109                 offset_into_extent = iter.pos.offset -
1110                         bkey_start_offset(k.k);
1111                 sectors = k.k->size - offset_into_extent;
1112
1113                 bch2_bkey_buf_reassemble(&sk, c, k);
1114
1115                 ret = bch2_read_indirect_extent(trans, &data_btree,
1116                                         &offset_into_extent, &sk);
1117                 if (ret)
1118                         break;
1119
1120                 k = bkey_i_to_s_c(sk.k);
1121
1122                 sectors = min(sectors, k.k->size - offset_into_extent);
1123
1124                 if (readpages_iter)
1125                         readpage_bio_extend(readpages_iter, &rbio->bio, sectors,
1126                                             extent_partial_reads_expensive(k));
1127
1128                 bytes = min(sectors, bio_sectors(&rbio->bio)) << 9;
1129                 swap(rbio->bio.bi_iter.bi_size, bytes);
1130
1131                 if (rbio->bio.bi_iter.bi_size == bytes)
1132                         flags |= BCH_READ_LAST_FRAGMENT;
1133
1134                 bch2_bio_page_state_set(&rbio->bio, k);
1135
1136                 bch2_read_extent(trans, rbio, iter.pos,
1137                                  data_btree, k, offset_into_extent, flags);
1138
1139                 if (flags & BCH_READ_LAST_FRAGMENT)
1140                         break;
1141
1142                 swap(rbio->bio.bi_iter.bi_size, bytes);
1143                 bio_advance(&rbio->bio, bytes);
1144
1145                 ret = btree_trans_too_many_iters(trans);
1146                 if (ret)
1147                         break;
1148         }
1149 err:
1150         bch2_trans_iter_exit(trans, &iter);
1151
1152         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
1153                 goto retry;
1154
1155         if (ret) {
1156                 bch_err_inum_offset_ratelimited(c,
1157                                 iter.pos.inode,
1158                                 iter.pos.offset << 9,
1159                                 "read error %i from btree lookup", ret);
1160                 rbio->bio.bi_status = BLK_STS_IOERR;
1161                 bio_endio(&rbio->bio);
1162         }
1163
1164         bch2_bkey_buf_exit(&sk, c);
1165 }
1166
1167 void bch2_readahead(struct readahead_control *ractl)
1168 {
1169         struct bch_inode_info *inode = to_bch_ei(ractl->mapping->host);
1170         struct bch_fs *c = inode->v.i_sb->s_fs_info;
1171         struct bch_io_opts opts = io_opts(c, &inode->ei_inode);
1172         struct btree_trans trans;
1173         struct page *page;
1174         struct readpages_iter readpages_iter;
1175         int ret;
1176
1177         ret = readpages_iter_init(&readpages_iter, ractl);
1178         BUG_ON(ret);
1179
1180         bch2_trans_init(&trans, c, 0, 0);
1181
1182         bch2_pagecache_add_get(inode);
1183
1184         while ((page = readpage_iter_next(&readpages_iter))) {
1185                 pgoff_t index = readpages_iter.offset + readpages_iter.idx;
1186                 unsigned n = min_t(unsigned,
1187                                    readpages_iter.nr_pages -
1188                                    readpages_iter.idx,
1189                                    BIO_MAX_VECS);
1190                 struct bch_read_bio *rbio =
1191                         rbio_init(bio_alloc_bioset(NULL, n, REQ_OP_READ,
1192                                                    GFP_NOFS, &c->bio_read),
1193                                   opts);
1194
1195                 readpages_iter.idx++;
1196
1197                 rbio->bio.bi_iter.bi_sector = (sector_t) index << PAGE_SECTORS_SHIFT;
1198                 rbio->bio.bi_end_io = bch2_readpages_end_io;
1199                 BUG_ON(!bio_add_page(&rbio->bio, page, PAGE_SIZE, 0));
1200
1201                 bchfs_read(&trans, rbio, inode_inum(inode),
1202                            &readpages_iter);
1203         }
1204
1205         bch2_pagecache_add_put(inode);
1206
1207         bch2_trans_exit(&trans);
1208         kfree(readpages_iter.pages);
1209 }
1210
1211 static void __bchfs_readpage(struct bch_fs *c, struct bch_read_bio *rbio,
1212                              subvol_inum inum, struct page *page)
1213 {
1214         struct btree_trans trans;
1215
1216         bch2_page_state_create(page, __GFP_NOFAIL);
1217
1218         bio_set_op_attrs(&rbio->bio, REQ_OP_READ, REQ_SYNC);
1219         rbio->bio.bi_iter.bi_sector =
1220                 (sector_t) page->index << PAGE_SECTORS_SHIFT;
1221         BUG_ON(!bio_add_page(&rbio->bio, page, PAGE_SIZE, 0));
1222
1223         bch2_trans_init(&trans, c, 0, 0);
1224         bchfs_read(&trans, rbio, inum, NULL);
1225         bch2_trans_exit(&trans);
1226 }
1227
1228 static void bch2_read_single_page_end_io(struct bio *bio)
1229 {
1230         complete(bio->bi_private);
1231 }
1232
1233 static int bch2_read_single_page(struct page *page,
1234                                  struct address_space *mapping)
1235 {
1236         struct bch_inode_info *inode = to_bch_ei(mapping->host);
1237         struct bch_fs *c = inode->v.i_sb->s_fs_info;
1238         struct bch_read_bio *rbio;
1239         int ret;
1240         DECLARE_COMPLETION_ONSTACK(done);
1241
1242         rbio = rbio_init(bio_alloc_bioset(NULL, 1, REQ_OP_READ, GFP_NOFS, &c->bio_read),
1243                          io_opts(c, &inode->ei_inode));
1244         rbio->bio.bi_private = &done;
1245         rbio->bio.bi_end_io = bch2_read_single_page_end_io;
1246
1247         __bchfs_readpage(c, rbio, inode_inum(inode), page);
1248         wait_for_completion(&done);
1249
1250         ret = blk_status_to_errno(rbio->bio.bi_status);
1251         bio_put(&rbio->bio);
1252
1253         if (ret < 0)
1254                 return ret;
1255
1256         SetPageUptodate(page);
1257         return 0;
1258 }
1259
1260 int bch2_read_folio(struct file *file, struct folio *folio)
1261 {
1262         struct page *page = &folio->page;
1263         int ret;
1264
1265         ret = bch2_read_single_page(page, page->mapping);
1266         folio_unlock(folio);
1267         return bch2_err_class(ret);
1268 }
1269
1270 /* writepages: */
1271
1272 struct bch_writepage_state {
1273         struct bch_writepage_io *io;
1274         struct bch_io_opts      opts;
1275 };
1276
1277 static inline struct bch_writepage_state bch_writepage_state_init(struct bch_fs *c,
1278                                                                   struct bch_inode_info *inode)
1279 {
1280         return (struct bch_writepage_state) {
1281                 .opts = io_opts(c, &inode->ei_inode)
1282         };
1283 }
1284
1285 static void bch2_writepage_io_done(struct bch_write_op *op)
1286 {
1287         struct bch_writepage_io *io =
1288                 container_of(op, struct bch_writepage_io, op);
1289         struct bch_fs *c = io->op.c;
1290         struct bio *bio = &io->op.wbio.bio;
1291         struct bvec_iter_all iter;
1292         struct bio_vec *bvec;
1293         unsigned i;
1294
1295         if (io->op.error) {
1296                 set_bit(EI_INODE_ERROR, &io->inode->ei_flags);
1297
1298                 bio_for_each_segment_all(bvec, bio, iter) {
1299                         struct bch_page_state *s;
1300
1301                         SetPageError(bvec->bv_page);
1302                         mapping_set_error(bvec->bv_page->mapping, -EIO);
1303
1304                         s = __bch2_page_state(bvec->bv_page);
1305                         spin_lock(&s->lock);
1306                         for (i = 0; i < PAGE_SECTORS; i++)
1307                                 s->s[i].nr_replicas = 0;
1308                         spin_unlock(&s->lock);
1309                 }
1310         }
1311
1312         if (io->op.flags & BCH_WRITE_WROTE_DATA_INLINE) {
1313                 bio_for_each_segment_all(bvec, bio, iter) {
1314                         struct bch_page_state *s;
1315
1316                         s = __bch2_page_state(bvec->bv_page);
1317                         spin_lock(&s->lock);
1318                         for (i = 0; i < PAGE_SECTORS; i++)
1319                                 s->s[i].nr_replicas = 0;
1320                         spin_unlock(&s->lock);
1321                 }
1322         }
1323
1324         /*
1325          * racing with fallocate can cause us to add fewer sectors than
1326          * expected - but we shouldn't add more sectors than expected:
1327          */
1328         WARN_ON_ONCE(io->op.i_sectors_delta > 0);
1329
1330         /*
1331          * (error (due to going RO) halfway through a page can screw that up
1332          * slightly)
1333          * XXX wtf?
1334            BUG_ON(io->op.op.i_sectors_delta >= PAGE_SECTORS);
1335          */
1336
1337         /*
1338          * PageWriteback is effectively our ref on the inode - fixup i_blocks
1339          * before calling end_page_writeback:
1340          */
1341         i_sectors_acct(c, io->inode, NULL, io->op.i_sectors_delta);
1342
1343         bio_for_each_segment_all(bvec, bio, iter) {
1344                 struct bch_page_state *s = __bch2_page_state(bvec->bv_page);
1345
1346                 if (atomic_dec_and_test(&s->write_count))
1347                         end_page_writeback(bvec->bv_page);
1348         }
1349
1350         bio_put(&io->op.wbio.bio);
1351 }
1352
1353 static void bch2_writepage_do_io(struct bch_writepage_state *w)
1354 {
1355         struct bch_writepage_io *io = w->io;
1356
1357         w->io = NULL;
1358         closure_call(&io->op.cl, bch2_write, NULL, NULL);
1359 }
1360
1361 /*
1362  * Get a bch_writepage_io and add @page to it - appending to an existing one if
1363  * possible, else allocating a new one:
1364  */
1365 static void bch2_writepage_io_alloc(struct bch_fs *c,
1366                                     struct writeback_control *wbc,
1367                                     struct bch_writepage_state *w,
1368                                     struct bch_inode_info *inode,
1369                                     u64 sector,
1370                                     unsigned nr_replicas)
1371 {
1372         struct bch_write_op *op;
1373
1374         w->io = container_of(bio_alloc_bioset(NULL, BIO_MAX_VECS,
1375                                               REQ_OP_WRITE,
1376                                               GFP_NOFS,
1377                                               &c->writepage_bioset),
1378                              struct bch_writepage_io, op.wbio.bio);
1379
1380         w->io->inode            = inode;
1381         op                      = &w->io->op;
1382         bch2_write_op_init(op, c, w->opts);
1383         op->target              = w->opts.foreground_target;
1384         op->nr_replicas         = nr_replicas;
1385         op->res.nr_replicas     = nr_replicas;
1386         op->write_point         = writepoint_hashed(inode->ei_last_dirtied);
1387         op->subvol              = inode->ei_subvol;
1388         op->pos                 = POS(inode->v.i_ino, sector);
1389         op->end_io              = bch2_writepage_io_done;
1390         op->devs_need_flush     = &inode->ei_devs_need_flush;
1391         op->wbio.bio.bi_iter.bi_sector = sector;
1392         op->wbio.bio.bi_opf     = wbc_to_write_flags(wbc);
1393 }
1394
1395 static int __bch2_writepage(struct page *page,
1396                             struct writeback_control *wbc,
1397                             void *data)
1398 {
1399         struct bch_inode_info *inode = to_bch_ei(page->mapping->host);
1400         struct bch_fs *c = inode->v.i_sb->s_fs_info;
1401         struct bch_writepage_state *w = data;
1402         struct bch_page_state *s, orig;
1403         unsigned i, offset, nr_replicas_this_write = U32_MAX;
1404         loff_t i_size = i_size_read(&inode->v);
1405         pgoff_t end_index = i_size >> PAGE_SHIFT;
1406         int ret;
1407
1408         EBUG_ON(!PageUptodate(page));
1409
1410         /* Is the page fully inside i_size? */
1411         if (page->index < end_index)
1412                 goto do_io;
1413
1414         /* Is the page fully outside i_size? (truncate in progress) */
1415         offset = i_size & (PAGE_SIZE - 1);
1416         if (page->index > end_index || !offset) {
1417                 unlock_page(page);
1418                 return 0;
1419         }
1420
1421         /*
1422          * The page straddles i_size.  It must be zeroed out on each and every
1423          * writepage invocation because it may be mmapped.  "A file is mapped
1424          * in multiples of the page size.  For a file that is not a multiple of
1425          * the  page size, the remaining memory is zeroed when mapped, and
1426          * writes to that region are not written out to the file."
1427          */
1428         zero_user_segment(page, offset, PAGE_SIZE);
1429 do_io:
1430         s = bch2_page_state_create(page, __GFP_NOFAIL);
1431
1432         /*
1433          * Things get really hairy with errors during writeback:
1434          */
1435         ret = bch2_get_page_disk_reservation(c, inode, page, false);
1436         BUG_ON(ret);
1437
1438         /* Before unlocking the page, get copy of reservations: */
1439         spin_lock(&s->lock);
1440         orig = *s;
1441         spin_unlock(&s->lock);
1442
1443         for (i = 0; i < PAGE_SECTORS; i++) {
1444                 if (s->s[i].state < SECTOR_DIRTY)
1445                         continue;
1446
1447                 nr_replicas_this_write =
1448                         min_t(unsigned, nr_replicas_this_write,
1449                               s->s[i].nr_replicas +
1450                               s->s[i].replicas_reserved);
1451         }
1452
1453         for (i = 0; i < PAGE_SECTORS; i++) {
1454                 if (s->s[i].state < SECTOR_DIRTY)
1455                         continue;
1456
1457                 s->s[i].nr_replicas = w->opts.compression
1458                         ? 0 : nr_replicas_this_write;
1459
1460                 s->s[i].replicas_reserved = 0;
1461                 s->s[i].state = SECTOR_ALLOCATED;
1462         }
1463
1464         BUG_ON(atomic_read(&s->write_count));
1465         atomic_set(&s->write_count, 1);
1466
1467         BUG_ON(PageWriteback(page));
1468         set_page_writeback(page);
1469
1470         unlock_page(page);
1471
1472         offset = 0;
1473         while (1) {
1474                 unsigned sectors = 0, dirty_sectors = 0, reserved_sectors = 0;
1475                 u64 sector;
1476
1477                 while (offset < PAGE_SECTORS &&
1478                        orig.s[offset].state < SECTOR_DIRTY)
1479                         offset++;
1480
1481                 if (offset == PAGE_SECTORS)
1482                         break;
1483
1484                 while (offset + sectors < PAGE_SECTORS &&
1485                        orig.s[offset + sectors].state >= SECTOR_DIRTY) {
1486                         reserved_sectors += orig.s[offset + sectors].replicas_reserved;
1487                         dirty_sectors += orig.s[offset + sectors].state == SECTOR_DIRTY;
1488                         sectors++;
1489                 }
1490                 BUG_ON(!sectors);
1491
1492                 sector = ((u64) page->index << PAGE_SECTORS_SHIFT) + offset;
1493
1494                 if (w->io &&
1495                     (w->io->op.res.nr_replicas != nr_replicas_this_write ||
1496                      bio_full(&w->io->op.wbio.bio, PAGE_SIZE) ||
1497                      w->io->op.wbio.bio.bi_iter.bi_size + (sectors << 9) >=
1498                      (BIO_MAX_VECS * PAGE_SIZE) ||
1499                      bio_end_sector(&w->io->op.wbio.bio) != sector))
1500                         bch2_writepage_do_io(w);
1501
1502                 if (!w->io)
1503                         bch2_writepage_io_alloc(c, wbc, w, inode, sector,
1504                                                 nr_replicas_this_write);
1505
1506                 atomic_inc(&s->write_count);
1507
1508                 BUG_ON(inode != w->io->inode);
1509                 BUG_ON(!bio_add_page(&w->io->op.wbio.bio, page,
1510                                      sectors << 9, offset << 9));
1511
1512                 /* Check for writing past i_size: */
1513                 WARN_ONCE((bio_end_sector(&w->io->op.wbio.bio) << 9) >
1514                           round_up(i_size, block_bytes(c)) &&
1515                           !test_bit(BCH_FS_EMERGENCY_RO, &c->flags),
1516                           "writing past i_size: %llu > %llu (unrounded %llu)\n",
1517                           bio_end_sector(&w->io->op.wbio.bio) << 9,
1518                           round_up(i_size, block_bytes(c)),
1519                           i_size);
1520
1521                 w->io->op.res.sectors += reserved_sectors;
1522                 w->io->op.i_sectors_delta -= dirty_sectors;
1523                 w->io->op.new_i_size = i_size;
1524
1525                 offset += sectors;
1526         }
1527
1528         if (atomic_dec_and_test(&s->write_count))
1529                 end_page_writeback(page);
1530
1531         return 0;
1532 }
1533
1534 int bch2_writepages(struct address_space *mapping, struct writeback_control *wbc)
1535 {
1536         struct bch_fs *c = mapping->host->i_sb->s_fs_info;
1537         struct bch_writepage_state w =
1538                 bch_writepage_state_init(c, to_bch_ei(mapping->host));
1539         struct blk_plug plug;
1540         int ret;
1541
1542         blk_start_plug(&plug);
1543         ret = write_cache_pages(mapping, wbc, __bch2_writepage, &w);
1544         if (w.io)
1545                 bch2_writepage_do_io(&w);
1546         blk_finish_plug(&plug);
1547         return bch2_err_class(ret);
1548 }
1549
1550 /* buffered writes: */
1551
1552 int bch2_write_begin(struct file *file, struct address_space *mapping,
1553                      loff_t pos, unsigned len,
1554                      struct page **pagep, void **fsdata)
1555 {
1556         struct bch_inode_info *inode = to_bch_ei(mapping->host);
1557         struct bch_fs *c = inode->v.i_sb->s_fs_info;
1558         struct bch2_page_reservation *res;
1559         pgoff_t index = pos >> PAGE_SHIFT;
1560         unsigned offset = pos & (PAGE_SIZE - 1);
1561         struct page *page;
1562         int ret = -ENOMEM;
1563
1564         res = kmalloc(sizeof(*res), GFP_KERNEL);
1565         if (!res)
1566                 return -ENOMEM;
1567
1568         bch2_page_reservation_init(c, inode, res);
1569         *fsdata = res;
1570
1571         bch2_pagecache_add_get(inode);
1572
1573         page = grab_cache_page_write_begin(mapping, index);
1574         if (!page)
1575                 goto err_unlock;
1576
1577         if (PageUptodate(page))
1578                 goto out;
1579
1580         /* If we're writing entire page, don't need to read it in first: */
1581         if (len == PAGE_SIZE)
1582                 goto out;
1583
1584         if (!offset && pos + len >= inode->v.i_size) {
1585                 zero_user_segment(page, len, PAGE_SIZE);
1586                 flush_dcache_page(page);
1587                 goto out;
1588         }
1589
1590         if (index > inode->v.i_size >> PAGE_SHIFT) {
1591                 zero_user_segments(page, 0, offset, offset + len, PAGE_SIZE);
1592                 flush_dcache_page(page);
1593                 goto out;
1594         }
1595 readpage:
1596         ret = bch2_read_single_page(page, mapping);
1597         if (ret)
1598                 goto err;
1599 out:
1600         if (!bch2_page_state_create(page, __GFP_NOFAIL)->uptodate) {
1601                 ret = bch2_page_state_set(c, inode_inum(inode), &page, 1);
1602                 if (ret)
1603                         goto err;
1604         }
1605
1606         ret = bch2_page_reservation_get(c, inode, page, res, offset, len);
1607         if (ret) {
1608                 if (!PageUptodate(page)) {
1609                         /*
1610                          * If the page hasn't been read in, we won't know if we
1611                          * actually need a reservation - we don't actually need
1612                          * to read here, we just need to check if the page is
1613                          * fully backed by uncompressed data:
1614                          */
1615                         goto readpage;
1616                 }
1617
1618                 goto err;
1619         }
1620
1621         *pagep = page;
1622         return 0;
1623 err:
1624         unlock_page(page);
1625         put_page(page);
1626         *pagep = NULL;
1627 err_unlock:
1628         bch2_pagecache_add_put(inode);
1629         kfree(res);
1630         *fsdata = NULL;
1631         return bch2_err_class(ret);
1632 }
1633
1634 int bch2_write_end(struct file *file, struct address_space *mapping,
1635                    loff_t pos, unsigned len, unsigned copied,
1636                    struct page *page, void *fsdata)
1637 {
1638         struct bch_inode_info *inode = to_bch_ei(mapping->host);
1639         struct bch_fs *c = inode->v.i_sb->s_fs_info;
1640         struct bch2_page_reservation *res = fsdata;
1641         unsigned offset = pos & (PAGE_SIZE - 1);
1642
1643         lockdep_assert_held(&inode->v.i_rwsem);
1644
1645         if (unlikely(copied < len && !PageUptodate(page))) {
1646                 /*
1647                  * The page needs to be read in, but that would destroy
1648                  * our partial write - simplest thing is to just force
1649                  * userspace to redo the write:
1650                  */
1651                 zero_user(page, 0, PAGE_SIZE);
1652                 flush_dcache_page(page);
1653                 copied = 0;
1654         }
1655
1656         spin_lock(&inode->v.i_lock);
1657         if (pos + copied > inode->v.i_size)
1658                 i_size_write(&inode->v, pos + copied);
1659         spin_unlock(&inode->v.i_lock);
1660
1661         if (copied) {
1662                 if (!PageUptodate(page))
1663                         SetPageUptodate(page);
1664
1665                 bch2_set_page_dirty(c, inode, page, res, offset, copied);
1666
1667                 inode->ei_last_dirtied = (unsigned long) current;
1668         }
1669
1670         unlock_page(page);
1671         put_page(page);
1672         bch2_pagecache_add_put(inode);
1673
1674         bch2_page_reservation_put(c, inode, res);
1675         kfree(res);
1676
1677         return copied;
1678 }
1679
1680 #define WRITE_BATCH_PAGES       32
1681
1682 static int __bch2_buffered_write(struct bch_inode_info *inode,
1683                                  struct address_space *mapping,
1684                                  struct iov_iter *iter,
1685                                  loff_t pos, unsigned len)
1686 {
1687         struct bch_fs *c = inode->v.i_sb->s_fs_info;
1688         struct page *pages[WRITE_BATCH_PAGES];
1689         struct bch2_page_reservation res;
1690         unsigned long index = pos >> PAGE_SHIFT;
1691         unsigned offset = pos & (PAGE_SIZE - 1);
1692         unsigned nr_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE);
1693         unsigned i, reserved = 0, set_dirty = 0;
1694         unsigned copied = 0, nr_pages_copied = 0;
1695         int ret = 0;
1696
1697         BUG_ON(!len);
1698         BUG_ON(nr_pages > ARRAY_SIZE(pages));
1699
1700         bch2_page_reservation_init(c, inode, &res);
1701
1702         for (i = 0; i < nr_pages; i++) {
1703                 pages[i] = grab_cache_page_write_begin(mapping, index + i);
1704                 if (!pages[i]) {
1705                         nr_pages = i;
1706                         if (!i) {
1707                                 ret = -ENOMEM;
1708                                 goto out;
1709                         }
1710                         len = min_t(unsigned, len,
1711                                     nr_pages * PAGE_SIZE - offset);
1712                         break;
1713                 }
1714         }
1715
1716         if (offset && !PageUptodate(pages[0])) {
1717                 ret = bch2_read_single_page(pages[0], mapping);
1718                 if (ret)
1719                         goto out;
1720         }
1721
1722         if ((pos + len) & (PAGE_SIZE - 1) &&
1723             !PageUptodate(pages[nr_pages - 1])) {
1724                 if ((index + nr_pages - 1) << PAGE_SHIFT >= inode->v.i_size) {
1725                         zero_user(pages[nr_pages - 1], 0, PAGE_SIZE);
1726                 } else {
1727                         ret = bch2_read_single_page(pages[nr_pages - 1], mapping);
1728                         if (ret)
1729                                 goto out;
1730                 }
1731         }
1732
1733         while (reserved < len) {
1734                 unsigned i = (offset + reserved) >> PAGE_SHIFT;
1735                 struct page *page = pages[i];
1736                 unsigned pg_offset = (offset + reserved) & (PAGE_SIZE - 1);
1737                 unsigned pg_len = min_t(unsigned, len - reserved,
1738                                         PAGE_SIZE - pg_offset);
1739
1740                 if (!bch2_page_state_create(page, __GFP_NOFAIL)->uptodate) {
1741                         ret = bch2_page_state_set(c, inode_inum(inode),
1742                                                   pages + i, nr_pages - i);
1743                         if (ret)
1744                                 goto out;
1745                 }
1746
1747                 /*
1748                  * XXX: per POSIX and fstests generic/275, on -ENOSPC we're
1749                  * supposed to write as much as we have disk space for.
1750                  *
1751                  * On failure here we should still write out a partial page if
1752                  * we aren't completely out of disk space - we don't do that
1753                  * yet:
1754                  */
1755                 ret = bch2_page_reservation_get(c, inode, page, &res,
1756                                                 pg_offset, pg_len);
1757                 if (unlikely(ret)) {
1758                         if (!reserved)
1759                                 goto out;
1760                         break;
1761                 }
1762
1763                 reserved += pg_len;
1764         }
1765
1766         if (mapping_writably_mapped(mapping))
1767                 for (i = 0; i < nr_pages; i++)
1768                         flush_dcache_page(pages[i]);
1769
1770         while (copied < reserved) {
1771                 struct page *page = pages[(offset + copied) >> PAGE_SHIFT];
1772                 unsigned pg_offset = (offset + copied) & (PAGE_SIZE - 1);
1773                 unsigned pg_len = min_t(unsigned, reserved - copied,
1774                                         PAGE_SIZE - pg_offset);
1775                 unsigned pg_copied = copy_page_from_iter_atomic(page,
1776                                                 pg_offset, pg_len, iter);
1777
1778                 if (!pg_copied)
1779                         break;
1780
1781                 if (!PageUptodate(page) &&
1782                     pg_copied != PAGE_SIZE &&
1783                     pos + copied + pg_copied < inode->v.i_size) {
1784                         zero_user(page, 0, PAGE_SIZE);
1785                         break;
1786                 }
1787
1788                 flush_dcache_page(page);
1789                 copied += pg_copied;
1790
1791                 if (pg_copied != pg_len)
1792                         break;
1793         }
1794
1795         if (!copied)
1796                 goto out;
1797
1798         spin_lock(&inode->v.i_lock);
1799         if (pos + copied > inode->v.i_size)
1800                 i_size_write(&inode->v, pos + copied);
1801         spin_unlock(&inode->v.i_lock);
1802
1803         while (set_dirty < copied) {
1804                 struct page *page = pages[(offset + set_dirty) >> PAGE_SHIFT];
1805                 unsigned pg_offset = (offset + set_dirty) & (PAGE_SIZE - 1);
1806                 unsigned pg_len = min_t(unsigned, copied - set_dirty,
1807                                         PAGE_SIZE - pg_offset);
1808
1809                 if (!PageUptodate(page))
1810                         SetPageUptodate(page);
1811
1812                 bch2_set_page_dirty(c, inode, page, &res, pg_offset, pg_len);
1813                 unlock_page(page);
1814                 put_page(page);
1815
1816                 set_dirty += pg_len;
1817         }
1818
1819         nr_pages_copied = DIV_ROUND_UP(offset + copied, PAGE_SIZE);
1820         inode->ei_last_dirtied = (unsigned long) current;
1821 out:
1822         for (i = nr_pages_copied; i < nr_pages; i++) {
1823                 unlock_page(pages[i]);
1824                 put_page(pages[i]);
1825         }
1826
1827         bch2_page_reservation_put(c, inode, &res);
1828
1829         return copied ?: ret;
1830 }
1831
1832 static ssize_t bch2_buffered_write(struct kiocb *iocb, struct iov_iter *iter)
1833 {
1834         struct file *file = iocb->ki_filp;
1835         struct address_space *mapping = file->f_mapping;
1836         struct bch_inode_info *inode = file_bch_inode(file);
1837         loff_t pos = iocb->ki_pos;
1838         ssize_t written = 0;
1839         int ret = 0;
1840
1841         bch2_pagecache_add_get(inode);
1842
1843         do {
1844                 unsigned offset = pos & (PAGE_SIZE - 1);
1845                 unsigned bytes = min_t(unsigned long, iov_iter_count(iter),
1846                               PAGE_SIZE * WRITE_BATCH_PAGES - offset);
1847 again:
1848                 /*
1849                  * Bring in the user page that we will copy from _first_.
1850                  * Otherwise there's a nasty deadlock on copying from the
1851                  * same page as we're writing to, without it being marked
1852                  * up-to-date.
1853                  *
1854                  * Not only is this an optimisation, but it is also required
1855                  * to check that the address is actually valid, when atomic
1856                  * usercopies are used, below.
1857                  */
1858                 if (unlikely(fault_in_iov_iter_readable(iter, bytes))) {
1859                         bytes = min_t(unsigned long, iov_iter_count(iter),
1860                                       PAGE_SIZE - offset);
1861
1862                         if (unlikely(fault_in_iov_iter_readable(iter, bytes))) {
1863                                 ret = -EFAULT;
1864                                 break;
1865                         }
1866                 }
1867
1868                 if (unlikely(fatal_signal_pending(current))) {
1869                         ret = -EINTR;
1870                         break;
1871                 }
1872
1873                 ret = __bch2_buffered_write(inode, mapping, iter, pos, bytes);
1874                 if (unlikely(ret < 0))
1875                         break;
1876
1877                 cond_resched();
1878
1879                 if (unlikely(ret == 0)) {
1880                         /*
1881                          * If we were unable to copy any data at all, we must
1882                          * fall back to a single segment length write.
1883                          *
1884                          * If we didn't fallback here, we could livelock
1885                          * because not all segments in the iov can be copied at
1886                          * once without a pagefault.
1887                          */
1888                         bytes = min_t(unsigned long, PAGE_SIZE - offset,
1889                                       iov_iter_single_seg_count(iter));
1890                         goto again;
1891                 }
1892                 pos += ret;
1893                 written += ret;
1894                 ret = 0;
1895
1896                 balance_dirty_pages_ratelimited(mapping);
1897         } while (iov_iter_count(iter));
1898
1899         bch2_pagecache_add_put(inode);
1900
1901         return written ? written : ret;
1902 }
1903
1904 /* O_DIRECT reads */
1905
1906 static void bio_check_or_release(struct bio *bio, bool check_dirty)
1907 {
1908         if (check_dirty) {
1909                 bio_check_pages_dirty(bio);
1910         } else {
1911                 bio_release_pages(bio, false);
1912                 bio_put(bio);
1913         }
1914 }
1915
1916 static void bch2_dio_read_complete(struct closure *cl)
1917 {
1918         struct dio_read *dio = container_of(cl, struct dio_read, cl);
1919
1920         dio->req->ki_complete(dio->req, dio->ret);
1921         bio_check_or_release(&dio->rbio.bio, dio->should_dirty);
1922 }
1923
1924 static void bch2_direct_IO_read_endio(struct bio *bio)
1925 {
1926         struct dio_read *dio = bio->bi_private;
1927
1928         if (bio->bi_status)
1929                 dio->ret = blk_status_to_errno(bio->bi_status);
1930
1931         closure_put(&dio->cl);
1932 }
1933
1934 static void bch2_direct_IO_read_split_endio(struct bio *bio)
1935 {
1936         struct dio_read *dio = bio->bi_private;
1937         bool should_dirty = dio->should_dirty;
1938
1939         bch2_direct_IO_read_endio(bio);
1940         bio_check_or_release(bio, should_dirty);
1941 }
1942
1943 static int bch2_direct_IO_read(struct kiocb *req, struct iov_iter *iter)
1944 {
1945         struct file *file = req->ki_filp;
1946         struct bch_inode_info *inode = file_bch_inode(file);
1947         struct bch_fs *c = inode->v.i_sb->s_fs_info;
1948         struct bch_io_opts opts = io_opts(c, &inode->ei_inode);
1949         struct dio_read *dio;
1950         struct bio *bio;
1951         loff_t offset = req->ki_pos;
1952         bool sync = is_sync_kiocb(req);
1953         size_t shorten;
1954         ssize_t ret;
1955
1956         if ((offset|iter->count) & (block_bytes(c) - 1))
1957                 return -EINVAL;
1958
1959         ret = min_t(loff_t, iter->count,
1960                     max_t(loff_t, 0, i_size_read(&inode->v) - offset));
1961
1962         if (!ret)
1963                 return ret;
1964
1965         shorten = iov_iter_count(iter) - round_up(ret, block_bytes(c));
1966         iter->count -= shorten;
1967
1968         bio = bio_alloc_bioset(NULL,
1969                                bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS),
1970                                REQ_OP_READ,
1971                                GFP_KERNEL,
1972                                &c->dio_read_bioset);
1973
1974         bio->bi_end_io = bch2_direct_IO_read_endio;
1975
1976         dio = container_of(bio, struct dio_read, rbio.bio);
1977         closure_init(&dio->cl, NULL);
1978
1979         /*
1980          * this is a _really_ horrible hack just to avoid an atomic sub at the
1981          * end:
1982          */
1983         if (!sync) {
1984                 set_closure_fn(&dio->cl, bch2_dio_read_complete, NULL);
1985                 atomic_set(&dio->cl.remaining,
1986                            CLOSURE_REMAINING_INITIALIZER -
1987                            CLOSURE_RUNNING +
1988                            CLOSURE_DESTRUCTOR);
1989         } else {
1990                 atomic_set(&dio->cl.remaining,
1991                            CLOSURE_REMAINING_INITIALIZER + 1);
1992         }
1993
1994         dio->req        = req;
1995         dio->ret        = ret;
1996         /*
1997          * This is one of the sketchier things I've encountered: we have to skip
1998          * the dirtying of requests that are internal from the kernel (i.e. from
1999          * loopback), because we'll deadlock on page_lock.
2000          */
2001         dio->should_dirty = iter_is_iovec(iter);
2002
2003         goto start;
2004         while (iter->count) {
2005                 bio = bio_alloc_bioset(NULL,
2006                                        bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS),
2007                                        REQ_OP_READ,
2008                                        GFP_KERNEL,
2009                                        &c->bio_read);
2010                 bio->bi_end_io          = bch2_direct_IO_read_split_endio;
2011 start:
2012                 bio_set_op_attrs(bio, REQ_OP_READ, REQ_SYNC);
2013                 bio->bi_iter.bi_sector  = offset >> 9;
2014                 bio->bi_private         = dio;
2015
2016                 ret = bio_iov_iter_get_pages(bio, iter);
2017                 if (ret < 0) {
2018                         /* XXX: fault inject this path */
2019                         bio->bi_status = BLK_STS_RESOURCE;
2020                         bio_endio(bio);
2021                         break;
2022                 }
2023
2024                 offset += bio->bi_iter.bi_size;
2025
2026                 if (dio->should_dirty)
2027                         bio_set_pages_dirty(bio);
2028
2029                 if (iter->count)
2030                         closure_get(&dio->cl);
2031
2032                 bch2_read(c, rbio_init(bio, opts), inode_inum(inode));
2033         }
2034
2035         iter->count += shorten;
2036
2037         if (sync) {
2038                 closure_sync(&dio->cl);
2039                 closure_debug_destroy(&dio->cl);
2040                 ret = dio->ret;
2041                 bio_check_or_release(&dio->rbio.bio, dio->should_dirty);
2042                 return ret;
2043         } else {
2044                 return -EIOCBQUEUED;
2045         }
2046 }
2047
2048 ssize_t bch2_read_iter(struct kiocb *iocb, struct iov_iter *iter)
2049 {
2050         struct file *file = iocb->ki_filp;
2051         struct bch_inode_info *inode = file_bch_inode(file);
2052         struct address_space *mapping = file->f_mapping;
2053         size_t count = iov_iter_count(iter);
2054         ssize_t ret;
2055
2056         if (!count)
2057                 return 0; /* skip atime */
2058
2059         if (iocb->ki_flags & IOCB_DIRECT) {
2060                 struct blk_plug plug;
2061
2062                 if (unlikely(mapping->nrpages)) {
2063                         ret = filemap_write_and_wait_range(mapping,
2064                                                 iocb->ki_pos,
2065                                                 iocb->ki_pos + count - 1);
2066                         if (ret < 0)
2067                                 goto out;
2068                 }
2069
2070                 file_accessed(file);
2071
2072                 blk_start_plug(&plug);
2073                 ret = bch2_direct_IO_read(iocb, iter);
2074                 blk_finish_plug(&plug);
2075
2076                 if (ret >= 0)
2077                         iocb->ki_pos += ret;
2078         } else {
2079                 bch2_pagecache_add_get(inode);
2080                 ret = generic_file_read_iter(iocb, iter);
2081                 bch2_pagecache_add_put(inode);
2082         }
2083 out:
2084         return bch2_err_class(ret);
2085 }
2086
2087 /* O_DIRECT writes */
2088
2089 static bool bch2_check_range_allocated(struct bch_fs *c, subvol_inum inum,
2090                                        u64 offset, u64 size,
2091                                        unsigned nr_replicas, bool compressed)
2092 {
2093         struct btree_trans trans;
2094         struct btree_iter iter;
2095         struct bkey_s_c k;
2096         u64 end = offset + size;
2097         u32 snapshot;
2098         bool ret = true;
2099         int err;
2100
2101         bch2_trans_init(&trans, c, 0, 0);
2102 retry:
2103         bch2_trans_begin(&trans);
2104
2105         err = bch2_subvolume_get_snapshot(&trans, inum.subvol, &snapshot);
2106         if (err)
2107                 goto err;
2108
2109         for_each_btree_key_norestart(&trans, iter, BTREE_ID_extents,
2110                            SPOS(inum.inum, offset, snapshot),
2111                            BTREE_ITER_SLOTS, k, err) {
2112                 if (bkey_cmp(bkey_start_pos(k.k), POS(inum.inum, end)) >= 0)
2113                         break;
2114
2115                 if (k.k->p.snapshot != snapshot ||
2116                     nr_replicas > bch2_bkey_replicas(c, k) ||
2117                     (!compressed && bch2_bkey_sectors_compressed(k))) {
2118                         ret = false;
2119                         break;
2120                 }
2121         }
2122
2123         offset = iter.pos.offset;
2124         bch2_trans_iter_exit(&trans, &iter);
2125 err:
2126         if (bch2_err_matches(err, BCH_ERR_transaction_restart))
2127                 goto retry;
2128         bch2_trans_exit(&trans);
2129
2130         return err ? false : ret;
2131 }
2132
2133 static noinline bool bch2_dio_write_check_allocated(struct dio_write *dio)
2134 {
2135         struct bch_fs *c = dio->op.c;
2136         struct bch_inode_info *inode = dio->inode;
2137         struct bio *bio = &dio->op.wbio.bio;
2138
2139         return bch2_check_range_allocated(c, inode_inum(inode),
2140                                 dio->op.pos.offset, bio_sectors(bio),
2141                                 dio->op.opts.data_replicas,
2142                                 dio->op.opts.compression != 0);
2143 }
2144
2145 static void bch2_dio_write_loop_async(struct bch_write_op *);
2146 static __always_inline long bch2_dio_write_done(struct dio_write *dio);
2147
2148 static noinline int bch2_dio_write_copy_iov(struct dio_write *dio)
2149 {
2150         struct iovec *iov = dio->inline_vecs;
2151
2152         if (dio->iter.nr_segs > ARRAY_SIZE(dio->inline_vecs)) {
2153                 iov = kmalloc_array(dio->iter.nr_segs, sizeof(*iov),
2154                                     GFP_KERNEL);
2155                 if (unlikely(!iov))
2156                         return -ENOMEM;
2157
2158                 dio->free_iov = true;
2159         }
2160
2161         memcpy(iov, dio->iter.iov, dio->iter.nr_segs * sizeof(*iov));
2162         dio->iter.iov = iov;
2163         return 0;
2164 }
2165
2166 static void bch2_dio_write_flush_done(struct closure *cl)
2167 {
2168         struct dio_write *dio = container_of(cl, struct dio_write, op.cl);
2169         struct bch_fs *c = dio->op.c;
2170
2171         closure_debug_destroy(cl);
2172
2173         dio->op.error = bch2_journal_error(&c->journal);
2174
2175         bch2_dio_write_done(dio);
2176 }
2177
2178 static noinline void bch2_dio_write_flush(struct dio_write *dio)
2179 {
2180         struct bch_fs *c = dio->op.c;
2181         struct bch_inode_unpacked inode;
2182         int ret;
2183
2184         dio->flush = 0;
2185
2186         closure_init(&dio->op.cl, NULL);
2187
2188         if (!dio->op.error) {
2189                 ret = bch2_inode_find_by_inum(c, inode_inum(dio->inode), &inode);
2190                 if (ret) {
2191                         dio->op.error = ret;
2192                 } else {
2193                         bch2_journal_flush_seq_async(&c->journal, inode.bi_journal_seq, &dio->op.cl);
2194                         bch2_inode_flush_nocow_writes_async(c, dio->inode, &dio->op.cl);
2195                 }
2196         }
2197
2198         if (dio->sync) {
2199                 closure_sync(&dio->op.cl);
2200                 closure_debug_destroy(&dio->op.cl);
2201         } else {
2202                 continue_at(&dio->op.cl, bch2_dio_write_flush_done, NULL);
2203         }
2204 }
2205
2206 static __always_inline long bch2_dio_write_done(struct dio_write *dio)
2207 {
2208         struct kiocb *req = dio->req;
2209         struct bch_inode_info *inode = dio->inode;
2210         bool sync = dio->sync;
2211         long ret;
2212
2213         if (unlikely(dio->flush)) {
2214                 bch2_dio_write_flush(dio);
2215                 if (!sync)
2216                         return -EIOCBQUEUED;
2217         }
2218
2219         bch2_pagecache_block_put(inode);
2220
2221         if (dio->free_iov)
2222                 kfree(dio->iter.iov);
2223
2224         ret = dio->op.error ?: ((long) dio->written << 9);
2225         bio_put(&dio->op.wbio.bio);
2226
2227         /* inode->i_dio_count is our ref on inode and thus bch_fs */
2228         inode_dio_end(&inode->v);
2229
2230         if (ret < 0)
2231                 ret = bch2_err_class(ret);
2232
2233         if (!sync) {
2234                 req->ki_complete(req, ret);
2235                 ret = -EIOCBQUEUED;
2236         }
2237         return ret;
2238 }
2239
2240 static __always_inline void bch2_dio_write_end(struct dio_write *dio)
2241 {
2242         struct bch_fs *c = dio->op.c;
2243         struct kiocb *req = dio->req;
2244         struct bch_inode_info *inode = dio->inode;
2245         struct bio *bio = &dio->op.wbio.bio;
2246         struct bvec_iter_all iter;
2247         struct bio_vec *bv;
2248
2249         req->ki_pos     += (u64) dio->op.written << 9;
2250         dio->written    += dio->op.written;
2251
2252         if (dio->extending) {
2253                 spin_lock(&inode->v.i_lock);
2254                 if (req->ki_pos > inode->v.i_size)
2255                         i_size_write(&inode->v, req->ki_pos);
2256                 spin_unlock(&inode->v.i_lock);
2257         }
2258
2259         if (dio->op.i_sectors_delta || dio->quota_res.sectors) {
2260                 mutex_lock(&inode->ei_quota_lock);
2261                 __i_sectors_acct(c, inode, &dio->quota_res, dio->op.i_sectors_delta);
2262                 __bch2_quota_reservation_put(c, inode, &dio->quota_res);
2263                 mutex_unlock(&inode->ei_quota_lock);
2264         }
2265
2266         if (likely(!bio_flagged(bio, BIO_NO_PAGE_REF)))
2267                 bio_for_each_segment_all(bv, bio, iter)
2268                         put_page(bv->bv_page);
2269
2270         if (unlikely(dio->op.error))
2271                 set_bit(EI_INODE_ERROR, &inode->ei_flags);
2272 }
2273
2274 static long bch2_dio_write_loop(struct dio_write *dio)
2275 {
2276         struct bch_fs *c = dio->op.c;
2277         struct kiocb *req = dio->req;
2278         struct address_space *mapping = dio->mapping;
2279         struct bch_inode_info *inode = dio->inode;
2280         struct bio *bio = &dio->op.wbio.bio;
2281         unsigned unaligned, iter_count;
2282         bool sync = dio->sync, dropped_locks;
2283         long ret;
2284
2285         while (1) {
2286                 iter_count = dio->iter.count;
2287
2288                 EBUG_ON(current->faults_disabled_mapping);
2289                 current->faults_disabled_mapping = mapping;
2290
2291                 ret = bio_iov_iter_get_pages(bio, &dio->iter);
2292
2293                 dropped_locks = fdm_dropped_locks();
2294
2295                 current->faults_disabled_mapping = NULL;
2296
2297                 /*
2298                  * If the fault handler returned an error but also signalled
2299                  * that it dropped & retook ei_pagecache_lock, we just need to
2300                  * re-shoot down the page cache and retry:
2301                  */
2302                 if (dropped_locks && ret)
2303                         ret = 0;
2304
2305                 if (unlikely(ret < 0))
2306                         goto err;
2307
2308                 if (unlikely(dropped_locks)) {
2309                         ret = write_invalidate_inode_pages_range(mapping,
2310                                         req->ki_pos,
2311                                         req->ki_pos + iter_count - 1);
2312                         if (unlikely(ret))
2313                                 goto err;
2314
2315                         if (!bio->bi_iter.bi_size)
2316                                 continue;
2317                 }
2318
2319                 unaligned = bio->bi_iter.bi_size & (block_bytes(c) - 1);
2320                 bio->bi_iter.bi_size -= unaligned;
2321                 iov_iter_revert(&dio->iter, unaligned);
2322
2323                 if (!bio->bi_iter.bi_size) {
2324                         /*
2325                          * bio_iov_iter_get_pages was only able to get <
2326                          * blocksize worth of pages:
2327                          */
2328                         ret = -EFAULT;
2329                         goto err;
2330                 }
2331
2332                 bch2_write_op_init(&dio->op, c, io_opts(c, &inode->ei_inode));
2333                 dio->op.end_io          = sync
2334                         ? NULL
2335                         : bch2_dio_write_loop_async;
2336                 dio->op.target          = dio->op.opts.foreground_target;
2337                 dio->op.write_point     = writepoint_hashed((unsigned long) current);
2338                 dio->op.nr_replicas     = dio->op.opts.data_replicas;
2339                 dio->op.subvol          = inode->ei_subvol;
2340                 dio->op.pos             = POS(inode->v.i_ino, (u64) req->ki_pos >> 9);
2341                 dio->op.devs_need_flush = &inode->ei_devs_need_flush;
2342
2343                 if (sync)
2344                         dio->op.flags |= BCH_WRITE_SYNC;
2345                 dio->op.flags |= BCH_WRITE_CHECK_ENOSPC;
2346
2347                 ret = bch2_quota_reservation_add(c, inode, &dio->quota_res,
2348                                                  bio_sectors(bio), true);
2349                 if (unlikely(ret))
2350                         goto err;
2351
2352                 ret = bch2_disk_reservation_get(c, &dio->op.res, bio_sectors(bio),
2353                                                 dio->op.opts.data_replicas, 0);
2354                 if (unlikely(ret) &&
2355                     !bch2_dio_write_check_allocated(dio))
2356                         goto err;
2357
2358                 task_io_account_write(bio->bi_iter.bi_size);
2359
2360                 if (unlikely(dio->iter.count) &&
2361                     !dio->sync &&
2362                     !dio->loop &&
2363                     bch2_dio_write_copy_iov(dio))
2364                         dio->sync = sync = true;
2365
2366                 dio->loop = true;
2367                 closure_call(&dio->op.cl, bch2_write, NULL, NULL);
2368
2369                 if (!sync)
2370                         return -EIOCBQUEUED;
2371
2372                 bch2_dio_write_end(dio);
2373
2374                 if (likely(!dio->iter.count) || dio->op.error)
2375                         break;
2376
2377                 bio_reset(bio, NULL, REQ_OP_WRITE);
2378         }
2379 out:
2380         return bch2_dio_write_done(dio);
2381 err:
2382         dio->op.error = ret;
2383
2384         if (!bio_flagged(bio, BIO_NO_PAGE_REF)) {
2385                 struct bvec_iter_all iter;
2386                 struct bio_vec *bv;
2387
2388                 bio_for_each_segment_all(bv, bio, iter)
2389                         put_page(bv->bv_page);
2390         }
2391
2392         bch2_quota_reservation_put(c, inode, &dio->quota_res);
2393         goto out;
2394 }
2395
2396 static void bch2_dio_write_loop_async(struct bch_write_op *op)
2397 {
2398         struct dio_write *dio = container_of(op, struct dio_write, op);
2399         struct mm_struct *mm = dio->mm;
2400
2401         bch2_dio_write_end(dio);
2402
2403         if (likely(!dio->iter.count) || dio->op.error) {
2404                 bch2_dio_write_done(dio);
2405                 return;
2406         }
2407
2408         bio_reset(&dio->op.wbio.bio, NULL, REQ_OP_WRITE);
2409
2410         if (mm)
2411                 kthread_use_mm(mm);
2412         bch2_dio_write_loop(dio);
2413         if (mm)
2414                 kthread_unuse_mm(mm);
2415 }
2416
2417 static noinline
2418 ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter)
2419 {
2420         struct file *file = req->ki_filp;
2421         struct address_space *mapping = file->f_mapping;
2422         struct bch_inode_info *inode = file_bch_inode(file);
2423         struct bch_fs *c = inode->v.i_sb->s_fs_info;
2424         struct dio_write *dio;
2425         struct bio *bio;
2426         bool locked = true, extending;
2427         ssize_t ret;
2428
2429         prefetch(&c->opts);
2430         prefetch((void *) &c->opts + 64);
2431         prefetch(&inode->ei_inode);
2432         prefetch((void *) &inode->ei_inode + 64);
2433
2434         inode_lock(&inode->v);
2435
2436         ret = generic_write_checks(req, iter);
2437         if (unlikely(ret <= 0))
2438                 goto err;
2439
2440         ret = file_remove_privs(file);
2441         if (unlikely(ret))
2442                 goto err;
2443
2444         ret = file_update_time(file);
2445         if (unlikely(ret))
2446                 goto err;
2447
2448         if (unlikely((req->ki_pos|iter->count) & (block_bytes(c) - 1)))
2449                 goto err;
2450
2451         inode_dio_begin(&inode->v);
2452         bch2_pagecache_block_get(inode);
2453
2454         extending = req->ki_pos + iter->count > inode->v.i_size;
2455         if (!extending) {
2456                 inode_unlock(&inode->v);
2457                 locked = false;
2458         }
2459
2460         bio = bio_alloc_bioset(NULL,
2461                                bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS),
2462                                REQ_OP_WRITE,
2463                                GFP_KERNEL,
2464                                &c->dio_write_bioset);
2465         dio = container_of(bio, struct dio_write, op.wbio.bio);
2466         dio->req                = req;
2467         dio->mapping            = mapping;
2468         dio->inode              = inode;
2469         dio->mm                 = current->mm;
2470         dio->loop               = false;
2471         dio->extending          = extending;
2472         dio->sync               = is_sync_kiocb(req) || extending;
2473         dio->flush              = iocb_is_dsync(req) && !c->opts.journal_flush_disabled;
2474         dio->free_iov           = false;
2475         dio->quota_res.sectors  = 0;
2476         dio->written            = 0;
2477         dio->iter               = *iter;
2478         dio->op.c               = c;
2479
2480         if (unlikely(mapping->nrpages)) {
2481                 ret = write_invalidate_inode_pages_range(mapping,
2482                                                 req->ki_pos,
2483                                                 req->ki_pos + iter->count - 1);
2484                 if (unlikely(ret))
2485                         goto err_put_bio;
2486         }
2487
2488         ret = bch2_dio_write_loop(dio);
2489 err:
2490         if (locked)
2491                 inode_unlock(&inode->v);
2492         return ret;
2493 err_put_bio:
2494         bch2_pagecache_block_put(inode);
2495         bio_put(bio);
2496         inode_dio_end(&inode->v);
2497         goto err;
2498 }
2499
2500 ssize_t bch2_write_iter(struct kiocb *iocb, struct iov_iter *from)
2501 {
2502         struct file *file = iocb->ki_filp;
2503         struct bch_inode_info *inode = file_bch_inode(file);
2504         ssize_t ret;
2505
2506         if (iocb->ki_flags & IOCB_DIRECT) {
2507                 ret = bch2_direct_write(iocb, from);
2508                 goto out;
2509         }
2510
2511         /* We can write back this queue in page reclaim */
2512         current->backing_dev_info = inode_to_bdi(&inode->v);
2513         inode_lock(&inode->v);
2514
2515         ret = generic_write_checks(iocb, from);
2516         if (ret <= 0)
2517                 goto unlock;
2518
2519         ret = file_remove_privs(file);
2520         if (ret)
2521                 goto unlock;
2522
2523         ret = file_update_time(file);
2524         if (ret)
2525                 goto unlock;
2526
2527         ret = bch2_buffered_write(iocb, from);
2528         if (likely(ret > 0))
2529                 iocb->ki_pos += ret;
2530 unlock:
2531         inode_unlock(&inode->v);
2532         current->backing_dev_info = NULL;
2533
2534         if (ret > 0)
2535                 ret = generic_write_sync(iocb, ret);
2536 out:
2537         return bch2_err_class(ret);
2538 }
2539
2540 /* fsync: */
2541
2542 /*
2543  * inode->ei_inode.bi_journal_seq won't be up to date since it's set in an
2544  * insert trigger: look up the btree inode instead
2545  */
2546 static int bch2_flush_inode(struct bch_fs *c,
2547                             struct bch_inode_info *inode)
2548 {
2549         struct bch_inode_unpacked u;
2550         int ret;
2551
2552         if (c->opts.journal_flush_disabled)
2553                 return 0;
2554
2555         ret = bch2_inode_find_by_inum(c, inode_inum(inode), &u);
2556         if (ret)
2557                 return ret;
2558
2559         return bch2_journal_flush_seq(&c->journal, u.bi_journal_seq) ?:
2560                 bch2_inode_flush_nocow_writes(c, inode);
2561 }
2562
2563 int bch2_fsync(struct file *file, loff_t start, loff_t end, int datasync)
2564 {
2565         struct bch_inode_info *inode = file_bch_inode(file);
2566         struct bch_fs *c = inode->v.i_sb->s_fs_info;
2567         int ret, ret2, ret3;
2568
2569         ret = file_write_and_wait_range(file, start, end);
2570         ret2 = sync_inode_metadata(&inode->v, 1);
2571         ret3 = bch2_flush_inode(c, inode);
2572
2573         return bch2_err_class(ret ?: ret2 ?: ret3);
2574 }
2575
2576 /* truncate: */
2577
2578 static inline int range_has_data(struct bch_fs *c, u32 subvol,
2579                                  struct bpos start,
2580                                  struct bpos end)
2581 {
2582         struct btree_trans trans;
2583         struct btree_iter iter;
2584         struct bkey_s_c k;
2585         int ret = 0;
2586
2587         bch2_trans_init(&trans, c, 0, 0);
2588 retry:
2589         bch2_trans_begin(&trans);
2590
2591         ret = bch2_subvolume_get_snapshot(&trans, subvol, &start.snapshot);
2592         if (ret)
2593                 goto err;
2594
2595         for_each_btree_key_norestart(&trans, iter, BTREE_ID_extents, start, 0, k, ret) {
2596                 if (bkey_cmp(bkey_start_pos(k.k), end) >= 0)
2597                         break;
2598
2599                 if (bkey_extent_is_data(k.k)) {
2600                         ret = 1;
2601                         break;
2602                 }
2603         }
2604         start = iter.pos;
2605         bch2_trans_iter_exit(&trans, &iter);
2606 err:
2607         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
2608                 goto retry;
2609
2610         bch2_trans_exit(&trans);
2611         return ret;
2612 }
2613
2614 static int __bch2_truncate_page(struct bch_inode_info *inode,
2615                                 pgoff_t index, loff_t start, loff_t end)
2616 {
2617         struct bch_fs *c = inode->v.i_sb->s_fs_info;
2618         struct address_space *mapping = inode->v.i_mapping;
2619         struct bch_page_state *s;
2620         unsigned start_offset = start & (PAGE_SIZE - 1);
2621         unsigned end_offset = ((end - 1) & (PAGE_SIZE - 1)) + 1;
2622         unsigned i;
2623         struct page *page;
2624         s64 i_sectors_delta = 0;
2625         int ret = 0;
2626
2627         /* Page boundary? Nothing to do */
2628         if (!((index == start >> PAGE_SHIFT && start_offset) ||
2629               (index == end >> PAGE_SHIFT && end_offset != PAGE_SIZE)))
2630                 return 0;
2631
2632         /* Above i_size? */
2633         if (index << PAGE_SHIFT >= inode->v.i_size)
2634                 return 0;
2635
2636         page = find_lock_page(mapping, index);
2637         if (!page) {
2638                 /*
2639                  * XXX: we're doing two index lookups when we end up reading the
2640                  * page
2641                  */
2642                 ret = range_has_data(c, inode->ei_subvol,
2643                                 POS(inode->v.i_ino, index << PAGE_SECTORS_SHIFT),
2644                                 POS(inode->v.i_ino, (index + 1) << PAGE_SECTORS_SHIFT));
2645                 if (ret <= 0)
2646                         return ret;
2647
2648                 page = find_or_create_page(mapping, index, GFP_KERNEL);
2649                 if (unlikely(!page)) {
2650                         ret = -ENOMEM;
2651                         goto out;
2652                 }
2653         }
2654
2655         s = bch2_page_state_create(page, 0);
2656         if (!s) {
2657                 ret = -ENOMEM;
2658                 goto unlock;
2659         }
2660
2661         if (!PageUptodate(page)) {
2662                 ret = bch2_read_single_page(page, mapping);
2663                 if (ret)
2664                         goto unlock;
2665         }
2666
2667         if (index != start >> PAGE_SHIFT)
2668                 start_offset = 0;
2669         if (index != end >> PAGE_SHIFT)
2670                 end_offset = PAGE_SIZE;
2671
2672         for (i = round_up(start_offset, block_bytes(c)) >> 9;
2673              i < round_down(end_offset, block_bytes(c)) >> 9;
2674              i++) {
2675                 s->s[i].nr_replicas     = 0;
2676                 if (s->s[i].state == SECTOR_DIRTY)
2677                         i_sectors_delta--;
2678                 s->s[i].state           = SECTOR_UNALLOCATED;
2679         }
2680
2681         i_sectors_acct(c, inode, NULL, i_sectors_delta);
2682
2683         /*
2684          * Caller needs to know whether this page will be written out by
2685          * writeback - doing an i_size update if necessary - or whether it will
2686          * be responsible for the i_size update:
2687          */
2688         ret = s->s[(min_t(u64, inode->v.i_size - (index << PAGE_SHIFT),
2689                           PAGE_SIZE) - 1) >> 9].state >= SECTOR_DIRTY;
2690
2691         zero_user_segment(page, start_offset, end_offset);
2692
2693         /*
2694          * Bit of a hack - we don't want truncate to fail due to -ENOSPC.
2695          *
2696          * XXX: because we aren't currently tracking whether the page has actual
2697          * data in it (vs. just 0s, or only partially written) this wrong. ick.
2698          */
2699         BUG_ON(bch2_get_page_disk_reservation(c, inode, page, false));
2700
2701         /*
2702          * This removes any writeable userspace mappings; we need to force
2703          * .page_mkwrite to be called again before any mmapped writes, to
2704          * redirty the full page:
2705          */
2706         page_mkclean(page);
2707         __set_page_dirty_nobuffers(page);
2708 unlock:
2709         unlock_page(page);
2710         put_page(page);
2711 out:
2712         return ret;
2713 }
2714
2715 static int bch2_truncate_page(struct bch_inode_info *inode, loff_t from)
2716 {
2717         return __bch2_truncate_page(inode, from >> PAGE_SHIFT,
2718                                     from, round_up(from, PAGE_SIZE));
2719 }
2720
2721 static int bch2_truncate_pages(struct bch_inode_info *inode,
2722                                loff_t start, loff_t end)
2723 {
2724         int ret = __bch2_truncate_page(inode, start >> PAGE_SHIFT,
2725                                        start, end);
2726
2727         if (ret >= 0 &&
2728             start >> PAGE_SHIFT != end >> PAGE_SHIFT)
2729                 ret = __bch2_truncate_page(inode,
2730                                            end >> PAGE_SHIFT,
2731                                            start, end);
2732         return ret;
2733 }
2734
2735 static int bch2_extend(struct user_namespace *mnt_userns,
2736                        struct bch_inode_info *inode,
2737                        struct bch_inode_unpacked *inode_u,
2738                        struct iattr *iattr)
2739 {
2740         struct address_space *mapping = inode->v.i_mapping;
2741         int ret;
2742
2743         /*
2744          * sync appends:
2745          *
2746          * this has to be done _before_ extending i_size:
2747          */
2748         ret = filemap_write_and_wait_range(mapping, inode_u->bi_size, S64_MAX);
2749         if (ret)
2750                 return ret;
2751
2752         truncate_setsize(&inode->v, iattr->ia_size);
2753
2754         return bch2_setattr_nonsize(mnt_userns, inode, iattr);
2755 }
2756
2757 static int bch2_truncate_finish_fn(struct bch_inode_info *inode,
2758                                    struct bch_inode_unpacked *bi,
2759                                    void *p)
2760 {
2761         bi->bi_flags &= ~BCH_INODE_I_SIZE_DIRTY;
2762         return 0;
2763 }
2764
2765 static int bch2_truncate_start_fn(struct bch_inode_info *inode,
2766                                   struct bch_inode_unpacked *bi, void *p)
2767 {
2768         u64 *new_i_size = p;
2769
2770         bi->bi_flags |= BCH_INODE_I_SIZE_DIRTY;
2771         bi->bi_size = *new_i_size;
2772         return 0;
2773 }
2774
2775 int bch2_truncate(struct user_namespace *mnt_userns,
2776                   struct bch_inode_info *inode, struct iattr *iattr)
2777 {
2778         struct bch_fs *c = inode->v.i_sb->s_fs_info;
2779         struct address_space *mapping = inode->v.i_mapping;
2780         struct bch_inode_unpacked inode_u;
2781         u64 new_i_size = iattr->ia_size;
2782         s64 i_sectors_delta = 0;
2783         int ret = 0;
2784
2785         /*
2786          * If the truncate call with change the size of the file, the
2787          * cmtimes should be updated. If the size will not change, we
2788          * do not need to update the cmtimes.
2789          */
2790         if (iattr->ia_size != inode->v.i_size) {
2791                 if (!(iattr->ia_valid & ATTR_MTIME))
2792                         ktime_get_coarse_real_ts64(&iattr->ia_mtime);
2793                 if (!(iattr->ia_valid & ATTR_CTIME))
2794                         ktime_get_coarse_real_ts64(&iattr->ia_ctime);
2795                 iattr->ia_valid |= ATTR_MTIME|ATTR_CTIME;
2796         }
2797
2798         inode_dio_wait(&inode->v);
2799         bch2_pagecache_block_get(inode);
2800
2801         ret = bch2_inode_find_by_inum(c, inode_inum(inode), &inode_u);
2802         if (ret)
2803                 goto err;
2804
2805         /*
2806          * check this before next assertion; on filesystem error our normal
2807          * invariants are a bit broken (truncate has to truncate the page cache
2808          * before the inode).
2809          */
2810         ret = bch2_journal_error(&c->journal);
2811         if (ret)
2812                 goto err;
2813
2814         WARN_ONCE(!test_bit(EI_INODE_ERROR, &inode->ei_flags) &&
2815                   inode->v.i_size < inode_u.bi_size,
2816                   "truncate spotted in mem i_size < btree i_size: %llu < %llu\n",
2817                   (u64) inode->v.i_size, inode_u.bi_size);
2818
2819         if (iattr->ia_size > inode->v.i_size) {
2820                 ret = bch2_extend(mnt_userns, inode, &inode_u, iattr);
2821                 goto err;
2822         }
2823
2824         iattr->ia_valid &= ~ATTR_SIZE;
2825
2826         ret = bch2_truncate_page(inode, iattr->ia_size);
2827         if (unlikely(ret < 0))
2828                 goto err;
2829
2830         /*
2831          * When extending, we're going to write the new i_size to disk
2832          * immediately so we need to flush anything above the current on disk
2833          * i_size first:
2834          *
2835          * Also, when extending we need to flush the page that i_size currently
2836          * straddles - if it's mapped to userspace, we need to ensure that
2837          * userspace has to redirty it and call .mkwrite -> set_page_dirty
2838          * again to allocate the part of the page that was extended.
2839          */
2840         if (iattr->ia_size > inode_u.bi_size)
2841                 ret = filemap_write_and_wait_range(mapping,
2842                                 inode_u.bi_size,
2843                                 iattr->ia_size - 1);
2844         else if (iattr->ia_size & (PAGE_SIZE - 1))
2845                 ret = filemap_write_and_wait_range(mapping,
2846                                 round_down(iattr->ia_size, PAGE_SIZE),
2847                                 iattr->ia_size - 1);
2848         if (ret)
2849                 goto err;
2850
2851         mutex_lock(&inode->ei_update_lock);
2852         ret = bch2_write_inode(c, inode, bch2_truncate_start_fn,
2853                                &new_i_size, 0);
2854         mutex_unlock(&inode->ei_update_lock);
2855
2856         if (unlikely(ret))
2857                 goto err;
2858
2859         truncate_setsize(&inode->v, iattr->ia_size);
2860
2861         ret = bch2_fpunch(c, inode_inum(inode),
2862                         round_up(iattr->ia_size, block_bytes(c)) >> 9,
2863                         U64_MAX, &i_sectors_delta);
2864         i_sectors_acct(c, inode, NULL, i_sectors_delta);
2865
2866         bch2_fs_inconsistent_on(!inode->v.i_size && inode->v.i_blocks &&
2867                                 !bch2_journal_error(&c->journal), c,
2868                                 "inode %lu truncated to 0 but i_blocks %llu (ondisk %lli)",
2869                                 inode->v.i_ino, (u64) inode->v.i_blocks,
2870                                 inode->ei_inode.bi_sectors);
2871         if (unlikely(ret))
2872                 goto err;
2873
2874         mutex_lock(&inode->ei_update_lock);
2875         ret = bch2_write_inode(c, inode, bch2_truncate_finish_fn, NULL, 0);
2876         mutex_unlock(&inode->ei_update_lock);
2877
2878         ret = bch2_setattr_nonsize(mnt_userns, inode, iattr);
2879 err:
2880         bch2_pagecache_block_put(inode);
2881         return bch2_err_class(ret);
2882 }
2883
2884 /* fallocate: */
2885
2886 static int inode_update_times_fn(struct bch_inode_info *inode,
2887                                  struct bch_inode_unpacked *bi, void *p)
2888 {
2889         struct bch_fs *c = inode->v.i_sb->s_fs_info;
2890
2891         bi->bi_mtime = bi->bi_ctime = bch2_current_time(c);
2892         return 0;
2893 }
2894
2895 static long bchfs_fpunch(struct bch_inode_info *inode, loff_t offset, loff_t len)
2896 {
2897         struct bch_fs *c = inode->v.i_sb->s_fs_info;
2898         u64 end         = offset + len;
2899         u64 block_start = round_up(offset, block_bytes(c));
2900         u64 block_end   = round_down(end, block_bytes(c));
2901         bool truncated_last_page;
2902         int ret = 0;
2903
2904         ret = bch2_truncate_pages(inode, offset, end);
2905         if (unlikely(ret < 0))
2906                 goto err;
2907
2908         truncated_last_page = ret;
2909
2910         truncate_pagecache_range(&inode->v, offset, end - 1);
2911
2912         if (block_start < block_end) {
2913                 s64 i_sectors_delta = 0;
2914
2915                 ret = bch2_fpunch(c, inode_inum(inode),
2916                                   block_start >> 9, block_end >> 9,
2917                                   &i_sectors_delta);
2918                 i_sectors_acct(c, inode, NULL, i_sectors_delta);
2919         }
2920
2921         mutex_lock(&inode->ei_update_lock);
2922         if (end >= inode->v.i_size && !truncated_last_page) {
2923                 ret = bch2_write_inode_size(c, inode, inode->v.i_size,
2924                                             ATTR_MTIME|ATTR_CTIME);
2925         } else {
2926                 ret = bch2_write_inode(c, inode, inode_update_times_fn, NULL,
2927                                        ATTR_MTIME|ATTR_CTIME);
2928         }
2929         mutex_unlock(&inode->ei_update_lock);
2930 err:
2931         return ret;
2932 }
2933
2934 static long bchfs_fcollapse_finsert(struct bch_inode_info *inode,
2935                                    loff_t offset, loff_t len,
2936                                    bool insert)
2937 {
2938         struct bch_fs *c = inode->v.i_sb->s_fs_info;
2939         struct address_space *mapping = inode->v.i_mapping;
2940         struct bkey_buf copy;
2941         struct btree_trans trans;
2942         struct btree_iter src, dst, del;
2943         loff_t shift, new_size;
2944         u64 src_start;
2945         int ret = 0;
2946
2947         if ((offset | len) & (block_bytes(c) - 1))
2948                 return -EINVAL;
2949
2950         if (insert) {
2951                 if (inode->v.i_sb->s_maxbytes - inode->v.i_size < len)
2952                         return -EFBIG;
2953
2954                 if (offset >= inode->v.i_size)
2955                         return -EINVAL;
2956
2957                 src_start       = U64_MAX;
2958                 shift           = len;
2959         } else {
2960                 if (offset + len >= inode->v.i_size)
2961                         return -EINVAL;
2962
2963                 src_start       = offset + len;
2964                 shift           = -len;
2965         }
2966
2967         new_size = inode->v.i_size + shift;
2968
2969         ret = write_invalidate_inode_pages_range(mapping, offset, LLONG_MAX);
2970         if (ret)
2971                 return ret;
2972
2973         if (insert) {
2974                 i_size_write(&inode->v, new_size);
2975                 mutex_lock(&inode->ei_update_lock);
2976                 ret = bch2_write_inode_size(c, inode, new_size,
2977                                             ATTR_MTIME|ATTR_CTIME);
2978                 mutex_unlock(&inode->ei_update_lock);
2979         } else {
2980                 s64 i_sectors_delta = 0;
2981
2982                 ret = bch2_fpunch(c, inode_inum(inode),
2983                                   offset >> 9, (offset + len) >> 9,
2984                                   &i_sectors_delta);
2985                 i_sectors_acct(c, inode, NULL, i_sectors_delta);
2986
2987                 if (ret)
2988                         return ret;
2989         }
2990
2991         bch2_bkey_buf_init(&copy);
2992         bch2_trans_init(&trans, c, BTREE_ITER_MAX, 1024);
2993         bch2_trans_iter_init(&trans, &src, BTREE_ID_extents,
2994                         POS(inode->v.i_ino, src_start >> 9),
2995                         BTREE_ITER_INTENT);
2996         bch2_trans_copy_iter(&dst, &src);
2997         bch2_trans_copy_iter(&del, &src);
2998
2999         while (ret == 0 ||
3000                bch2_err_matches(ret, BCH_ERR_transaction_restart)) {
3001                 struct disk_reservation disk_res =
3002                         bch2_disk_reservation_init(c, 0);
3003                 struct bkey_i delete;
3004                 struct bkey_s_c k;
3005                 struct bpos next_pos;
3006                 struct bpos move_pos = POS(inode->v.i_ino, offset >> 9);
3007                 struct bpos atomic_end;
3008                 unsigned trigger_flags = 0;
3009                 u32 snapshot;
3010
3011                 bch2_trans_begin(&trans);
3012
3013                 ret = bch2_subvolume_get_snapshot(&trans,
3014                                         inode->ei_subvol, &snapshot);
3015                 if (ret)
3016                         continue;
3017
3018                 bch2_btree_iter_set_snapshot(&src, snapshot);
3019                 bch2_btree_iter_set_snapshot(&dst, snapshot);
3020                 bch2_btree_iter_set_snapshot(&del, snapshot);
3021
3022                 bch2_trans_begin(&trans);
3023
3024                 k = insert
3025                         ? bch2_btree_iter_peek_prev(&src)
3026                         : bch2_btree_iter_peek(&src);
3027                 if ((ret = bkey_err(k)))
3028                         continue;
3029
3030                 if (!k.k || k.k->p.inode != inode->v.i_ino)
3031                         break;
3032
3033                 if (insert &&
3034                     bkey_cmp(k.k->p, POS(inode->v.i_ino, offset >> 9)) <= 0)
3035                         break;
3036 reassemble:
3037                 bch2_bkey_buf_reassemble(&copy, c, k);
3038
3039                 if (insert &&
3040                     bkey_cmp(bkey_start_pos(k.k), move_pos) < 0)
3041                         bch2_cut_front(move_pos, copy.k);
3042
3043                 copy.k->k.p.offset += shift >> 9;
3044                 bch2_btree_iter_set_pos(&dst, bkey_start_pos(&copy.k->k));
3045
3046                 ret = bch2_extent_atomic_end(&trans, &dst, copy.k, &atomic_end);
3047                 if (ret)
3048                         continue;
3049
3050                 if (bkey_cmp(atomic_end, copy.k->k.p)) {
3051                         if (insert) {
3052                                 move_pos = atomic_end;
3053                                 move_pos.offset -= shift >> 9;
3054                                 goto reassemble;
3055                         } else {
3056                                 bch2_cut_back(atomic_end, copy.k);
3057                         }
3058                 }
3059
3060                 bkey_init(&delete.k);
3061                 delete.k.p = copy.k->k.p;
3062                 delete.k.size = copy.k->k.size;
3063                 delete.k.p.offset -= shift >> 9;
3064                 bch2_btree_iter_set_pos(&del, bkey_start_pos(&delete.k));
3065
3066                 next_pos = insert ? bkey_start_pos(&delete.k) : delete.k.p;
3067
3068                 if (copy.k->k.size != k.k->size) {
3069                         /* We might end up splitting compressed extents: */
3070                         unsigned nr_ptrs =
3071                                 bch2_bkey_nr_ptrs_allocated(bkey_i_to_s_c(copy.k));
3072
3073                         ret = bch2_disk_reservation_get(c, &disk_res,
3074                                         copy.k->k.size, nr_ptrs,
3075                                         BCH_DISK_RESERVATION_NOFAIL);
3076                         BUG_ON(ret);
3077                 }
3078
3079                 ret =   bch2_btree_iter_traverse(&del) ?:
3080                         bch2_trans_update(&trans, &del, &delete, trigger_flags) ?:
3081                         bch2_trans_update(&trans, &dst, copy.k, trigger_flags) ?:
3082                         bch2_trans_commit(&trans, &disk_res, NULL,
3083                                           BTREE_INSERT_NOFAIL);
3084                 bch2_disk_reservation_put(c, &disk_res);
3085
3086                 if (!ret)
3087                         bch2_btree_iter_set_pos(&src, next_pos);
3088         }
3089         bch2_trans_iter_exit(&trans, &del);
3090         bch2_trans_iter_exit(&trans, &dst);
3091         bch2_trans_iter_exit(&trans, &src);
3092         bch2_trans_exit(&trans);
3093         bch2_bkey_buf_exit(&copy, c);
3094
3095         if (ret)
3096                 return ret;
3097
3098         mutex_lock(&inode->ei_update_lock);
3099         if (!insert) {
3100                 i_size_write(&inode->v, new_size);
3101                 ret = bch2_write_inode_size(c, inode, new_size,
3102                                             ATTR_MTIME|ATTR_CTIME);
3103         } else {
3104                 /* We need an inode update to update bi_journal_seq for fsync: */
3105                 ret = bch2_write_inode(c, inode, inode_update_times_fn, NULL,
3106                                        ATTR_MTIME|ATTR_CTIME);
3107         }
3108         mutex_unlock(&inode->ei_update_lock);
3109         return ret;
3110 }
3111
3112 static int __bchfs_fallocate(struct bch_inode_info *inode, int mode,
3113                              u64 start_sector, u64 end_sector)
3114 {
3115         struct bch_fs *c = inode->v.i_sb->s_fs_info;
3116         struct btree_trans trans;
3117         struct btree_iter iter;
3118         struct bpos end_pos = POS(inode->v.i_ino, end_sector);
3119         struct bch_io_opts opts = io_opts(c, &inode->ei_inode);
3120         int ret = 0;
3121
3122         bch2_trans_init(&trans, c, BTREE_ITER_MAX, 512);
3123
3124         bch2_trans_iter_init(&trans, &iter, BTREE_ID_extents,
3125                         POS(inode->v.i_ino, start_sector),
3126                         BTREE_ITER_SLOTS|BTREE_ITER_INTENT);
3127
3128         while (!ret && bkey_cmp(iter.pos, end_pos) < 0) {
3129                 s64 i_sectors_delta = 0;
3130                 struct quota_res quota_res = { 0 };
3131                 struct bkey_s_c k;
3132                 unsigned sectors;
3133                 u32 snapshot;
3134
3135                 bch2_trans_begin(&trans);
3136
3137                 ret = bch2_subvolume_get_snapshot(&trans,
3138                                         inode->ei_subvol, &snapshot);
3139                 if (ret)
3140                         goto bkey_err;
3141
3142                 bch2_btree_iter_set_snapshot(&iter, snapshot);
3143
3144                 k = bch2_btree_iter_peek_slot(&iter);
3145                 if ((ret = bkey_err(k)))
3146                         goto bkey_err;
3147
3148                 /* already reserved */
3149                 if (bkey_extent_is_reservation(k) &&
3150                     bch2_bkey_nr_ptrs_fully_allocated(k) >= opts.data_replicas) {
3151                         bch2_btree_iter_advance(&iter);
3152                         continue;
3153                 }
3154
3155                 if (bkey_extent_is_data(k.k) &&
3156                     !(mode & FALLOC_FL_ZERO_RANGE)) {
3157                         bch2_btree_iter_advance(&iter);
3158                         continue;
3159                 }
3160
3161                 /*
3162                  * XXX: for nocow mode, we should promote shared extents to
3163                  * unshared here
3164                  */
3165
3166                 sectors = bpos_min(k.k->p, end_pos).offset - iter.pos.offset;
3167
3168                 if (!bkey_extent_is_allocation(k.k)) {
3169                         ret = bch2_quota_reservation_add(c, inode,
3170                                         &quota_res,
3171                                         sectors, true);
3172                         if (unlikely(ret))
3173                                 goto bkey_err;
3174                 }
3175
3176                 ret = bch2_extent_fallocate(&trans, inode_inum(inode), &iter,
3177                                             sectors, opts, &i_sectors_delta,
3178                                             writepoint_hashed((unsigned long) current));
3179                 if (ret)
3180                         goto bkey_err;
3181
3182                 i_sectors_acct(c, inode, &quota_res, i_sectors_delta);
3183 bkey_err:
3184                 bch2_quota_reservation_put(c, inode, &quota_res);
3185                 if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
3186                         ret = 0;
3187         }
3188
3189         bch2_trans_unlock(&trans); /* lock ordering, before taking pagecache locks: */
3190         mark_pagecache_reserved(inode, start_sector, iter.pos.offset);
3191
3192         if (bch2_err_matches(ret, ENOSPC) && (mode & FALLOC_FL_ZERO_RANGE)) {
3193                 struct quota_res quota_res = { 0 };
3194                 s64 i_sectors_delta = 0;
3195
3196                 bch2_fpunch_at(&trans, &iter, inode_inum(inode),
3197                                end_sector, &i_sectors_delta);
3198                 i_sectors_acct(c, inode, &quota_res, i_sectors_delta);
3199                 bch2_quota_reservation_put(c, inode, &quota_res);
3200         }
3201
3202         bch2_trans_iter_exit(&trans, &iter);
3203         bch2_trans_exit(&trans);
3204         return ret;
3205 }
3206
3207 static long bchfs_fallocate(struct bch_inode_info *inode, int mode,
3208                             loff_t offset, loff_t len)
3209 {
3210         struct bch_fs *c = inode->v.i_sb->s_fs_info;
3211         u64 end         = offset + len;
3212         u64 block_start = round_down(offset,    block_bytes(c));
3213         u64 block_end   = round_up(end,         block_bytes(c));
3214         bool truncated_last_page = false;
3215         int ret, ret2 = 0;
3216
3217         if (!(mode & FALLOC_FL_KEEP_SIZE) && end > inode->v.i_size) {
3218                 ret = inode_newsize_ok(&inode->v, end);
3219                 if (ret)
3220                         return ret;
3221         }
3222
3223         if (mode & FALLOC_FL_ZERO_RANGE) {
3224                 ret = bch2_truncate_pages(inode, offset, end);
3225                 if (unlikely(ret < 0))
3226                         return ret;
3227
3228                 truncated_last_page = ret;
3229
3230                 truncate_pagecache_range(&inode->v, offset, end - 1);
3231
3232                 block_start     = round_up(offset,      block_bytes(c));
3233                 block_end       = round_down(end,       block_bytes(c));
3234         }
3235
3236         ret = __bchfs_fallocate(inode, mode, block_start >> 9, block_end >> 9);
3237
3238         /*
3239          * On -ENOSPC in ZERO_RANGE mode, we still want to do the inode update,
3240          * so that the VFS cache i_size is consistent with the btree i_size:
3241          */
3242         if (ret &&
3243             !(bch2_err_matches(ret, ENOSPC) && (mode & FALLOC_FL_ZERO_RANGE)))
3244                 return ret;
3245
3246         if (mode & FALLOC_FL_KEEP_SIZE && end > inode->v.i_size)
3247                 end = inode->v.i_size;
3248
3249         if (end >= inode->v.i_size &&
3250             (((mode & FALLOC_FL_ZERO_RANGE) && !truncated_last_page) ||
3251              !(mode & FALLOC_FL_KEEP_SIZE))) {
3252                 spin_lock(&inode->v.i_lock);
3253                 i_size_write(&inode->v, end);
3254                 spin_unlock(&inode->v.i_lock);
3255
3256                 mutex_lock(&inode->ei_update_lock);
3257                 ret2 = bch2_write_inode_size(c, inode, end, 0);
3258                 mutex_unlock(&inode->ei_update_lock);
3259         }
3260
3261         return ret ?: ret2;
3262 }
3263
3264 long bch2_fallocate_dispatch(struct file *file, int mode,
3265                              loff_t offset, loff_t len)
3266 {
3267         struct bch_inode_info *inode = file_bch_inode(file);
3268         struct bch_fs *c = inode->v.i_sb->s_fs_info;
3269         long ret;
3270
3271         if (!percpu_ref_tryget_live(&c->writes))
3272                 return -EROFS;
3273
3274         inode_lock(&inode->v);
3275         inode_dio_wait(&inode->v);
3276         bch2_pagecache_block_get(inode);
3277
3278         ret = file_modified(file);
3279         if (ret)
3280                 goto err;
3281
3282         if (!(mode & ~(FALLOC_FL_KEEP_SIZE|FALLOC_FL_ZERO_RANGE)))
3283                 ret = bchfs_fallocate(inode, mode, offset, len);
3284         else if (mode == (FALLOC_FL_PUNCH_HOLE|FALLOC_FL_KEEP_SIZE))
3285                 ret = bchfs_fpunch(inode, offset, len);
3286         else if (mode == FALLOC_FL_INSERT_RANGE)
3287                 ret = bchfs_fcollapse_finsert(inode, offset, len, true);
3288         else if (mode == FALLOC_FL_COLLAPSE_RANGE)
3289                 ret = bchfs_fcollapse_finsert(inode, offset, len, false);
3290         else
3291                 ret = -EOPNOTSUPP;
3292 err:
3293         bch2_pagecache_block_put(inode);
3294         inode_unlock(&inode->v);
3295         percpu_ref_put(&c->writes);
3296
3297         return bch2_err_class(ret);
3298 }
3299
3300 static int quota_reserve_range(struct bch_inode_info *inode,
3301                                struct quota_res *res,
3302                                u64 start, u64 end)
3303 {
3304         struct bch_fs *c = inode->v.i_sb->s_fs_info;
3305         struct btree_trans trans;
3306         struct btree_iter iter;
3307         struct bkey_s_c k;
3308         u32 snapshot;
3309         u64 sectors = end - start;
3310         u64 pos = start;
3311         int ret;
3312
3313         bch2_trans_init(&trans, c, 0, 0);
3314 retry:
3315         bch2_trans_begin(&trans);
3316
3317         ret = bch2_subvolume_get_snapshot(&trans, inode->ei_subvol, &snapshot);
3318         if (ret)
3319                 goto err;
3320
3321         bch2_trans_iter_init(&trans, &iter, BTREE_ID_extents,
3322                              SPOS(inode->v.i_ino, pos, snapshot), 0);
3323
3324         while (!(ret = btree_trans_too_many_iters(&trans)) &&
3325                (k = bch2_btree_iter_peek_upto(&iter, POS(inode->v.i_ino, end - 1))).k &&
3326                !(ret = bkey_err(k))) {
3327                 if (bkey_extent_is_allocation(k.k)) {
3328                         u64 s = min(end, k.k->p.offset) -
3329                                 max(start, bkey_start_offset(k.k));
3330                         BUG_ON(s > sectors);
3331                         sectors -= s;
3332                 }
3333                 bch2_btree_iter_advance(&iter);
3334         }
3335         pos = iter.pos.offset;
3336         bch2_trans_iter_exit(&trans, &iter);
3337 err:
3338         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
3339                 goto retry;
3340
3341         bch2_trans_exit(&trans);
3342
3343         if (ret)
3344                 return ret;
3345
3346         return bch2_quota_reservation_add(c, inode, res, sectors, true);
3347 }
3348
3349 loff_t bch2_remap_file_range(struct file *file_src, loff_t pos_src,
3350                              struct file *file_dst, loff_t pos_dst,
3351                              loff_t len, unsigned remap_flags)
3352 {
3353         struct bch_inode_info *src = file_bch_inode(file_src);
3354         struct bch_inode_info *dst = file_bch_inode(file_dst);
3355         struct bch_fs *c = src->v.i_sb->s_fs_info;
3356         struct quota_res quota_res = { 0 };
3357         s64 i_sectors_delta = 0;
3358         u64 aligned_len;
3359         loff_t ret = 0;
3360
3361         if (remap_flags & ~(REMAP_FILE_DEDUP|REMAP_FILE_ADVISORY))
3362                 return -EINVAL;
3363
3364         if (remap_flags & REMAP_FILE_DEDUP)
3365                 return -EOPNOTSUPP;
3366
3367         if ((pos_src & (block_bytes(c) - 1)) ||
3368             (pos_dst & (block_bytes(c) - 1)))
3369                 return -EINVAL;
3370
3371         if (src == dst &&
3372             abs(pos_src - pos_dst) < len)
3373                 return -EINVAL;
3374
3375         bch2_lock_inodes(INODE_LOCK|INODE_PAGECACHE_BLOCK, src, dst);
3376
3377         inode_dio_wait(&src->v);
3378         inode_dio_wait(&dst->v);
3379
3380         ret = generic_remap_file_range_prep(file_src, pos_src,
3381                                             file_dst, pos_dst,
3382                                             &len, remap_flags);
3383         if (ret < 0 || len == 0)
3384                 goto err;
3385
3386         aligned_len = round_up((u64) len, block_bytes(c));
3387
3388         ret = write_invalidate_inode_pages_range(dst->v.i_mapping,
3389                                 pos_dst, pos_dst + len - 1);
3390         if (ret)
3391                 goto err;
3392
3393         ret = quota_reserve_range(dst, &quota_res, pos_dst >> 9,
3394                                   (pos_dst + aligned_len) >> 9);
3395         if (ret)
3396                 goto err;
3397
3398         file_update_time(file_dst);
3399
3400         mark_pagecache_unallocated(src, pos_src >> 9,
3401                                    (pos_src + aligned_len) >> 9);
3402
3403         ret = bch2_remap_range(c,
3404                                inode_inum(dst), pos_dst >> 9,
3405                                inode_inum(src), pos_src >> 9,
3406                                aligned_len >> 9,
3407                                pos_dst + len, &i_sectors_delta);
3408         if (ret < 0)
3409                 goto err;
3410
3411         /*
3412          * due to alignment, we might have remapped slightly more than requsted
3413          */
3414         ret = min((u64) ret << 9, (u64) len);
3415
3416         i_sectors_acct(c, dst, &quota_res, i_sectors_delta);
3417
3418         spin_lock(&dst->v.i_lock);
3419         if (pos_dst + ret > dst->v.i_size)
3420                 i_size_write(&dst->v, pos_dst + ret);
3421         spin_unlock(&dst->v.i_lock);
3422
3423         if ((file_dst->f_flags & (__O_SYNC | O_DSYNC)) ||
3424             IS_SYNC(file_inode(file_dst)))
3425                 ret = bch2_flush_inode(c, dst);
3426 err:
3427         bch2_quota_reservation_put(c, dst, &quota_res);
3428         bch2_unlock_inodes(INODE_LOCK|INODE_PAGECACHE_BLOCK, src, dst);
3429
3430         return bch2_err_class(ret);
3431 }
3432
3433 /* fseek: */
3434
3435 static int page_data_offset(struct page *page, unsigned offset)
3436 {
3437         struct bch_page_state *s = bch2_page_state(page);
3438         unsigned i;
3439
3440         if (s)
3441                 for (i = offset >> 9; i < PAGE_SECTORS; i++)
3442                         if (s->s[i].state >= SECTOR_DIRTY)
3443                                 return i << 9;
3444
3445         return -1;
3446 }
3447
3448 static loff_t bch2_seek_pagecache_data(struct inode *vinode,
3449                                        loff_t start_offset,
3450                                        loff_t end_offset)
3451 {
3452         struct folio_batch fbatch;
3453         pgoff_t start_index     = start_offset >> PAGE_SHIFT;
3454         pgoff_t end_index       = end_offset >> PAGE_SHIFT;
3455         pgoff_t index           = start_index;
3456         unsigned i;
3457         loff_t ret;
3458         int offset;
3459
3460         folio_batch_init(&fbatch);
3461
3462         while (filemap_get_folios(vinode->i_mapping,
3463                                   &index, end_index, &fbatch)) {
3464                 for (i = 0; i < folio_batch_count(&fbatch); i++) {
3465                         struct folio *folio = fbatch.folios[i];
3466
3467                         folio_lock(folio);
3468
3469                         offset = page_data_offset(&folio->page,
3470                                         folio->index == start_index
3471                                         ? start_offset & (PAGE_SIZE - 1)
3472                                         : 0);
3473                         if (offset >= 0) {
3474                                 ret = clamp(((loff_t) folio->index << PAGE_SHIFT) +
3475                                             offset,
3476                                             start_offset, end_offset);
3477                                 folio_unlock(folio);
3478                                 folio_batch_release(&fbatch);
3479                                 return ret;
3480                         }
3481
3482                         folio_unlock(folio);
3483                 }
3484                 folio_batch_release(&fbatch);
3485                 cond_resched();
3486         }
3487
3488         return end_offset;
3489 }
3490
3491 static loff_t bch2_seek_data(struct file *file, u64 offset)
3492 {
3493         struct bch_inode_info *inode = file_bch_inode(file);
3494         struct bch_fs *c = inode->v.i_sb->s_fs_info;
3495         struct btree_trans trans;
3496         struct btree_iter iter;
3497         struct bkey_s_c k;
3498         subvol_inum inum = inode_inum(inode);
3499         u64 isize, next_data = MAX_LFS_FILESIZE;
3500         u32 snapshot;
3501         int ret;
3502
3503         isize = i_size_read(&inode->v);
3504         if (offset >= isize)
3505                 return -ENXIO;
3506
3507         bch2_trans_init(&trans, c, 0, 0);
3508 retry:
3509         bch2_trans_begin(&trans);
3510
3511         ret = bch2_subvolume_get_snapshot(&trans, inum.subvol, &snapshot);
3512         if (ret)
3513                 goto err;
3514
3515         for_each_btree_key_norestart(&trans, iter, BTREE_ID_extents,
3516                            SPOS(inode->v.i_ino, offset >> 9, snapshot), 0, k, ret) {
3517                 if (k.k->p.inode != inode->v.i_ino) {
3518                         break;
3519                 } else if (bkey_extent_is_data(k.k)) {
3520                         next_data = max(offset, bkey_start_offset(k.k) << 9);
3521                         break;
3522                 } else if (k.k->p.offset >> 9 > isize)
3523                         break;
3524         }
3525         bch2_trans_iter_exit(&trans, &iter);
3526 err:
3527         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
3528                 goto retry;
3529
3530         bch2_trans_exit(&trans);
3531         if (ret)
3532                 return ret;
3533
3534         if (next_data > offset)
3535                 next_data = bch2_seek_pagecache_data(&inode->v,
3536                                                      offset, next_data);
3537
3538         if (next_data >= isize)
3539                 return -ENXIO;
3540
3541         return vfs_setpos(file, next_data, MAX_LFS_FILESIZE);
3542 }
3543
3544 static int __page_hole_offset(struct page *page, unsigned offset)
3545 {
3546         struct bch_page_state *s = bch2_page_state(page);
3547         unsigned i;
3548
3549         if (!s)
3550                 return 0;
3551
3552         for (i = offset >> 9; i < PAGE_SECTORS; i++)
3553                 if (s->s[i].state < SECTOR_DIRTY)
3554                         return i << 9;
3555
3556         return -1;
3557 }
3558
3559 static loff_t page_hole_offset(struct address_space *mapping, loff_t offset)
3560 {
3561         pgoff_t index = offset >> PAGE_SHIFT;
3562         struct page *page;
3563         int pg_offset;
3564         loff_t ret = -1;
3565
3566         page = find_lock_page(mapping, index);
3567         if (!page)
3568                 return offset;
3569
3570         pg_offset = __page_hole_offset(page, offset & (PAGE_SIZE - 1));
3571         if (pg_offset >= 0)
3572                 ret = ((loff_t) index << PAGE_SHIFT) + pg_offset;
3573
3574         unlock_page(page);
3575
3576         return ret;
3577 }
3578
3579 static loff_t bch2_seek_pagecache_hole(struct inode *vinode,
3580                                        loff_t start_offset,
3581                                        loff_t end_offset)
3582 {
3583         struct address_space *mapping = vinode->i_mapping;
3584         loff_t offset = start_offset, hole;
3585
3586         while (offset < end_offset) {
3587                 hole = page_hole_offset(mapping, offset);
3588                 if (hole >= 0 && hole <= end_offset)
3589                         return max(start_offset, hole);
3590
3591                 offset += PAGE_SIZE;
3592                 offset &= PAGE_MASK;
3593         }
3594
3595         return end_offset;
3596 }
3597
3598 static loff_t bch2_seek_hole(struct file *file, u64 offset)
3599 {
3600         struct bch_inode_info *inode = file_bch_inode(file);
3601         struct bch_fs *c = inode->v.i_sb->s_fs_info;
3602         struct btree_trans trans;
3603         struct btree_iter iter;
3604         struct bkey_s_c k;
3605         subvol_inum inum = inode_inum(inode);
3606         u64 isize, next_hole = MAX_LFS_FILESIZE;
3607         u32 snapshot;
3608         int ret;
3609
3610         isize = i_size_read(&inode->v);
3611         if (offset >= isize)
3612                 return -ENXIO;
3613
3614         bch2_trans_init(&trans, c, 0, 0);
3615 retry:
3616         bch2_trans_begin(&trans);
3617
3618         ret = bch2_subvolume_get_snapshot(&trans, inum.subvol, &snapshot);
3619         if (ret)
3620                 goto err;
3621
3622         for_each_btree_key_norestart(&trans, iter, BTREE_ID_extents,
3623                            SPOS(inode->v.i_ino, offset >> 9, snapshot),
3624                            BTREE_ITER_SLOTS, k, ret) {
3625                 if (k.k->p.inode != inode->v.i_ino) {
3626                         next_hole = bch2_seek_pagecache_hole(&inode->v,
3627                                         offset, MAX_LFS_FILESIZE);
3628                         break;
3629                 } else if (!bkey_extent_is_data(k.k)) {
3630                         next_hole = bch2_seek_pagecache_hole(&inode->v,
3631                                         max(offset, bkey_start_offset(k.k) << 9),
3632                                         k.k->p.offset << 9);
3633
3634                         if (next_hole < k.k->p.offset << 9)
3635                                 break;
3636                 } else {
3637                         offset = max(offset, bkey_start_offset(k.k) << 9);
3638                 }
3639         }
3640         bch2_trans_iter_exit(&trans, &iter);
3641 err:
3642         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
3643                 goto retry;
3644
3645         bch2_trans_exit(&trans);
3646         if (ret)
3647                 return ret;
3648
3649         if (next_hole > isize)
3650                 next_hole = isize;
3651
3652         return vfs_setpos(file, next_hole, MAX_LFS_FILESIZE);
3653 }
3654
3655 loff_t bch2_llseek(struct file *file, loff_t offset, int whence)
3656 {
3657         loff_t ret;
3658
3659         switch (whence) {
3660         case SEEK_SET:
3661         case SEEK_CUR:
3662         case SEEK_END:
3663                 ret = generic_file_llseek(file, offset, whence);
3664                 break;
3665         case SEEK_DATA:
3666                 ret = bch2_seek_data(file, offset);
3667                 break;
3668         case SEEK_HOLE:
3669                 ret = bch2_seek_hole(file, offset);
3670                 break;
3671         default:
3672                 ret = -EINVAL;
3673                 break;
3674         }
3675
3676         return bch2_err_class(ret);
3677 }
3678
3679 void bch2_fs_fsio_exit(struct bch_fs *c)
3680 {
3681         bioset_exit(&c->nocow_flush_bioset);
3682         bioset_exit(&c->dio_write_bioset);
3683         bioset_exit(&c->dio_read_bioset);
3684         bioset_exit(&c->writepage_bioset);
3685 }
3686
3687 int bch2_fs_fsio_init(struct bch_fs *c)
3688 {
3689         int ret = 0;
3690
3691         pr_verbose_init(c->opts, "");
3692
3693         if (bioset_init(&c->writepage_bioset,
3694                         4, offsetof(struct bch_writepage_io, op.wbio.bio),
3695                         BIOSET_NEED_BVECS) ||
3696             bioset_init(&c->dio_read_bioset,
3697                         4, offsetof(struct dio_read, rbio.bio),
3698                         BIOSET_NEED_BVECS) ||
3699             bioset_init(&c->dio_write_bioset,
3700                         4, offsetof(struct dio_write, op.wbio.bio),
3701                         BIOSET_NEED_BVECS) ||
3702             bioset_init(&c->nocow_flush_bioset,
3703                         1, offsetof(struct nocow_flush, bio), 0))
3704                 ret = -ENOMEM;
3705
3706         pr_verbose_init(c->opts, "ret %i", ret);
3707         return ret;
3708 }
3709
3710 #endif /* NO_BCACHEFS_FS */